feat: update task routing and coze handling
This commit is contained in:
+36
@@ -0,0 +1,36 @@
|
||||
package com.nanri.aiimage.modules.task.controller;
|
||||
|
||||
import com.nanri.aiimage.common.api.ApiResponse;
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest;
|
||||
import com.nanri.aiimage.modules.task.model.vo.TaskHeartbeatVo;
|
||||
import com.nanri.aiimage.modules.task.service.TaskHeartbeatService;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.Parameter;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
import jakarta.validation.Valid;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
@RestController
|
||||
@RequiredArgsConstructor
|
||||
@RequestMapping("/api/tasks")
|
||||
@Tag(name = "Task heartbeat", description = "Unified heartbeat API for Python task workers")
|
||||
public class TaskHeartbeatController {
|
||||
|
||||
private final TaskHeartbeatService taskHeartbeatService;
|
||||
|
||||
@PostMapping("/{taskId}/heartbeat")
|
||||
@Operation(
|
||||
summary = "Task heartbeat",
|
||||
description = "Python calls this during execution. The backend resolves biz_file_task or brand task by taskId. user_id is optional.")
|
||||
public ApiResponse<TaskHeartbeatVo> heartbeat(
|
||||
@Parameter(description = "Task ID", required = true, example = "200")
|
||||
@PathVariable Long taskId,
|
||||
@Valid @RequestBody(required = false) TaskHeartbeatRequest request) {
|
||||
return ApiResponse.success(taskHeartbeatService.heartbeat(taskId, request));
|
||||
}
|
||||
}
|
||||
+7
-1
@@ -2,11 +2,14 @@ package com.nanri.aiimage.modules.task.mapper;
|
||||
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskImageCacheEntity;
|
||||
import org.apache.ibatis.annotations.Delete;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
import org.apache.ibatis.annotations.Select;
|
||||
import org.apache.ibatis.annotations.Update;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
/**
|
||||
* P2-12:图片缩略图缓存 mapper。提供 LRU 命中刷新、按 url_hash 直读字节两个轻量入口,
|
||||
* 其余 CRUD 走 {@link BaseMapper} 默认实现。
|
||||
@@ -15,7 +18,7 @@ import org.apache.ibatis.annotations.Update;
|
||||
public interface TaskImageCacheMapper extends BaseMapper<TaskImageCacheEntity> {
|
||||
|
||||
/**
|
||||
* 命中时同步 bumping last_used_at,便于后续按 LRU 清理 7 天前未用记录。
|
||||
* 命中时同步 bumping last_used_at,便于后续按 LRU 清理过期未用记录。
|
||||
*/
|
||||
@Update("UPDATE biz_task_image_cache SET last_used_at = NOW(3) WHERE url_hash = #{urlHash}")
|
||||
int touchLastUsed(@Param("urlHash") String urlHash);
|
||||
@@ -26,4 +29,7 @@ public interface TaskImageCacheMapper extends BaseMapper<TaskImageCacheEntity> {
|
||||
*/
|
||||
@Select("SELECT image_bytes FROM biz_task_image_cache WHERE url_hash = #{urlHash} LIMIT 1")
|
||||
byte[] selectBytesByUrlHash(@Param("urlHash") String urlHash);
|
||||
|
||||
@Delete("DELETE FROM biz_task_image_cache WHERE last_used_at < #{cutoff} ORDER BY last_used_at LIMIT #{limit}")
|
||||
int deleteExpiredBatch(@Param("cutoff") LocalDateTime cutoff, @Param("limit") int limit);
|
||||
}
|
||||
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
package com.nanri.aiimage.modules.task.model.dto;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@Schema(description = "Task heartbeat request")
|
||||
public class TaskHeartbeatRequest {
|
||||
|
||||
@JsonProperty("user_id")
|
||||
@Schema(description = "Optional task owner user ID. Python workers can omit it.", example = "453")
|
||||
private Long userId;
|
||||
|
||||
@Schema(description = "Optional phase label", example = "crawling")
|
||||
private String phase;
|
||||
|
||||
@Schema(description = "Optional current progress", example = "42")
|
||||
private Integer current;
|
||||
|
||||
@Schema(description = "Optional total progress", example = "1100")
|
||||
private Integer total;
|
||||
}
|
||||
+1
-1
@@ -16,7 +16,7 @@ import java.time.LocalDateTime;
|
||||
* <li>{@link #urlHash}:sha256 lowercase hex,作为唯一键避免 1024 长 URL 命中索引长度限制;</li>
|
||||
* <li>{@link #imageBytes}:已 resize 后的 JPEG 缩略图字节,最大约 300KB(保持与
|
||||
* {@code SimilarAsinImageEmbedder.MAX_THUMB_SIZE_BYTES} 对齐);</li>
|
||||
* <li>{@link #lastUsedAt}:用于后续 LRU 清理(>7 天未用)。</li>
|
||||
* <li>{@link #lastUsedAt}:用于后续 LRU 清理(默认 >3 天未用)。</li>
|
||||
* </ul>
|
||||
*/
|
||||
@Data
|
||||
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
package com.nanri.aiimage.modules.task.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@Schema(description = "Task heartbeat response")
|
||||
public class TaskHeartbeatVo {
|
||||
|
||||
@Schema(description = "Whether the task can continue", example = "true")
|
||||
private boolean alive;
|
||||
|
||||
@Schema(description = "Task status", example = "RUNNING")
|
||||
private String status;
|
||||
|
||||
@Schema(description = "Resolved task module", example = "PRODUCT_RISK_RESOLVE")
|
||||
private String moduleType;
|
||||
|
||||
@Schema(description = "Message", example = "ok")
|
||||
private String message;
|
||||
|
||||
public static TaskHeartbeatVo alive(String moduleType, String status) {
|
||||
TaskHeartbeatVo vo = new TaskHeartbeatVo();
|
||||
vo.setAlive(true);
|
||||
vo.setModuleType(moduleType);
|
||||
vo.setStatus(status);
|
||||
vo.setMessage("ok");
|
||||
return vo;
|
||||
}
|
||||
|
||||
public static TaskHeartbeatVo notAlive(String moduleType, String status, String message) {
|
||||
TaskHeartbeatVo vo = new TaskHeartbeatVo();
|
||||
vo.setAlive(false);
|
||||
vo.setModuleType(moduleType);
|
||||
vo.setStatus(status);
|
||||
vo.setMessage(message);
|
||||
return vo;
|
||||
}
|
||||
}
|
||||
+211
@@ -0,0 +1,211 @@
|
||||
package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskCacheService;
|
||||
import com.nanri.aiimage.modules.brand.mapper.BrandCrawlTaskMapper;
|
||||
import com.nanri.aiimage.modules.brand.model.entity.BrandCrawlTaskEntity;
|
||||
import com.nanri.aiimage.modules.brand.service.BrandTaskProgressCacheService;
|
||||
import com.nanri.aiimage.modules.deletebrand.service.DeleteBrandTaskCacheService;
|
||||
import com.nanri.aiimage.modules.patroldelete.service.PatrolDeleteTaskCacheService;
|
||||
import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskCacheService;
|
||||
import com.nanri.aiimage.modules.productrisk.service.ProductRiskTaskCacheService;
|
||||
import com.nanri.aiimage.modules.queryasin.service.QueryAsinTaskCacheService;
|
||||
import com.nanri.aiimage.modules.shopmatch.service.ShopMatchTaskCacheService;
|
||||
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskCacheService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.vo.TaskHeartbeatVo;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class TaskHeartbeatService {
|
||||
|
||||
private static final String STATUS_RUNNING = "RUNNING";
|
||||
private static final String BRAND_STATUS_RUNNING = "running";
|
||||
|
||||
private static final String MODULE_DELETE_BRAND = "DELETE_BRAND";
|
||||
private static final String MODULE_PRODUCT_RISK = "PRODUCT_RISK_RESOLVE";
|
||||
private static final String MODULE_PRICE_TRACK = "PRICE_TRACK";
|
||||
private static final String MODULE_SHOP_MATCH = "SHOP_MATCH";
|
||||
private static final String MODULE_PATROL_DELETE = "PATROL_DELETE";
|
||||
private static final String MODULE_QUERY_ASIN = "QUERY_ASIN";
|
||||
private static final String MODULE_APPEARANCE_PATENT = "APPEARANCE_PATENT";
|
||||
private static final String MODULE_SIMILAR_ASIN = "SIMILAR_ASIN";
|
||||
private static final String MODULE_BRAND = "BRAND";
|
||||
|
||||
private final FileTaskMapper fileTaskMapper;
|
||||
private final BrandCrawlTaskMapper brandCrawlTaskMapper;
|
||||
private final ProductRiskTaskCacheService productRiskTaskCacheService;
|
||||
private final PriceTrackTaskCacheService priceTrackTaskCacheService;
|
||||
private final ShopMatchTaskCacheService shopMatchTaskCacheService;
|
||||
private final PatrolDeleteTaskCacheService patrolDeleteTaskCacheService;
|
||||
private final QueryAsinTaskCacheService queryAsinTaskCacheService;
|
||||
private final AppearancePatentTaskCacheService appearancePatentTaskCacheService;
|
||||
private final SimilarAsinTaskCacheService similarAsinTaskCacheService;
|
||||
private final DeleteBrandTaskCacheService deleteBrandTaskCacheService;
|
||||
private final BrandTaskProgressCacheService brandTaskProgressCacheService;
|
||||
|
||||
public TaskHeartbeatVo heartbeat(Long taskId, TaskHeartbeatRequest request) {
|
||||
if (taskId == null || taskId <= 0) {
|
||||
return TaskHeartbeatVo.notAlive(null, null, "invalid taskId");
|
||||
}
|
||||
Long userId = request == null ? null : request.getUserId();
|
||||
|
||||
LambdaQueryWrapper<FileTaskEntity> fileQuery = new LambdaQueryWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getId, taskId)
|
||||
.last("limit 1");
|
||||
if (userId != null && userId > 0) {
|
||||
fileQuery.eq(FileTaskEntity::getUserId, userId);
|
||||
}
|
||||
FileTaskEntity fileTask = fileTaskMapper.selectOne(fileQuery);
|
||||
|
||||
LambdaQueryWrapper<BrandCrawlTaskEntity> brandQuery = new LambdaQueryWrapper<BrandCrawlTaskEntity>()
|
||||
.eq(BrandCrawlTaskEntity::getId, taskId)
|
||||
.last("limit 1");
|
||||
if (userId != null && userId > 0) {
|
||||
brandQuery.eq(BrandCrawlTaskEntity::getUserId, userId);
|
||||
}
|
||||
BrandCrawlTaskEntity brandTask = brandCrawlTaskMapper.selectOne(brandQuery);
|
||||
|
||||
TaskHeartbeatVo fileResult = touchFileTaskIfRunning(fileTask, request);
|
||||
TaskHeartbeatVo brandResult = touchBrandTaskIfRunning(brandTask, request);
|
||||
|
||||
if (fileResult != null && fileResult.isAlive() && brandResult != null && brandResult.isAlive()) {
|
||||
return TaskHeartbeatVo.alive(fileResult.getModuleType() + "," + MODULE_BRAND, STATUS_RUNNING);
|
||||
}
|
||||
if (fileResult != null && fileResult.isAlive()) {
|
||||
return fileResult;
|
||||
}
|
||||
if (brandResult != null && brandResult.isAlive()) {
|
||||
return brandResult;
|
||||
}
|
||||
if (fileResult != null) {
|
||||
return fileResult;
|
||||
}
|
||||
if (brandResult != null) {
|
||||
return brandResult;
|
||||
}
|
||||
return TaskHeartbeatVo.notAlive(null, null, "task not found");
|
||||
}
|
||||
|
||||
private TaskHeartbeatVo touchFileTaskIfRunning(FileTaskEntity task, TaskHeartbeatRequest request) {
|
||||
if (task == null) {
|
||||
return null;
|
||||
}
|
||||
String moduleType = task.getModuleType() == null ? "" : task.getModuleType();
|
||||
String status = task.getStatus();
|
||||
if (!STATUS_RUNNING.equals(status)) {
|
||||
return TaskHeartbeatVo.notAlive(moduleType, status, "task is not running");
|
||||
}
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||
.eq(FileTaskEntity::getId, task.getId())
|
||||
.eq(FileTaskEntity::getUserId, task.getUserId())
|
||||
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
|
||||
.set(FileTaskEntity::getUpdatedAt, now));
|
||||
if (updated <= 0) {
|
||||
FileTaskEntity latest = fileTaskMapper.selectById(task.getId());
|
||||
return TaskHeartbeatVo.notAlive(moduleType, latest == null ? status : latest.getStatus(), "task is not running");
|
||||
}
|
||||
touchModuleHeartbeat(moduleType, task.getId(), request);
|
||||
task.setUpdatedAt(now);
|
||||
saveFileTaskCache(moduleType, task);
|
||||
return TaskHeartbeatVo.alive(moduleType, STATUS_RUNNING);
|
||||
}
|
||||
|
||||
private TaskHeartbeatVo touchBrandTaskIfRunning(BrandCrawlTaskEntity task, TaskHeartbeatRequest request) {
|
||||
if (task == null) {
|
||||
return null;
|
||||
}
|
||||
String status = task.getStatus();
|
||||
if (!BRAND_STATUS_RUNNING.equals(status)) {
|
||||
return TaskHeartbeatVo.notAlive(MODULE_BRAND, status, "task is not running");
|
||||
}
|
||||
int updated = brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper<BrandCrawlTaskEntity>()
|
||||
.eq(BrandCrawlTaskEntity::getId, task.getId())
|
||||
.eq(BrandCrawlTaskEntity::getUserId, task.getUserId())
|
||||
.eq(BrandCrawlTaskEntity::getStatus, BRAND_STATUS_RUNNING)
|
||||
.set(BrandCrawlTaskEntity::getUpdatedAt, LocalDateTime.now()));
|
||||
if (updated <= 0) {
|
||||
BrandCrawlTaskEntity latest = brandCrawlTaskMapper.selectById(task.getId());
|
||||
return TaskHeartbeatVo.notAlive(MODULE_BRAND, latest == null ? status : latest.getStatus(), "task is not running");
|
||||
}
|
||||
brandTaskProgressCacheService.touchHeartbeat(
|
||||
task.getId(),
|
||||
request == null ? null : request.getPhase(),
|
||||
request == null ? null : request.getCurrent(),
|
||||
request == null ? null : request.getTotal());
|
||||
return TaskHeartbeatVo.alive(MODULE_BRAND, BRAND_STATUS_RUNNING);
|
||||
}
|
||||
|
||||
private void touchModuleHeartbeat(String moduleType, Long taskId, TaskHeartbeatRequest request) {
|
||||
switch (moduleType) {
|
||||
case MODULE_PRODUCT_RISK -> {
|
||||
productRiskTaskCacheService.touchTaskHeartbeat(taskId);
|
||||
}
|
||||
case MODULE_PRICE_TRACK -> {
|
||||
priceTrackTaskCacheService.touchTaskHeartbeat(taskId);
|
||||
}
|
||||
case MODULE_SHOP_MATCH -> {
|
||||
shopMatchTaskCacheService.touchTaskHeartbeat(taskId);
|
||||
}
|
||||
case MODULE_PATROL_DELETE -> {
|
||||
patrolDeleteTaskCacheService.touchTaskHeartbeat(taskId);
|
||||
}
|
||||
case MODULE_QUERY_ASIN -> {
|
||||
queryAsinTaskCacheService.touchTaskHeartbeat(taskId);
|
||||
}
|
||||
case MODULE_APPEARANCE_PATENT -> {
|
||||
appearancePatentTaskCacheService.touchTaskHeartbeat(taskId);
|
||||
}
|
||||
case MODULE_SIMILAR_ASIN -> {
|
||||
similarAsinTaskCacheService.touchTaskHeartbeat(taskId);
|
||||
}
|
||||
case MODULE_DELETE_BRAND -> {
|
||||
deleteBrandTaskCacheService.saveProgress(taskId, buildDeleteBrandHeartbeatProgress(request), true);
|
||||
}
|
||||
default -> {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String, String> buildDeleteBrandHeartbeatProgress(TaskHeartbeatRequest request) {
|
||||
String now = String.valueOf(System.currentTimeMillis());
|
||||
Map<String, String> values = new LinkedHashMap<>();
|
||||
values.put("last_heartbeat_at", now);
|
||||
values.put("updated_at", now);
|
||||
if (request != null) {
|
||||
putIfPresent(values, "phase", request.getPhase());
|
||||
putIfPresent(values, "current", request.getCurrent());
|
||||
putIfPresent(values, "total", request.getTotal());
|
||||
}
|
||||
return values;
|
||||
}
|
||||
|
||||
private void saveFileTaskCache(String moduleType, FileTaskEntity task) {
|
||||
switch (moduleType) {
|
||||
case MODULE_PRODUCT_RISK -> productRiskTaskCacheService.saveTaskCache(task);
|
||||
case MODULE_PRICE_TRACK -> priceTrackTaskCacheService.saveTaskCache(task);
|
||||
case MODULE_SHOP_MATCH -> shopMatchTaskCacheService.saveTaskCache(task);
|
||||
case MODULE_PATROL_DELETE -> patrolDeleteTaskCacheService.saveTaskCache(task);
|
||||
case MODULE_QUERY_ASIN -> queryAsinTaskCacheService.saveTaskCache(task);
|
||||
case MODULE_DELETE_BRAND -> deleteBrandTaskCacheService.saveTaskCache(task);
|
||||
default -> {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void putIfPresent(Map<String, String> values, String key, Object value) {
|
||||
if (value != null) {
|
||||
values.put(key, String.valueOf(value));
|
||||
}
|
||||
}
|
||||
}
|
||||
+69
@@ -0,0 +1,69 @@
|
||||
package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.config.TaskImageCacheCleanupProperties;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskImageCacheMapper;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class TaskImageCacheCleanupService {
|
||||
|
||||
private static final Duration CLEANUP_LOCK_TTL = Duration.ofMinutes(30);
|
||||
|
||||
private final TaskImageCacheMapper taskImageCacheMapper;
|
||||
private final TaskImageCacheCleanupProperties cleanupProperties;
|
||||
private final DistributedJobLockService distributedJobLockService;
|
||||
|
||||
@Scheduled(cron = "${aiimage.task-image-cache-cleanup.cron:0 30 2 * * *}")
|
||||
public void cleanupExpiredImageCache() {
|
||||
if (!cleanupProperties.isEnabled()) {
|
||||
return;
|
||||
}
|
||||
DistributedJobLockService.LockHandle lockHandle =
|
||||
distributedJobLockService.tryLock("task-image-cache:cleanup", CLEANUP_LOCK_TTL);
|
||||
if (lockHandle == null) {
|
||||
log.info("[task-image-cache-cleanup] skip because another instance holds the distributed lock");
|
||||
return;
|
||||
}
|
||||
try (lockHandle) {
|
||||
int retentionDays = Math.max(1, cleanupProperties.getRetentionDays());
|
||||
int batchSize = Math.max(1, cleanupProperties.getBatchSize());
|
||||
int maxBatches = Math.max(1, cleanupProperties.getMaxBatchesPerRun());
|
||||
LocalDateTime cutoff = LocalDateTime.now().minusDays(retentionDays);
|
||||
|
||||
int deletedTotal = 0;
|
||||
int batches = 0;
|
||||
while (batches < maxBatches) {
|
||||
int deleted = taskImageCacheMapper.deleteExpiredBatch(cutoff, batchSize);
|
||||
if (deleted <= 0) {
|
||||
break;
|
||||
}
|
||||
deletedTotal += deleted;
|
||||
batches++;
|
||||
if (!lockHandle.renew(CLEANUP_LOCK_TTL)) {
|
||||
log.warn("[task-image-cache-cleanup] stop because lock renew failed deletedTotal={} batches={}",
|
||||
deletedTotal, batches);
|
||||
break;
|
||||
}
|
||||
if (deleted < batchSize) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (deletedTotal > 0 || batches >= maxBatches) {
|
||||
log.info("[task-image-cache-cleanup] finished cutoff={} retentionDays={} deletedTotal={} batches={} batchSize={} maxBatches={}",
|
||||
cutoff, retentionDays, deletedTotal, batches, batchSize, maxBatches);
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
log.warn("[task-image-cache-cleanup] failed msg={}", ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user