feat: update task routing and coze handling

This commit is contained in:
super
2026-06-03 17:14:50 +08:00
parent 7310abd769
commit e540d8e765
37 changed files with 1883 additions and 254 deletions
@@ -0,0 +1,36 @@
package com.nanri.aiimage.modules.task.controller;
import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest;
import com.nanri.aiimage.modules.task.model.vo.TaskHeartbeatVo;
import com.nanri.aiimage.modules.task.service.TaskHeartbeatService;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
@RestController
@RequiredArgsConstructor
@RequestMapping("/api/tasks")
@Tag(name = "Task heartbeat", description = "Unified heartbeat API for Python task workers")
public class TaskHeartbeatController {
private final TaskHeartbeatService taskHeartbeatService;
@PostMapping("/{taskId}/heartbeat")
@Operation(
summary = "Task heartbeat",
description = "Python calls this during execution. The backend resolves biz_file_task or brand task by taskId. user_id is optional.")
public ApiResponse<TaskHeartbeatVo> heartbeat(
@Parameter(description = "Task ID", required = true, example = "200")
@PathVariable Long taskId,
@Valid @RequestBody(required = false) TaskHeartbeatRequest request) {
return ApiResponse.success(taskHeartbeatService.heartbeat(taskId, request));
}
}
@@ -2,11 +2,14 @@ package com.nanri.aiimage.modules.task.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskImageCacheEntity;
import org.apache.ibatis.annotations.Delete;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Select;
import org.apache.ibatis.annotations.Update;
import java.time.LocalDateTime;
/**
* P2-12:图片缩略图缓存 mapper。提供 LRU 命中刷新、按 url_hash 直读字节两个轻量入口,
* 其余 CRUD 走 {@link BaseMapper} 默认实现。
@@ -15,7 +18,7 @@ import org.apache.ibatis.annotations.Update;
public interface TaskImageCacheMapper extends BaseMapper<TaskImageCacheEntity> {
/**
* 命中时同步 bumping last_used_at,便于后续按 LRU 清理 7 天前未用记录。
* 命中时同步 bumping last_used_at,便于后续按 LRU 清理过期未用记录。
*/
@Update("UPDATE biz_task_image_cache SET last_used_at = NOW(3) WHERE url_hash = #{urlHash}")
int touchLastUsed(@Param("urlHash") String urlHash);
@@ -26,4 +29,7 @@ public interface TaskImageCacheMapper extends BaseMapper<TaskImageCacheEntity> {
*/
@Select("SELECT image_bytes FROM biz_task_image_cache WHERE url_hash = #{urlHash} LIMIT 1")
byte[] selectBytesByUrlHash(@Param("urlHash") String urlHash);
@Delete("DELETE FROM biz_task_image_cache WHERE last_used_at < #{cutoff} ORDER BY last_used_at LIMIT #{limit}")
int deleteExpiredBatch(@Param("cutoff") LocalDateTime cutoff, @Param("limit") int limit);
}
@@ -0,0 +1,23 @@
package com.nanri.aiimage.modules.task.model.dto;
import com.fasterxml.jackson.annotation.JsonProperty;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@Schema(description = "Task heartbeat request")
public class TaskHeartbeatRequest {
@JsonProperty("user_id")
@Schema(description = "Optional task owner user ID. Python workers can omit it.", example = "453")
private Long userId;
@Schema(description = "Optional phase label", example = "crawling")
private String phase;
@Schema(description = "Optional current progress", example = "42")
private Integer current;
@Schema(description = "Optional total progress", example = "1100")
private Integer total;
}
@@ -16,7 +16,7 @@ import java.time.LocalDateTime;
* <li>{@link #urlHash}:sha256 lowercase hex,作为唯一键避免 1024 长 URL 命中索引长度限制;</li>
* <li>{@link #imageBytes}:已 resize 后的 JPEG 缩略图字节,最大约 300KB(保持与
* {@code SimilarAsinImageEmbedder.MAX_THUMB_SIZE_BYTES} 对齐);</li>
* <li>{@link #lastUsedAt}:用于后续 LRU 清理(>7 天未用)。</li>
* <li>{@link #lastUsedAt}:用于后续 LRU 清理(默认 >3 天未用)。</li>
* </ul>
*/
@Data
@@ -0,0 +1,39 @@
package com.nanri.aiimage.modules.task.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@Schema(description = "Task heartbeat response")
public class TaskHeartbeatVo {
@Schema(description = "Whether the task can continue", example = "true")
private boolean alive;
@Schema(description = "Task status", example = "RUNNING")
private String status;
@Schema(description = "Resolved task module", example = "PRODUCT_RISK_RESOLVE")
private String moduleType;
@Schema(description = "Message", example = "ok")
private String message;
public static TaskHeartbeatVo alive(String moduleType, String status) {
TaskHeartbeatVo vo = new TaskHeartbeatVo();
vo.setAlive(true);
vo.setModuleType(moduleType);
vo.setStatus(status);
vo.setMessage("ok");
return vo;
}
public static TaskHeartbeatVo notAlive(String moduleType, String status, String message) {
TaskHeartbeatVo vo = new TaskHeartbeatVo();
vo.setAlive(false);
vo.setModuleType(moduleType);
vo.setStatus(status);
vo.setMessage(message);
return vo;
}
}
@@ -0,0 +1,211 @@
package com.nanri.aiimage.modules.task.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskCacheService;
import com.nanri.aiimage.modules.brand.mapper.BrandCrawlTaskMapper;
import com.nanri.aiimage.modules.brand.model.entity.BrandCrawlTaskEntity;
import com.nanri.aiimage.modules.brand.service.BrandTaskProgressCacheService;
import com.nanri.aiimage.modules.deletebrand.service.DeleteBrandTaskCacheService;
import com.nanri.aiimage.modules.patroldelete.service.PatrolDeleteTaskCacheService;
import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskCacheService;
import com.nanri.aiimage.modules.productrisk.service.ProductRiskTaskCacheService;
import com.nanri.aiimage.modules.queryasin.service.QueryAsinTaskCacheService;
import com.nanri.aiimage.modules.shopmatch.service.ShopMatchTaskCacheService;
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskCacheService;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.vo.TaskHeartbeatVo;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import java.time.LocalDateTime;
import java.util.LinkedHashMap;
import java.util.Map;
@Service
@RequiredArgsConstructor
public class TaskHeartbeatService {
private static final String STATUS_RUNNING = "RUNNING";
private static final String BRAND_STATUS_RUNNING = "running";
private static final String MODULE_DELETE_BRAND = "DELETE_BRAND";
private static final String MODULE_PRODUCT_RISK = "PRODUCT_RISK_RESOLVE";
private static final String MODULE_PRICE_TRACK = "PRICE_TRACK";
private static final String MODULE_SHOP_MATCH = "SHOP_MATCH";
private static final String MODULE_PATROL_DELETE = "PATROL_DELETE";
private static final String MODULE_QUERY_ASIN = "QUERY_ASIN";
private static final String MODULE_APPEARANCE_PATENT = "APPEARANCE_PATENT";
private static final String MODULE_SIMILAR_ASIN = "SIMILAR_ASIN";
private static final String MODULE_BRAND = "BRAND";
private final FileTaskMapper fileTaskMapper;
private final BrandCrawlTaskMapper brandCrawlTaskMapper;
private final ProductRiskTaskCacheService productRiskTaskCacheService;
private final PriceTrackTaskCacheService priceTrackTaskCacheService;
private final ShopMatchTaskCacheService shopMatchTaskCacheService;
private final PatrolDeleteTaskCacheService patrolDeleteTaskCacheService;
private final QueryAsinTaskCacheService queryAsinTaskCacheService;
private final AppearancePatentTaskCacheService appearancePatentTaskCacheService;
private final SimilarAsinTaskCacheService similarAsinTaskCacheService;
private final DeleteBrandTaskCacheService deleteBrandTaskCacheService;
private final BrandTaskProgressCacheService brandTaskProgressCacheService;
public TaskHeartbeatVo heartbeat(Long taskId, TaskHeartbeatRequest request) {
if (taskId == null || taskId <= 0) {
return TaskHeartbeatVo.notAlive(null, null, "invalid taskId");
}
Long userId = request == null ? null : request.getUserId();
LambdaQueryWrapper<FileTaskEntity> fileQuery = new LambdaQueryWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getId, taskId)
.last("limit 1");
if (userId != null && userId > 0) {
fileQuery.eq(FileTaskEntity::getUserId, userId);
}
FileTaskEntity fileTask = fileTaskMapper.selectOne(fileQuery);
LambdaQueryWrapper<BrandCrawlTaskEntity> brandQuery = new LambdaQueryWrapper<BrandCrawlTaskEntity>()
.eq(BrandCrawlTaskEntity::getId, taskId)
.last("limit 1");
if (userId != null && userId > 0) {
brandQuery.eq(BrandCrawlTaskEntity::getUserId, userId);
}
BrandCrawlTaskEntity brandTask = brandCrawlTaskMapper.selectOne(brandQuery);
TaskHeartbeatVo fileResult = touchFileTaskIfRunning(fileTask, request);
TaskHeartbeatVo brandResult = touchBrandTaskIfRunning(brandTask, request);
if (fileResult != null && fileResult.isAlive() && brandResult != null && brandResult.isAlive()) {
return TaskHeartbeatVo.alive(fileResult.getModuleType() + "," + MODULE_BRAND, STATUS_RUNNING);
}
if (fileResult != null && fileResult.isAlive()) {
return fileResult;
}
if (brandResult != null && brandResult.isAlive()) {
return brandResult;
}
if (fileResult != null) {
return fileResult;
}
if (brandResult != null) {
return brandResult;
}
return TaskHeartbeatVo.notAlive(null, null, "task not found");
}
private TaskHeartbeatVo touchFileTaskIfRunning(FileTaskEntity task, TaskHeartbeatRequest request) {
if (task == null) {
return null;
}
String moduleType = task.getModuleType() == null ? "" : task.getModuleType();
String status = task.getStatus();
if (!STATUS_RUNNING.equals(status)) {
return TaskHeartbeatVo.notAlive(moduleType, status, "task is not running");
}
LocalDateTime now = LocalDateTime.now();
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getId, task.getId())
.eq(FileTaskEntity::getUserId, task.getUserId())
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
.set(FileTaskEntity::getUpdatedAt, now));
if (updated <= 0) {
FileTaskEntity latest = fileTaskMapper.selectById(task.getId());
return TaskHeartbeatVo.notAlive(moduleType, latest == null ? status : latest.getStatus(), "task is not running");
}
touchModuleHeartbeat(moduleType, task.getId(), request);
task.setUpdatedAt(now);
saveFileTaskCache(moduleType, task);
return TaskHeartbeatVo.alive(moduleType, STATUS_RUNNING);
}
private TaskHeartbeatVo touchBrandTaskIfRunning(BrandCrawlTaskEntity task, TaskHeartbeatRequest request) {
if (task == null) {
return null;
}
String status = task.getStatus();
if (!BRAND_STATUS_RUNNING.equals(status)) {
return TaskHeartbeatVo.notAlive(MODULE_BRAND, status, "task is not running");
}
int updated = brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper<BrandCrawlTaskEntity>()
.eq(BrandCrawlTaskEntity::getId, task.getId())
.eq(BrandCrawlTaskEntity::getUserId, task.getUserId())
.eq(BrandCrawlTaskEntity::getStatus, BRAND_STATUS_RUNNING)
.set(BrandCrawlTaskEntity::getUpdatedAt, LocalDateTime.now()));
if (updated <= 0) {
BrandCrawlTaskEntity latest = brandCrawlTaskMapper.selectById(task.getId());
return TaskHeartbeatVo.notAlive(MODULE_BRAND, latest == null ? status : latest.getStatus(), "task is not running");
}
brandTaskProgressCacheService.touchHeartbeat(
task.getId(),
request == null ? null : request.getPhase(),
request == null ? null : request.getCurrent(),
request == null ? null : request.getTotal());
return TaskHeartbeatVo.alive(MODULE_BRAND, BRAND_STATUS_RUNNING);
}
private void touchModuleHeartbeat(String moduleType, Long taskId, TaskHeartbeatRequest request) {
switch (moduleType) {
case MODULE_PRODUCT_RISK -> {
productRiskTaskCacheService.touchTaskHeartbeat(taskId);
}
case MODULE_PRICE_TRACK -> {
priceTrackTaskCacheService.touchTaskHeartbeat(taskId);
}
case MODULE_SHOP_MATCH -> {
shopMatchTaskCacheService.touchTaskHeartbeat(taskId);
}
case MODULE_PATROL_DELETE -> {
patrolDeleteTaskCacheService.touchTaskHeartbeat(taskId);
}
case MODULE_QUERY_ASIN -> {
queryAsinTaskCacheService.touchTaskHeartbeat(taskId);
}
case MODULE_APPEARANCE_PATENT -> {
appearancePatentTaskCacheService.touchTaskHeartbeat(taskId);
}
case MODULE_SIMILAR_ASIN -> {
similarAsinTaskCacheService.touchTaskHeartbeat(taskId);
}
case MODULE_DELETE_BRAND -> {
deleteBrandTaskCacheService.saveProgress(taskId, buildDeleteBrandHeartbeatProgress(request), true);
}
default -> {
}
}
}
private Map<String, String> buildDeleteBrandHeartbeatProgress(TaskHeartbeatRequest request) {
String now = String.valueOf(System.currentTimeMillis());
Map<String, String> values = new LinkedHashMap<>();
values.put("last_heartbeat_at", now);
values.put("updated_at", now);
if (request != null) {
putIfPresent(values, "phase", request.getPhase());
putIfPresent(values, "current", request.getCurrent());
putIfPresent(values, "total", request.getTotal());
}
return values;
}
private void saveFileTaskCache(String moduleType, FileTaskEntity task) {
switch (moduleType) {
case MODULE_PRODUCT_RISK -> productRiskTaskCacheService.saveTaskCache(task);
case MODULE_PRICE_TRACK -> priceTrackTaskCacheService.saveTaskCache(task);
case MODULE_SHOP_MATCH -> shopMatchTaskCacheService.saveTaskCache(task);
case MODULE_PATROL_DELETE -> patrolDeleteTaskCacheService.saveTaskCache(task);
case MODULE_QUERY_ASIN -> queryAsinTaskCacheService.saveTaskCache(task);
case MODULE_DELETE_BRAND -> deleteBrandTaskCacheService.saveTaskCache(task);
default -> {
}
}
}
private void putIfPresent(Map<String, String> values, String key, Object value) {
if (value != null) {
values.put(key, String.valueOf(value));
}
}
}
@@ -0,0 +1,69 @@
package com.nanri.aiimage.modules.task.service;
import com.nanri.aiimage.common.service.DistributedJobLockService;
import com.nanri.aiimage.config.TaskImageCacheCleanupProperties;
import com.nanri.aiimage.modules.task.mapper.TaskImageCacheMapper;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import java.time.Duration;
import java.time.LocalDateTime;
@Service
@RequiredArgsConstructor
@Slf4j
public class TaskImageCacheCleanupService {
private static final Duration CLEANUP_LOCK_TTL = Duration.ofMinutes(30);
private final TaskImageCacheMapper taskImageCacheMapper;
private final TaskImageCacheCleanupProperties cleanupProperties;
private final DistributedJobLockService distributedJobLockService;
@Scheduled(cron = "${aiimage.task-image-cache-cleanup.cron:0 30 2 * * *}")
public void cleanupExpiredImageCache() {
if (!cleanupProperties.isEnabled()) {
return;
}
DistributedJobLockService.LockHandle lockHandle =
distributedJobLockService.tryLock("task-image-cache:cleanup", CLEANUP_LOCK_TTL);
if (lockHandle == null) {
log.info("[task-image-cache-cleanup] skip because another instance holds the distributed lock");
return;
}
try (lockHandle) {
int retentionDays = Math.max(1, cleanupProperties.getRetentionDays());
int batchSize = Math.max(1, cleanupProperties.getBatchSize());
int maxBatches = Math.max(1, cleanupProperties.getMaxBatchesPerRun());
LocalDateTime cutoff = LocalDateTime.now().minusDays(retentionDays);
int deletedTotal = 0;
int batches = 0;
while (batches < maxBatches) {
int deleted = taskImageCacheMapper.deleteExpiredBatch(cutoff, batchSize);
if (deleted <= 0) {
break;
}
deletedTotal += deleted;
batches++;
if (!lockHandle.renew(CLEANUP_LOCK_TTL)) {
log.warn("[task-image-cache-cleanup] stop because lock renew failed deletedTotal={} batches={}",
deletedTotal, batches);
break;
}
if (deleted < batchSize) {
break;
}
}
if (deletedTotal > 0 || batches >= maxBatches) {
log.info("[task-image-cache-cleanup] finished cutoff={} retentionDays={} deletedTotal={} batches={} batchSize={} maxBatches={}",
cutoff, retentionDays, deletedTotal, batches, batchSize, maxBatches);
}
} catch (Exception ex) {
log.warn("[task-image-cache-cleanup] failed msg={}", ex.getMessage(), ex);
}
}
}