super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

后端架构更新

Browse Source
This commit is contained in:
super
2026-04-27 09:18:10 +08:00
parent 03e697f5d3
commit 225b525ba1
150 changed files with 7013 additions and 804 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@@ -43,3 +43,5 @@ app/assets/
app/new_web_source
app/user_data/
OPS_REDIS_MYSQL_OPTIMIZATION_NOTES.md
架构.md

5
app/.env
View File

@@ -13,7 +13,8 @@ client_name=ShuFuAI
# java_api_base=http://47.111.163.154:18080
java_api_base=http://127.0.0.1:18080
# java_api_base=http://8.136.19.173:18080
# java_api_base=http://127.0.0.1:18080
java_api_base=http://8.136.19.173:18080
# java_api_base=http://121.196.149.225:18080

BIN
app/amazon/__pycache__/main.cpython-312.pyc
View File

Binary file not shown.

2
app/assets/convert.js
View File

File diff suppressed because one or more lines are too long

2
app/assets/dedupe.js
View File

File diff suppressed because one or more lines are too long

2
app/assets/delete-brand.js
View File

File diff suppressed because one or more lines are too long

2
app/assets/split.js
View File

File diff suppressed because one or more lines are too long

8
app/new_web_source/convert.html
View File

@@ -5,10 +5,10 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>格式转换 - 数富AI</title>
<script type="module" crossorigin src="/assets/convert.js"></script>
<link rel="modulepreload" crossorigin href="/assets/pywebview-CeWJDVeG.js">
<link rel="modulepreload" crossorigin href="/assets/brand-UU-ckLq6.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-ij4pgMq8.css">
<link rel="stylesheet" crossorigin href="/assets/convert-7wWJ02Tw.css">
<link rel="modulepreload" crossorigin href="/assets/pywebview-C66x_2Dh.js">
<link rel="modulepreload" crossorigin href="/assets/brand-BZije8D7.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-Dp5dN8OO.css">
<link rel="stylesheet" crossorigin href="/assets/convert-BKSNvX8i.css">
</head>
<body>
<div id="app"></div>

8
app/new_web_source/dedupe.html
View File

@@ -5,10 +5,10 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>数据去重 - 数富AI</title>
<script type="module" crossorigin src="/assets/dedupe.js"></script>
<link rel="modulepreload" crossorigin href="/assets/pywebview-CeWJDVeG.js">
<link rel="modulepreload" crossorigin href="/assets/brand-UU-ckLq6.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-ij4pgMq8.css">
<link rel="stylesheet" crossorigin href="/assets/dedupe-BpNHwt51.css">
<link rel="modulepreload" crossorigin href="/assets/pywebview-C66x_2Dh.js">
<link rel="modulepreload" crossorigin href="/assets/brand-BZije8D7.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-Dp5dN8OO.css">
<link rel="stylesheet" crossorigin href="/assets/dedupe-DNlVfFj-.css">
</head>
<body>
<div id="app"></div>

8
app/new_web_source/delete-brand.html
View File

@@ -5,10 +5,10 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>删除品牌 - 数富AI</title>
<script type="module" crossorigin src="/assets/delete-brand.js"></script>
<link rel="modulepreload" crossorigin href="/assets/pywebview-CeWJDVeG.js">
<link rel="modulepreload" crossorigin href="/assets/brand-UU-ckLq6.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-ij4pgMq8.css">
<link rel="stylesheet" crossorigin href="/assets/delete-brand-CWLpe7lu.css">
<link rel="modulepreload" crossorigin href="/assets/pywebview-C66x_2Dh.js">
<link rel="modulepreload" crossorigin href="/assets/brand-BZije8D7.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-Dp5dN8OO.css">
<link rel="stylesheet" crossorigin href="/assets/delete-brand-BP3XWKAC.css">
</head>
<body>
<div id="app"></div>

8
app/new_web_source/split.html
View File

@@ -5,10 +5,10 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>数据拆分 - 数富AI</title>
<script type="module" crossorigin src="/assets/split.js"></script>
<link rel="modulepreload" crossorigin href="/assets/pywebview-CeWJDVeG.js">
<link rel="modulepreload" crossorigin href="/assets/brand-UU-ckLq6.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-ij4pgMq8.css">
<link rel="stylesheet" crossorigin href="/assets/split-CRUIYKS6.css">
<link rel="modulepreload" crossorigin href="/assets/pywebview-C66x_2Dh.js">
<link rel="modulepreload" crossorigin href="/assets/brand-BZije8D7.js">
<link rel="stylesheet" crossorigin href="/assets/pywebview-Dp5dN8OO.css">
<link rel="stylesheet" crossorigin href="/assets/split-BoBVrLdC.css">
</head>
<body>
<div id="app"></div>

30
app/web_source/brand.html
View File

@@ -784,6 +784,23 @@
else if (status === 'cancelled') showToast('已取消');
}
function verifyTerminalTaskStatus(taskId, expectedStatus) {
return fetch('/api/brand/tasks/' + taskId, {
credentials: 'include',
headers: withUidHeaders({ 'X-Requested-With': 'XMLHttpRequest' })
})
.then(function(r) { return r.json(); })
.then(function(res) {
if (!res.success || !res.task) return null;
var actualStatus = (res.task.status || '').toLowerCase();
if (actualStatus === 'success' || actualStatus === 'failed' || actualStatus === 'cancelled') {
return actualStatus;
}
return null;
})
.catch(function() { return null; });
}
function pollRunTask(taskId) {
pollTaskId = taskId;
var runProgressFill = document.getElementById('runProgressFill');
@@ -850,12 +867,15 @@
}
if (terminalStatus) {
var verifiedStatus = await verifyTerminalTaskStatus(taskId, terminalStatus);
if (verifiedStatus) {
try { runTaskEventAbortController.abort(); } catch (e2) {}
runTaskEventAbortController = null;
onRunTaskFinished(terminalStatus);
onRunTaskFinished(verifiedStatus);
return;
}
}
}
} catch (err) {}
})();
@@ -917,7 +937,7 @@
});
}
tick();
pollTimer = setInterval(tick, 10000);
pollTimer = setInterval(tick, 3000);
}
document.getElementById('btnRun').onclick = function() {
@@ -1127,7 +1147,11 @@
});
loadTasks();
// setInterval(loadTasks, 10000);
setInterval(function() {
if (cachedTasks.some(function(t) { return (t.status || '').toLowerCase() === 'running'; })) {
loadTasks();
}
}, 3000);
if (window.addEventListener) {
window.addEventListener('pywebviewready', function() {

1106
backend-java/jstack-33832.txt Normal file
View File

File diff suppressed because it is too large Load Diff

12
backend-java/pom.xml
View File

@@ -23,6 +23,8 @@
<easyexcel.version>4.0.3</easyexcel.version>
<aliyun.oss.version>3.17.4</aliyun.oss.version>
<hutool.version>5.8.36</hutool.version>
<rocketmq-spring.version>2.3.5</rocketmq-spring.version>
<minio.version>8.5.17</minio.version>
</properties>
<dependencies>
@@ -81,6 +83,16 @@
<artifactId>hutool-all</artifactId>
<version>${hutool.version}</version>
</dependency>
<dependency>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-spring-boot-starter</artifactId>
<version>${rocketmq-spring.version}</version>
</dependency>
<dependency>
<groupId>io.minio</groupId>
<artifactId>minio</artifactId>
<version>${minio.version}</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>

3
backend-java/src/main/java/com/nanri/aiimage/common/exception/GlobalExceptionHandler.java
View File

@@ -13,6 +13,9 @@ public class GlobalExceptionHandler {
@ExceptionHandler(BusinessException.class)
public ApiResponse<Void> handleBusinessException(BusinessException ex) {
if (Integer.valueOf(40901).equals(ex.getCode())) {
return ApiResponse.success("任务已结束，忽略重复提交", null);
}
return ex.getCode() == null
? ApiResponse.fail(ex.getMessage())
: ApiResponse.fail(ex.getCode(), ex.getMessage());

8
backend-java/src/main/java/com/nanri/aiimage/common/service/DistributedJobLockService.java
View File

@@ -36,7 +36,13 @@ public class DistributedJobLockService {
Duration actualTtl = (ttl == null || ttl.isNegative() || ttl.isZero()) ? Duration.ofMinutes(10) : ttl;
String key = buildLockKey(jobName);
String token = ownerPrefix + ":" + UUID.randomUUID();
Boolean locked = stringRedisTemplate.opsForValue().setIfAbsent(key, token, actualTtl);
Boolean locked;
try {
locked = stringRedisTemplate.opsForValue().setIfAbsent(key, token, actualTtl);
} catch (Exception ex) {
log.warn("[job-lock] acquire skipped jobName={} key={} msg={}", jobName, key, ex.getMessage());
return null;
}
if (!Boolean.TRUE.equals(locked)) {
return null;
}

16
backend-java/src/main/java/com/nanri/aiimage/config/AppearancePatentProperties.java Normal file
View File

@@ -0,0 +1,16 @@
package com.nanri.aiimage.config;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
@Data
@ConfigurationProperties(prefix = "aiimage.appearance-patent")
public class AppearancePatentProperties {
private String cozeBaseUrl = "https://api.coze.cn";
private String cozeWorkflowPath = "/v1/workflow/run";
private String cozeWorkflowId = "7632683471312355338";
private String cozeToken = "";
private int cozeBatchSize = 10;
private int staleTimeoutMinutes = 20;
private String staleFinalizeCron = "0 */2 * * * *";
}

2
backend-java/src/main/java/com/nanri/aiimage/config/PropertiesConfig.java
View File

@@ -4,6 +4,6 @@ import org.springframework.boot.context.properties.EnableConfigurationProperties
import org.springframework.context.annotation.Configuration;
@Configuration
@EnableConfigurationProperties({OssProperties.class, StorageProperties.class, BrandProgressProperties.class, DeleteBrandProgressProperties.class, ZiniaoProperties.class, ModuleCleanupProperties.class, TaskPressureProperties.class})
@EnableConfigurationProperties({OssProperties.class, TransientStorageProperties.class, StorageProperties.class, BrandProgressProperties.class, DeleteBrandProgressProperties.class, ZiniaoProperties.class, ModuleCleanupProperties.class, TaskPressureProperties.class, AppearancePatentProperties.class})
public class PropertiesConfig {
}

27
backend-java/src/main/java/com/nanri/aiimage/config/TaskFileJobConfig.java Normal file
View File

@@ -0,0 +1,27 @@
package com.nanri.aiimage.config;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.task.TaskExecutor;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
@Configuration
public class TaskFileJobConfig {
@Bean("taskFileJobDispatchExecutor")
public TaskExecutor taskFileJobDispatchExecutor(
@Value("${aiimage.result-file-job.local-dispatch-pool-size:2}") int poolSize,
@Value("${aiimage.result-file-job.local-dispatch-queue-capacity:200}") int queueCapacity) {
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
int normalizedPoolSize = Math.max(1, poolSize);
executor.setCorePoolSize(normalizedPoolSize);
executor.setMaxPoolSize(normalizedPoolSize);
executor.setQueueCapacity(Math.max(10, queueCapacity));
executor.setThreadNamePrefix("task-file-job-dispatch-");
executor.setWaitForTasksToCompleteOnShutdown(true);
executor.setAwaitTerminationSeconds(30);
executor.initialize();
return executor;
}
}

15
backend-java/src/main/java/com/nanri/aiimage/config/TransientStorageProperties.java Normal file
View File

@@ -0,0 +1,15 @@
package com.nanri.aiimage.config;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
@Data
@ConfigurationProperties(prefix = "aiimage.transient-storage")
public class TransientStorageProperties {
private boolean enabled = false;
private String endpoint;
private String bucket;
private String accessKeyId;
private String accessKeySecret;
private String region = "us-east-1";
}

156
backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/client/AppearancePatentCozeClient.java Normal file
View File

@@ -0,0 +1,156 @@
package com.nanri.aiimage.modules.appearancepatent.client;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.AppearancePatentProperties;
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultRowDto;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.http.MediaType;
import org.springframework.http.client.SimpleClientHttpRequestFactory;
import org.springframework.stereotype.Component;
import org.springframework.web.client.RestClient;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@Component
@RequiredArgsConstructor
@Slf4j
public class AppearancePatentCozeClient {
private final AppearancePatentProperties properties;
private final ObjectMapper objectMapper;
public List<AppearancePatentResultRowDto> inspect(List<AppearancePatentResultRowDto> rows, String prompt) {
if (rows == null || rows.isEmpty()) {
return List.of();
}
if (properties.getCozeToken() == null || properties.getCozeToken().isBlank()) {
log.warn("[appearance-patent] coze token not configured, keep raw rows size={}", rows.size());
return rows.stream().map(this::copy).toList();
}
try {
String raw = postWorkflow(rows, prompt);
List<CozeResult> results = parseResults(raw);
List<AppearancePatentResultRowDto> merged = new ArrayList<>();
for (int i = 0; i < rows.size(); i++) {
AppearancePatentResultRowDto row = copy(rows.get(i));
if (i < results.size()) {
CozeResult result = results.get(i);
row.setTitleRisk(result.title());
row.setAppearanceRisk(result.appearance());
row.setPatentRisk(result.patent());
row.setConclusion(result.result());
}
merged.add(row);
}
return merged;
} catch (Exception ex) {
log.warn("[appearance-patent] coze batch failed size={} err={}", rows.size(), ex.getMessage());
return rows.stream().map(this::copy).toList();
}
}
private String postWorkflow(List<AppearancePatentResultRowDto> rows, String prompt) {
Map<String, Object> parameters = new LinkedHashMap<>();
parameters.put("title_list", rows.stream().map(row -> nonBlank(row.getTitle(), row.getAsin())).toList());
parameters.put("url_list", rows.stream().map(row -> nonBlank(row.getUrl(), "")).toList());
parameters.put("prompt", prompt == null ? "" : prompt);
Map<String, Object> body = new LinkedHashMap<>();
body.put("workflow_id", properties.getCozeWorkflowId());
body.put("parameters", parameters);
RestClient.RequestBodySpec request = restClient().post()
.uri(joinUrl(properties.getCozeBaseUrl(), properties.getCozeWorkflowPath()))
.headers(headers -> {
headers.setBearerAuth(stripBearer(properties.getCozeToken()));
headers.setContentType(MediaType.APPLICATION_JSON);
});
request.body(body);
return request.retrieve().body(String.class);
}
private List<CozeResult> parseResults(String raw) throws Exception {
JsonNode root = objectMapper.readTree(raw);
if (root.path("code").asInt(-1) != 0) {
throw new IllegalStateException(root.path("msg").asText("Coze response code is not 0"));
}
String dataText = root.path("data").asText("");
if (dataText.isBlank()) {
return List.of();
}
JsonNode dataRoot = objectMapper.readTree(dataText);
JsonNode array = dataRoot.path("data");
List<CozeResult> results = new ArrayList<>();
if (array.isArray()) {
for (JsonNode node : array) {
results.add(new CozeResult(
text(node.get("title")),
text(node.get("appearance")),
text(firstNonNull(node.get("patent"), node.get("patent "))),
text(node.get("result"))
));
}
}
return results;
}
private RestClient restClient() {
SimpleClientHttpRequestFactory requestFactory = new SimpleClientHttpRequestFactory();
requestFactory.setConnectTimeout(10000);
requestFactory.setReadTimeout(60000);
return RestClient.builder().requestFactory(requestFactory).build();
}
private AppearancePatentResultRowDto copy(AppearancePatentResultRowDto source) {
AppearancePatentResultRowDto row = new AppearancePatentResultRowDto();
row.setId(source.getId());
row.setAsin(source.getAsin());
row.setCountry(source.getCountry());
row.setUrl(source.getUrl());
row.setTitle(source.getTitle());
row.setError(source.getError());
row.setDone(source.getDone());
row.setTitleRisk(source.getTitleRisk());
row.setAppearanceRisk(source.getAppearanceRisk());
row.setPatentRisk(source.getPatentRisk());
row.setConclusion(source.getConclusion());
return row;
}
private JsonNode firstNonNull(JsonNode left, JsonNode right) {
return left == null || left.isNull() ? right : left;
}
private String text(JsonNode node) {
return node == null || node.isNull() ? null : node.asText();
}
private String nonBlank(String value, String fallback) {
return value == null || value.isBlank() ? fallback : value;
}
private String stripBearer(String token) {
String normalized = token == null ? "" : token.trim();
return normalized.regionMatches(true, 0, "Bearer ", 0, 7) ? normalized.substring(7).trim() : normalized;
}
private String joinUrl(String baseUrl, String path) {
String base = baseUrl == null ? "" : baseUrl.trim();
String suffix = path == null ? "" : path.trim();
if (base.endsWith("/") && suffix.startsWith("/")) {
return base + suffix.substring(1);
}
if (!base.endsWith("/") && !suffix.startsWith("/")) {
return base + "/" + suffix;
}
return base + suffix;
}
private record CozeResult(String title, String appearance, String patent, String result) {
}
}

143
backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/controller/AppearancePatentController.java Normal file
View File

@@ -0,0 +1,143 @@
package com.nanri.aiimage.modules.appearancepatent.controller;
import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentParseRequest;
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentSubmitResultRequest;
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentTaskBatchRequest;
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentDashboardVo;
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentHistoryVo;
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParseVo;
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentTaskBatchVo;
import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskService;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.server.ResponseStatusException;
import java.io.InputStream;
import java.net.URI;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
@RestController
@RequiredArgsConstructor
@RequestMapping("/api/appearance-patent")
@Tag(name = "外观专利检测", description = "外观专利检测任务接口。前端上传 Excel 后由 Java 解析并创建任务；Python 回传商品数据；Java 负责攒批调用 Coze、补齐子行、生成最终 xlsx 并上传 OSS。")
public class AppearancePatentController {
private final AppearancePatentTaskService service;
@PostMapping("/parse")
@Operation(summary = "解析 Excel 并创建任务", description = "解析上传后的 Excel 文件，提取 id、ASIN、国家、URL、标题等字段。返回给前端的数据只包含整数 id 和 n_1 行；n_2、n_3 等子行会保存在 OSS 解析载荷中，用于最终结果补齐。创建后的任务状态为 PENDING，不会自动推送 Python。")
public ApiResponse<AppearancePatentParseVo> parse(@Valid @RequestBody AppearancePatentParseRequest request) {
return ApiResponse.success(service.parseAndCreateTask(request));
}
@GetMapping("/dashboard")
@Operation(summary = "查询外观专利检测总览", description = "查询当前用户的运行中、成功、失败和已结束任务数量。页面进入时请求一次即可，不需要持续轮询。")
public ApiResponse<AppearancePatentDashboardVo> dashboard(
@Parameter(description = "当前用户 ID，用于隔离不同用户的任务和历史记录。", required = true, example = "1")
@RequestParam("user_id") Long userId) {
return ApiResponse.success(service.dashboard(userId));
}
@GetMapping("/history")
@Operation(summary = "查询外观专利检测历史", description = "查询当前用户最近的外观专利检测历史记录，包含源文件名、任务状态、行数、错误信息和最终 xlsx 下载地址。")
public ApiResponse<AppearancePatentHistoryVo> history(
@Parameter(description = "当前用户 ID，用于查询该用户自己的历史记录。", required = true, example = "1")
@RequestParam("user_id") Long userId) {
return ApiResponse.success(service.history(userId));
}
@PostMapping("/tasks/progress/batch")
@Operation(summary = "批量查询任务进度", description = "前端只对活跃任务调用该接口，建议 6 秒一次。接口只返回轻量任务状态，不返回明细结果。")
public ApiResponse<AppearancePatentTaskBatchVo> progress(@Valid @RequestBody AppearancePatentTaskBatchRequest request) {
return ApiResponse.success(service.progressBatch(request.getTaskIds()));
}
@PostMapping("/tasks/{taskId}/activate")
@Operation(summary = "激活任务", description = "前端手动推送 Python 队列成功后调用，将任务从 PENDING 改为 RUNNING，并记录后端内部活跃时间。后续活跃时间由 Python 回传结果接口自动刷新，不需要单独心跳接口。")
public ApiResponse<Void> activate(
@Parameter(description = "外观专利检测任务 ID，即解析接口返回的 taskId。", required = true, example = "3938")
@PathVariable Long taskId,
@Parameter(description = "当前用户 ID，必须与创建任务的用户一致。", required = true, example = "1")
@RequestParam("user_id") Long userId) {
service.activateTask(taskId, userId);
return ApiResponse.success(null);
}
@PostMapping("/tasks/{taskId}/result")
@Operation(summary = "提交 Python 回传结果", description = "Python 回传商品数据接口。items 可以是一条或多条；Java 先原样保存回传数据，再内部攒够 10 条调用 Coze。done=true 表示 Python 已完成全部回传，Java 会强制处理剩余不足 10 条的数据并生成最终 xlsx。")
public ApiResponse<Void> result(
@Parameter(description = "外观专利检测任务 ID。任务必须处于 RUNNING 状态。", required = true, example = "3938")
@PathVariable Long taskId,
@Valid @RequestBody AppearancePatentSubmitResultRequest request) {
service.submitResult(taskId, request);
return ApiResponse.success(null);
}
@DeleteMapping("/tasks/{taskId}")
@Operation(summary = "删除任务", description = "删除当前用户的一条外观专利检测任务，同时清理任务结果、scope 状态和分片记录。")
public ApiResponse<Void> deleteTask(
@Parameter(description = "外观专利检测任务 ID。", required = true, example = "3938")
@PathVariable Long taskId,
@Parameter(description = "当前用户 ID，必须与创建任务的用户一致。", required = true, example = "1")
@RequestParam("user_id") Long userId) {
service.deleteTask(taskId, userId);
return ApiResponse.success(null);
}
@DeleteMapping("/history/{resultId}")
@Operation(summary = "删除历史记录", description = "删除当前用户的一条外观专利检测历史记录。只删除 biz_file_result 记录，不主动删除任务主记录。")
public ApiResponse<Void> deleteHistory(
@Parameter(description = "历史结果记录 ID，即 history 接口返回的 resultId。", required = true, example = "1001")
@PathVariable Long resultId,
@Parameter(description = "当前用户 ID。", required = true, example = "1")
@RequestParam("user_id") Long userId) {
service.deleteHistory(resultId, userId);
return ApiResponse.success(null);
}
@GetMapping("/results/{resultId}/download")
@Operation(summary = "下载外观专利检测结果文件")
public void downloadResult(
@Parameter(description = "结果记录 ID", required = true, example = "1001")
@PathVariable Long resultId,
@Parameter(description = "当前用户 ID", required = true, example = "1")
@RequestParam("user_id") Long userId,
jakarta.servlet.http.HttpServletResponse response) {
String url = service.resolveResultDownloadUrl(resultId, userId);
String filename = service.resolveResultDownloadFilename(resultId, userId);
if (url == null || url.isBlank()) {
throw new ResponseStatusException(HttpStatus.NOT_FOUND, "暂无可下载结果");
}
try {
String encodedFilename = URLEncoder.encode(filename, StandardCharsets.UTF_8).replace("+", "%20");
response.setContentType("application/octet-stream");
response.setHeader(HttpHeaders.CONTENT_DISPOSITION,
"attachment; filename=\"" + filename + "\"; filename*=UTF-8''" + encodedFilename);
try (InputStream in = URI.create(url).toURL().openStream()) {
byte[] buffer = new byte[65536];
int read;
while ((read = in.read(buffer)) != -1) {
response.getOutputStream().write(buffer, 0, read);
}
response.getOutputStream().flush();
}
} catch (Exception ex) {
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "下载失败");
}
}
}

28
backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/dto/AppearancePatentParseRequest.java Normal file
View File

@@ -0,0 +1,28 @@
package com.nanri.aiimage.modules.appearancepatent.model.dto;
import com.fasterxml.jackson.annotation.JsonAlias;
import com.fasterxml.jackson.annotation.JsonProperty;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotEmpty;
import jakarta.validation.constraints.NotNull;
import lombok.Data;
import java.util.List;
@Data
@Schema(description = "外观专利检测解析请求")
public class AppearancePatentParseRequest {
@JsonProperty("user_id")
@NotNull
@Schema(description = "当前用户 ID。后端会把创建的任务、历史记录和结果文件归属到该用户。", example = "1", requiredMode = Schema.RequiredMode.REQUIRED)
private Long userId;
@NotEmpty
@Schema(description = "已上传的 Excel 文件列表。当前外观专利检测只读取第一个文件；文件对象来自统一上传接口返回值。", requiredMode = Schema.RequiredMode.REQUIRED)
private List<AppearancePatentSourceFileDto> files;
@JsonProperty("ai_prompt")
@JsonAlias({"aiPrompt", "prompt"})
@Schema(description = "AI 提示词。非必填；为空时前端会使用默认提示词。后端会保存该提示词，并在调用 Coze workflow 时作为 prompt 参数传入。", example = "请排查这些亚马逊商品在英国及欧洲地区是否存在知识产权侵权风险。")
private String aiPrompt;
}

21
backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/dto/AppearancePatentParsedPayloadDto.java Normal file
View File

@@ -0,0 +1,21 @@
package com.nanri.aiimage.modules.appearancepatent.model.dto;
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedRowVo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "外观专利检测解析载荷。通常保存到 OSS，数据库只保存 oss 指针。")
public class AppearancePatentParsedPayloadDto {
@Schema(description = "AI 提示词。")
private String aiPrompt;
@Schema(description = "Excel 原始表头列表。")
private List<String> headers = new ArrayList<>();
@Schema(description = "返回前端和推给 Python 的代表行，只包含整数 id 和 n_1 行。")
private List<AppearancePatentParsedRowVo> items = new ArrayList<>();
@Schema(description = "完整有效行，包含 n_2、n_3 等解析后被前端过滤但最终需要补回的子行。")
private List<AppearancePatentParsedRowVo> allItems = new ArrayList<>();
}

33
backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/dto/AppearancePatentResultRowDto.java Normal file
View File

@@ -0,0 +1,33 @@
package com.nanri.aiimage.modules.appearancepatent.model.dto;
import com.fasterxml.jackson.annotation.JsonAlias;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@Schema(description = "外观专利检测单行商品数据")
public class AppearancePatentResultRowDto {
@Schema(description = "Excel 中的 id。代表行通常是整数 id 或 n_1，例如 2_1；最终生成 xlsx 时，2_2、2_3 会复用同组 2_1 的 Coze 检测结果。", example = "2_1")
private String id;
@Schema(description = "亚马逊 ASIN。后端会统一按大写处理和匹配。", example = "B0CJ8SNXXV")
private String asin;
@Schema(description = "站点或国家。来自 Excel 的国家列，例如英国、德国、法国。", example = "英国")
private String country;
@Schema(description = "商品主图或待检测图片 URL。Java 调用 Coze 时会放入 url_list。", example = "https://webstatic.aiproxy.vip/output/20260425/103322/demo.jpg")
private String url;
@Schema(description = "商品标题。Java 调用 Coze 时会放入 title_list；为空时会回退使用 ASIN。", example = "Women Floral Dress Summer Casual")
private String title;
@Schema(description = "单行错误信息。通常用于记录 Python 单行处理异常；Coze 失败时后端会尽量保留原始行，不强行写入风险结果。", example = "图片地址为空")
private String error;
@Schema(description = "单行完成标记。当前主要使用请求体顶层 done 控制任务收尾，该字段仅作兼容。", example = "true")
private Boolean done;
@Schema(description = "Java 调用 Coze 后生成的标题维度检测结果，对应最终 xlsx 的“标题维度（商标）”列。Python 回传请求不要传该字段；即使传入，后端也会以 Java/Coze 处理结果为准。", example = "标题未发现明显商标侵权风险。", accessMode = Schema.AccessMode.READ_ONLY)
private String titleRisk;
@Schema(description = "Java 调用 Coze 后生成的外观维度检测结果，对应最终 xlsx 的“外观维度（外观设计专利）”列。Python 回传请求不要传该字段。", example = "未发现明显外观设计专利侵权风险。", accessMode = Schema.AccessMode.READ_ONLY)
private String appearanceRisk;
@JsonAlias({"patent ", "patent"})
@Schema(description = "Java 调用 Coze 后生成的专利维度检测结果，对应最终 xlsx 的“专利维度（发明/实用新型专利）”列。兼容 Coze 返回字段 patent 和 patent 后带空格的情况；Python 回传请求不要传该字段。", example = "未发现明显发明或实用新型专利侵权风险。", accessMode = Schema.AccessMode.READ_ONLY)
private String patentRisk;
@Schema(description = "Java 调用 Coze 后生成的最终结论，对应最终 xlsx 的“结论”列。Python 回传请求不要传该字段。", example = "未发现明显侵权风险", accessMode = Schema.AccessMode.READ_ONLY)
private String conclusion;
}

15
backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/dto/AppearancePatentSourceFileDto.java Normal file
View File

@@ -0,0 +1,15 @@
package com.nanri.aiimage.modules.appearancepatent.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@Schema(description = "外观专利检测源文件信息")
public class AppearancePatentSourceFileDto {
@Schema(description = "上传接口返回的临时文件 key。后端根据该 key 查找本地临时 Excel 文件并解析。", example = "uploads/20260426/appearance_patent_17.xlsx", requiredMode = Schema.RequiredMode.REQUIRED)
private String fileKey;
@Schema(description = "原始文件名。用于历史记录展示和最终结果文件命名。", example = "17.xlsx")
private String originalFilename;
@Schema(description = "相对目录路径。当前仅记录来源，外观专利检测不依赖该字段处理。", example = "xlsx/17.xlsx")
private String relativePath;
}

24
backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/dto/AppearancePatentSubmitResultRequest.java Normal file
View File

@@ -0,0 +1,24 @@
package com.nanri.aiimage.modules.appearancepatent.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "Python 回传外观专利检测结果请求")
public class AppearancePatentSubmitResultRequest {
@Schema(description = "本次 Python 回传的提交批次标识。建议同一个任务固定使用同一个值，例如 appearance-patent-{taskId}；后端会结合该值和 chunkIndex 做分片幂等。", example = "appearance-patent-3938")
private String submissionId;
@Schema(description = "当前回传分片序号。建议从 1 开始递增；同一个 submissionId 下相同 chunkIndex 重复提交会被后端识别为重复分片并跳过重复处理。", example = "1")
private Integer chunkIndex;
@Schema(description = "本任务预计总分片数。如果 Python 是一条一条回传，可设置为总商品数；如果无法预估，可传 0 或 1，最终以 done=true 触发收尾。", example = "458")
private Integer chunkTotal;
@Schema(description = "是否为最后一次回传。true 表示 Python 已完成该任务全部数据回传；Java 会处理剩余不足 10 条的数据、组装最终 xlsx、上传 OSS 并收尾任务。", example = "false")
private Boolean done;
@Schema(description = "Python 侧任务级错误信息。非空时 Java 会记录错误并按失败任务收尾；普通单行 Coze 失败不建议写这里。", example = "浏览器执行异常，任务提前结束")
private String error;
@Schema(description = "本次回传的商品原始数据列表。可以一条一条传，也可以一次多条传；Python 只需要传 id、asin、country、url、title、error、done 等原始/执行字段。titleRisk、appearanceRisk、patentRisk、conclusion 由 Java 攒够 10 条调用 Coze 后生成，Python 不要传。")
private List<AppearancePatentResultRowDto> items = new ArrayList<>();
}

15
backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/dto/AppearancePatentTaskBatchRequest.java Normal file
View File

@@ -0,0 +1,15 @@
package com.nanri.aiimage.modules.appearancepatent.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotEmpty;
import lombok.Data;
import java.util.List;
@Data
@Schema(description = "外观专利检测批量进度查询请求")
public class AppearancePatentTaskBatchRequest {
@NotEmpty
@Schema(description = "需要查询进度的任务 ID 列表。前端只传正在轮询的活跃任务；后端会批量查询，避免每个任务单独请求。", example = "[3938,3939]", requiredMode = Schema.RequiredMode.REQUIRED)
private List<Long> taskIds;
}

17
backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/vo/AppearancePatentDashboardVo.java Normal file
View File

@@ -0,0 +1,17 @@
package com.nanri.aiimage.modules.appearancepatent.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@Schema(description = "外观专利检测总览统计")
public class AppearancePatentDashboardVo {
@Schema(description = "运行中任务数量。这里统计 RUNNING 状态任务。", example = "1")
private Long pendingTaskCount;
@Schema(description = "已结束任务数量，等于成功任务数加失败任务数。", example = "12")
private Long processedTaskCount;
@Schema(description = "成功任务数量。", example = "10")
private Long successTaskCount;
@Schema(description = "失败任务数量。", example = "2")
private Long failedTaskCount;
}

33
backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/vo/AppearancePatentHistoryItemVo.java Normal file
View File

@@ -0,0 +1,33 @@
package com.nanri.aiimage.modules.appearancepatent.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@Schema(description = "外观专利检测历史记录项")
public class AppearancePatentHistoryItemVo {
@Schema(description = "结果记录 ID。删除历史、下载结果时使用。", example = "1001")
private Long resultId;
@Schema(description = "任务 ID。", example = "3938")
private Long taskId;
@Schema(description = "源 Excel 文件名。", example = "17.xlsx")
private String sourceFilename;
@Schema(description = "最终结果文件名。任务完成并生成 xlsx 后返回。", example = "17-result.xlsx")
private String resultFilename;
@Schema(description = "最终结果文件下载地址。后端基于 OSS objectKey 生成的新鲜预签名 URL。", example = "https://bucket.oss-cn-hangzhou.aliyuncs.com/result/appearance_patent/xxx/17-result.xlsx?Expires=...")
private String downloadUrl;
private Long fileJobId;
private String fileStatus;
private String fileError;
private Boolean fileReady;
@Schema(description = "任务状态：PENDING=已解析待推送，RUNNING=执行中，SUCCESS=成功，FAILED=失败。", example = "SUCCESS")
private String taskStatus;
@Schema(description = "结果是否成功。true 表示任务完成并生成结果文件；false 表示失败或未完成。", example = "true")
private Boolean success;
@Schema(description = "错误信息。任务失败时返回，例如 Python 超时、结果文件生成失败等。", example = "Python interrupted before uploading final appearance patent result")
private String error;
@Schema(description = "最终结果行数。包含解析阶段被过滤但最终需要补回的 2_2、2_3 等子行。", example = "716")
private Integer rowCount;
@Schema(description = "历史记录创建时间，ISO 本地时间字符串。", example = "2026-04-26T10:30:00")
private String createdAt;
}

14
backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/vo/AppearancePatentHistoryVo.java Normal file
View File

@@ -0,0 +1,14 @@
package com.nanri.aiimage.modules.appearancepatent.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "外观专利检测历史记录列表")
public class AppearancePatentHistoryVo {
@Schema(description = "历史记录项列表，默认返回最近 100 条。")
private List<AppearancePatentHistoryItemVo> items = new ArrayList<>();
}

26
backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/vo/AppearancePatentParseVo.java Normal file
View File

@@ -0,0 +1,26 @@
package com.nanri.aiimage.modules.appearancepatent.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "外观专利检测解析结果")
public class AppearancePatentParseVo {
@Schema(description = "新创建的任务 ID。后续手动推 Python、激活任务、回传结果、查询进度都使用该 ID。", example = "3938")
private Long taskId;
@Schema(description = "源 Excel 文件名。", example = "17.xlsx")
private String sourceFilename;
@Schema(description = "Excel 中检测到的有效数据总行数，不含空行。", example = "716")
private Integer totalRows;
@Schema(description = "返回前端并准备推给 Python 的代表行数量。只包含整数 id 和 n_1 行。", example = "458")
private Integer acceptedRows;
@Schema(description = "解析时被过滤或缺少必要字段的行数。n_2、n_3 等子行会计入过滤数，但仍会保存在 OSS 解析载荷中用于最终补齐。", example = "258")
private Integer droppedRows;
@Schema(description = "本任务最终使用的 AI 提示词。", example = "请排查这些亚马逊商品在英国及欧洲地区是否存在知识产权侵权风险。")
private String aiPrompt;
@Schema(description = "返回前端的代表行列表。前端只展示样例，不展示完整明细；完整 allItems 已保存在 OSS 解析载荷中。")
private List<AppearancePatentParsedRowVo> items = new ArrayList<>();
}

28
backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/vo/AppearancePatentParsedRowVo.java Normal file
View File

@@ -0,0 +1,28 @@
package com.nanri.aiimage.modules.appearancepatent.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.LinkedHashMap;
import java.util.Map;
@Data
@Schema(description = "外观专利检测解析出的单行数据")
public class AppearancePatentParsedRowVo {
@Schema(description = "Excel 原始行号，从 1 开始。", example = "2")
private Integer rowIndex;
@Schema(description = "Excel 中原始 id 值。", example = "2_1")
private String sourceId;
@Schema(description = "前端展示和 Python 回传使用的 id。整数 id 原样保留，子数据第一条如 2_1 原样保留。", example = "2_1")
private String displayId;
@Schema(description = "亚马逊 ASIN。", example = "B0CJ8SNXXV")
private String asin;
@Schema(description = "国家或站点。", example = "英国")
private String country;
@Schema(description = "商品图片 URL 或商品 URL，供 Coze 检测使用。", example = "https://webstatic.aiproxy.vip/output/demo.jpg")
private String url;
@Schema(description = "商品标题。", example = "Women Floral Dress Summer Casual")
private String title;
@Schema(description = "该 Excel 行的原始列值映射。最终生成 xlsx 时可从这里读取价格等字段。", example = "{\"id\":\"2_1\",\"asin\":\"B0CJ8SNXXV\",\"国家\":\"英国\",\"价格\":\"12.99\"}")
private Map<String, String> values = new LinkedHashMap<>();
}

16
backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/vo/AppearancePatentTaskBatchVo.java Normal file
View File

@@ -0,0 +1,16 @@
package com.nanri.aiimage.modules.appearancepatent.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "外观专利检测批量进度响应")
public class AppearancePatentTaskBatchVo {
@Schema(description = "查询到的任务详情列表。顺序按请求 taskIds 处理。")
private List<AppearancePatentTaskDetailVo> items = new ArrayList<>();
@Schema(description = "未找到或不属于外观专利检测模块的任务 ID 列表。", example = "[99999]")
private List<Long> missingTaskIds = new ArrayList<>();
}

16
backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/vo/AppearancePatentTaskDetailVo.java Normal file
View File

@@ -0,0 +1,16 @@
package com.nanri.aiimage.modules.appearancepatent.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "外观专利检测任务进度详情")
public class AppearancePatentTaskDetailVo {
@Schema(description = "任务主记录轻量信息。")
private AppearancePatentTaskItemVo task;
@Schema(description = "预留的任务明细列表。当前进度接口主要返回任务轻量状态，不返回完整结果明细。")
private List<AppearancePatentHistoryItemVo> items = new ArrayList<>();
}

23
backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/model/vo/AppearancePatentTaskItemVo.java Normal file
View File

@@ -0,0 +1,23 @@
package com.nanri.aiimage.modules.appearancepatent.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@Schema(description = "外观专利检测任务轻量信息")
public class AppearancePatentTaskItemVo {
@Schema(description = "任务 ID。", example = "3938")
private Long id;
@Schema(description = "任务编号，后端自动生成。", example = "APPEARANCE_PATENT-1780000000000000000")
private String taskNo;
@Schema(description = "任务状态：PENDING=已解析待推送，RUNNING=执行中，SUCCESS=成功，FAILED=失败。", example = "RUNNING")
private String status;
@Schema(description = "任务级错误信息。失败时返回。", example = "生成外观专利检测结果失败")
private String errorMessage;
@Schema(description = "创建时间，ISO 本地时间字符串。", example = "2026-04-26T10:00:00")
private String createdAt;
@Schema(description = "最后更新时间，通常由 Python 回传结果或任务收尾更新。", example = "2026-04-26T10:05:00")
private String updatedAt;
@Schema(description = "完成时间。任务未结束时为空。", example = "2026-04-26T10:10:00")
private String finishedAt;
}

144
backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskCacheService.java Normal file
View File

@@ -0,0 +1,144 @@
package com.nanri.aiimage.modules.appearancepatent.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultRowDto;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
@Service
@RequiredArgsConstructor
@Slf4j
public class AppearancePatentTaskCacheService {
private static final long TTL_HOURS = 24;
private final StringRedisTemplate stringRedisTemplate;
private final ObjectMapper objectMapper;
public void appendPendingRow(Long taskId, Integer chunkIndex, AppearancePatentResultRowDto row) {
if (taskId == null || taskId <= 0 || chunkIndex == null || row == null) {
return;
}
try {
stringRedisTemplate.opsForList().rightPush(
pendingRowsKey(taskId),
objectMapper.writeValueAsString(new PendingRow(chunkIndex, row))
);
stringRedisTemplate.expire(pendingRowsKey(taskId), Duration.ofHours(TTL_HOURS));
touchTaskHeartbeat(taskId);
} catch (Exception ignored) {
}
}
public long pendingRowCount(Long taskId) {
Long size;
try {
size = stringRedisTemplate.opsForList().size(pendingRowsKey(taskId));
} catch (Exception ex) {
log.warn("[appearance-patent-cache] pending row count degraded taskId={} msg={}", taskId, ex.getMessage());
return 0L;
}
return size == null ? 0L : size;
}
public List<PendingRow> drainPendingRows(Long taskId, int limit) {
if (taskId == null || taskId <= 0 || limit <= 0) {
return List.of();
}
String key = pendingRowsKey(taskId);
List<String> values;
try {
values = stringRedisTemplate.opsForList().range(key, 0, limit - 1L);
} catch (Exception ex) {
log.warn("[appearance-patent-cache] drain range degraded taskId={} msg={}", taskId, ex.getMessage());
return List.of();
}
if (values == null || values.isEmpty()) {
return List.of();
}
try {
stringRedisTemplate.opsForList().trim(key, values.size(), -1);
} catch (Exception ex) {
log.warn("[appearance-patent-cache] drain trim degraded taskId={} msg={}", taskId, ex.getMessage());
return List.of();
}
List<PendingRow> rows = new ArrayList<>();
for (String value : values) {
if (value == null || value.isBlank()) {
continue;
}
try {
rows.add(objectMapper.readValue(value, PendingRow.class));
} catch (Exception ignored) {
}
}
touchTaskHeartbeat(taskId);
return rows;
}
public void touchTaskHeartbeat(Long taskId) {
if (taskId == null || taskId <= 0) {
return;
}
try {
stringRedisTemplate.opsForValue().set(
heartbeatKey(taskId),
String.valueOf(Instant.now().toEpochMilli()),
Duration.ofHours(TTL_HOURS)
);
} catch (Exception ex) {
log.warn("[appearance-patent-cache] touch heartbeat degraded taskId={} msg={}", taskId, ex.getMessage());
}
}
public long getTaskHeartbeatMillis(Long taskId) {
if (taskId == null || taskId <= 0) {
return 0L;
}
String raw;
try {
raw = stringRedisTemplate.opsForValue().get(heartbeatKey(taskId));
} catch (Exception ex) {
log.warn("[appearance-patent-cache] get heartbeat degraded taskId={} msg={}", taskId, ex.getMessage());
return 0L;
}
if (raw == null || raw.isBlank()) {
return 0L;
}
try {
return Long.parseLong(raw);
} catch (NumberFormatException ignored) {
return 0L;
}
}
public void deleteTaskCache(Long taskId) {
if (taskId == null || taskId <= 0) {
return;
}
try {
stringRedisTemplate.delete(pendingRowsKey(taskId));
stringRedisTemplate.delete(heartbeatKey(taskId));
} catch (Exception ex) {
log.warn("[appearance-patent-cache] delete degraded taskId={} msg={}", taskId, ex.getMessage());
}
}
private String pendingRowsKey(Long taskId) {
return "appearance-patent:task:pending-rows:" + taskId;
}
private String heartbeatKey(Long taskId) {
return "appearance-patent:task:heartbeat:" + taskId;
}
public record PendingRow(Integer chunkIndex, AppearancePatentResultRowDto row) {
}
}

1001
backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java Normal file
View File

File diff suppressed because it is too large Load Diff

32
backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskProgressCacheService.java
View File

@@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.brand.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.BrandProgressProperties;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
@@ -11,6 +12,7 @@ import java.util.LinkedHashMap;
import java.util.Map;
@Service
@Slf4j
public class BrandTaskProgressCacheService {
public static final String PHASE_CRAWLING = "crawling";
@@ -53,8 +55,12 @@ public class BrandTaskProgressCacheService {
values.put("finished_files", String.valueOf(Math.max(finishedFiles, 0)));
values.put("updated_at", now);
values.put("last_heartbeat_at", now);
try {
stringRedisTemplate.opsForHash().putAll(key, values);
stringRedisTemplate.expire(key, ttl());
} catch (Exception ex) {
log.warn("[brand-progress-cache] save progress degraded taskId={} msg={}", taskId, ex.getMessage());
}
}
public void updatePhase(Long taskId, String phase, int finishedFiles, int fileTotal) {
@@ -66,8 +72,12 @@ public class BrandTaskProgressCacheService {
values.put("file_total", String.valueOf(Math.max(fileTotal, 0)));
values.put("updated_at", now);
values.put("last_heartbeat_at", now);
try {
stringRedisTemplate.opsForHash().putAll(key, values);
stringRedisTemplate.expire(key, ttl());
} catch (Exception ex) {
log.warn("[brand-progress-cache] update phase degraded taskId={} msg={}", taskId, ex.getMessage());
}
}
public void markFailed(Long taskId, String message) {
@@ -77,27 +87,49 @@ public class BrandTaskProgressCacheService {
values.put("phase", PHASE_FAILED);
values.put("updated_at", now);
values.put("error_message", blankToEmpty(message));
try {
stringRedisTemplate.opsForHash().putAll(key, values);
stringRedisTemplate.expire(key, Duration.ofHours(brandProgressProperties.getFailedTtlHours()));
} catch (Exception ex) {
log.warn("[brand-progress-cache] mark failed degraded taskId={} msg={}", taskId, ex.getMessage());
}
}
public Map<Object, Object> getProgress(Long taskId) {
try {
return stringRedisTemplate.opsForHash().entries(buildKey(taskId));
} catch (Exception ex) {
log.warn("[brand-progress-cache] get progress degraded taskId={} msg={}", taskId, ex.getMessage());
return Map.of();
}
}
public boolean acquireFinalizeLock(Long taskId) {
try {
Boolean ok = stringRedisTemplate.opsForValue()
.setIfAbsent(buildFinalizeLockKey(taskId), String.valueOf(Instant.now().toEpochMilli()), FINALIZE_LOCK_TTL);
return Boolean.TRUE.equals(ok);
} catch (Exception ex) {
log.warn("[brand-progress-cache] acquire finalize lock degraded taskId={} msg={}", taskId, ex.getMessage());
return false;
}
}
public void releaseFinalizeLock(Long taskId) {
try {
stringRedisTemplate.delete(buildFinalizeLockKey(taskId));
} catch (Exception ex) {
log.warn("[brand-progress-cache] release finalize lock degraded taskId={} msg={}", taskId, ex.getMessage());
}
}
public void delete(Long taskId) {
try {
stringRedisTemplate.delete(buildKey(taskId));
stringRedisTemplate.delete(buildFinalizeLockKey(taskId));
} catch (Exception ex) {
log.warn("[brand-progress-cache] delete degraded taskId={} msg={}", taskId, ex.getMessage());
}
}
public String buildKey(Long taskId) {

79
backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskService.java
View File

@@ -31,6 +31,9 @@ import com.nanri.aiimage.modules.brand.model.vo.LegacyBrandTaskDetailVo;
import com.nanri.aiimage.modules.brand.model.vo.LegacyBrandTaskItemVo;
import com.nanri.aiimage.modules.brand.model.vo.LegacyBrandTaskListVo;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.poi.ss.usermodel.Cell;
@@ -77,6 +80,7 @@ public class BrandTaskService {
private static final String STATUS_SUCCESS = "success";
private static final String STATUS_FAILED = "failed";
private static final String STATUS_CANCELLED = "cancelled";
private static final String MODULE_TYPE = "BRAND";
private final BrandCrawlTaskMapper brandCrawlTaskMapper;
private final OssStorageService ossStorageService;
@@ -86,6 +90,8 @@ public class BrandTaskService {
private final BrandTaskStorageService brandTaskStorageService;
private final DistributedJobLockService distributedJobLockService;
private final ObjectMapper objectMapper;
private final TaskFileJobService taskFileJobService;
private final TaskProgressSnapshotService taskProgressSnapshotService;
public BrandTaskCreateVo createTask(Long userId, BrandTaskCreateRequest request) {
if (userId == null || userId <= 0) {
@@ -244,6 +250,7 @@ public class BrandTaskService {
int totalCount = sourceFiles.size();
markTaskRunning(taskId, totalCount);
afterTaskStarted(taskId, totalCount);
log.info("[brand-submit] taskId={} chunks={} files={} thread={}",
taskId,
@@ -295,6 +302,7 @@ public class BrandTaskService {
currentTotalLines,
finishedCount);
updateTaskProgress(taskId, finishedCount, totalCount);
saveBrandProgressSnapshot(taskId, STATUS_RUNNING, totalCount, finishedCount, 0, null);
}
tryFinalizeTask(taskId, request.getStrategy(), sourceFiles, cachedByUrl, finishedCount, totalCount);
@@ -357,6 +365,23 @@ public class BrandTaskService {
return ossStorageService.generateFreshDownloadUrl(stored);
}
public String resolveResultObjectKey(Long taskId) {
BrandCrawlTaskEntity task = requireTask(taskId);
Object raw = parseJsonValue(task.getResultPaths());
if (!(raw instanceof Map<?, ?> map)) {
return null;
}
Object zipObjectKey = map.get("zip_object_key");
if (zipObjectKey instanceof String key && !key.isBlank()) {
return key;
}
Object zipUrl = map.get("zip_url");
if (zipUrl instanceof String url && !url.isBlank()) {
return url;
}
return null;
}
private List<BrandCrawlTaskEntity> listTaskEntities(Long userId) {
if (userId == null || userId <= 0) {
throw new BusinessException("userId 不合法");
@@ -496,7 +521,7 @@ public class BrandTaskService {
private void markTaskRunning(Long taskId, int totalCount) {
int started = brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper<BrandCrawlTaskEntity>()
.eq(BrandCrawlTaskEntity::getId, taskId)
.ne(BrandCrawlTaskEntity::getStatus, STATUS_CANCELLED)
.in(BrandCrawlTaskEntity::getStatus, STATUS_PENDING, STATUS_RUNNING)
.set(BrandCrawlTaskEntity::getStatus, STATUS_RUNNING)
.set(BrandCrawlTaskEntity::getProgressTotal, totalCount)
.set(BrandCrawlTaskEntity::getUpdatedAt, LocalDateTime.now())
@@ -505,6 +530,13 @@ public class BrandTaskService {
throw new BusinessException("任务已取消");
}
}
private void afterTaskStarted(Long taskId, int totalCount) {
saveBrandProgressSnapshot(taskId, STATUS_RUNNING, totalCount, 0, 0, null);
}
private void saveBrandProgressSnapshot(Long taskId, String status, int totalCount, int successCount, int failedCount, String message) {
taskProgressSnapshotService.save(taskId, MODULE_TYPE, status, totalCount, successCount, failedCount, null, message, null);
}
private BrandCrawlTaskEntity requireActiveTask(Long taskId) {
BrandCrawlTaskEntity task = requireTask(taskId);
@@ -533,12 +565,33 @@ public class BrandTaskService {
return;
}
try {
finalizeTask(taskId, strategy, sourceFiles, cachedByUrl, totalCount);
enqueueFinalizeTask(taskId, totalCount);
} finally {
brandTaskProgressCacheService.releaseFinalizeLock(taskId);
}
}
private void enqueueFinalizeTask(Long taskId, int totalCount) {
brandTaskProgressCacheService.updatePhase(taskId, BrandTaskProgressCacheService.PHASE_ASSEMBLING, totalCount, totalCount);
taskProgressSnapshotService.save(taskId, MODULE_TYPE, STATUS_RUNNING, totalCount, totalCount, 0,
null, "result file assembly queued", null);
taskFileJobService.enqueueAssembleResult(taskId, MODULE_TYPE, taskId, "task:" + taskId);
}
public void processResultFileJob(TaskFileJobEntity job) {
if (job == null || job.getTaskId() == null) {
throw new BusinessException("结果文件任务参数不完整");
}
BrandCrawlTaskEntity task = requireTask(job.getTaskId());
List<BrandSourceFileDto> sourceFiles = parseSourceFiles(task.getFilePaths());
if (sourceFiles.isEmpty()) {
throw new BusinessException("任务没有源文件");
}
List<BrandParsedFileCacheDto> cachedFiles = brandTaskStorageService.getParsedPayload(task.getId());
Map<String, BrandParsedFileCacheDto> cachedByUrl = indexCachedFiles(cachedFiles);
finalizeTask(task.getId(), normalizeStrategy(task.getStrategy()), sourceFiles, cachedByUrl, sourceFiles.size());
}
private void finalizeTask(Long taskId,
String strategy,
List<BrandSourceFileDto> sourceFiles,
@@ -601,6 +654,7 @@ public class BrandTaskService {
if (updated == 0) {
throw new BusinessException("任务已取消");
}
saveBrandProgressSnapshot(taskId, STATUS_SUCCESS, totalCount, totalCount, 0, null);
brandTaskStorageService.deleteTaskData(taskId);
brandTaskProgressCacheService.delete(taskId);
log.info("[brand-finalize] taskId={} finalized success files={} elapsedMs={}",
@@ -617,6 +671,7 @@ public class BrandTaskService {
.set(BrandCrawlTaskEntity::getProgressTotal, totalCount)
.set(BrandCrawlTaskEntity::getErrorMessage, ex.getMessage()));
brandTaskProgressCacheService.markFailed(taskId, ex.getMessage());
saveBrandProgressSnapshot(taskId, STATUS_FAILED, totalCount, brandTaskStorageService.countCompletedFiles(taskId), 1, ex.getMessage());
if (ex instanceof BusinessException businessException) {
throw businessException;
}
@@ -643,10 +698,23 @@ public class BrandTaskService {
}
}
private Map<String, BrandParsedFileCacheDto> indexCachedFiles(List<BrandParsedFileCacheDto> cachedFiles) {
Map<String, BrandParsedFileCacheDto> cachedByUrl = new LinkedHashMap<>();
if (cachedFiles == null) {
return cachedByUrl;
}
for (BrandParsedFileCacheDto cachedFile : cachedFiles) {
if (cachedFile != null && cachedFile.getFileUrl() != null) {
cachedByUrl.put(cachedFile.getFileUrl(), cachedFile);
}
}
return cachedByUrl;
}
private void updateTaskProgress(Long taskId, int finishedCount, int totalCount) {
brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper<BrandCrawlTaskEntity>()
.eq(BrandCrawlTaskEntity::getId, taskId)
.ne(BrandCrawlTaskEntity::getStatus, STATUS_CANCELLED)
.in(BrandCrawlTaskEntity::getStatus, STATUS_PENDING, STATUS_RUNNING)
.set(BrandCrawlTaskEntity::getStatus, STATUS_RUNNING)
.set(BrandCrawlTaskEntity::getProgressCurrent, finishedCount)
.set(BrandCrawlTaskEntity::getProgressTotal, totalCount)
@@ -951,13 +1019,14 @@ public class BrandTaskService {
List<String> fullUrls = new ArrayList<>();
for (OutputEntry entry : entries) {
// 存储完整公开地址，下载时会自动通过 resolveObjectKey 提取并重新签名（见 OssStorageService.resolveObjectKey）
String objectKey = ossStorageService.uploadResultFile(entry.resultFile(), "BRAND");
String objectKey = ossStorageService.uploadResultFile(entry.resultFile(), MODULE_TYPE);
fullUrls.add(ossStorageService.getPublicUrl(objectKey));
}
Map<String, Object> result = new LinkedHashMap<>();
result.put("urls", fullUrls);
File zipFile = packageAsZip(taskId, entries);
String zipObjectKey = ossStorageService.uploadResultFile(zipFile, "BRAND");
String zipObjectKey = ossStorageService.uploadResultFile(zipFile, MODULE_TYPE);
result.put("zip_object_key", zipObjectKey);
result.put("zip_url", ossStorageService.getPublicUrl(zipObjectKey));
return result;
}

82
backend-java/src/main/java/com/nanri/aiimage/modules/brand/service/BrandTaskStorageService.java
View File

@@ -7,11 +7,11 @@ import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.brand.model.dto.BrandCrawlResultFileDto;
import com.nanri.aiimage.modules.brand.model.dto.BrandFileAggregateCacheDto;
import com.nanri.aiimage.modules.brand.model.dto.BrandParsedFileCacheDto;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import lombok.RequiredArgsConstructor;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.stereotype.Service;
@@ -30,12 +30,11 @@ import java.util.Map;
public class BrandTaskStorageService {
private static final String MODULE_TYPE = "BRAND";
private static final String OSS_POINTER_PREFIX = "oss:";
private final TaskChunkMapper taskChunkMapper;
private final TaskScopeStateMapper taskScopeStateMapper;
private final ObjectMapper objectMapper;
private final OssStorageService ossStorageService;
private final TransientPayloadStorageService transientPayloadStorageService;
@Transactional
public void saveParsedPayload(Long taskId, List<BrandParsedFileCacheDto> payload) {
@@ -73,7 +72,7 @@ public class BrandTaskStorageService {
if (state == null) {
throw new BusinessException("Save brand task parsed payload failed");
}
deleteParsedPayloadIfNeeded(state.getParsedPayloadJson(), parsedJson);
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(state.getParsedPayloadJson(), parsedJson);
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getId, state.getId())
.set(TaskScopeStateEntity::getScopeKey, scopeKey)
@@ -132,6 +131,7 @@ public class BrandTaskStorageService {
throw new BusinessException("Brand task chunk conflict: " + scopeKey + "#" + file.getChunkIndex());
}
String storedPayload = transientPayloadStorageService.storeChunkPayload(MODULE_TYPE, taskId, scopeHash, file.getChunkIndex(), payloadJson);
TaskChunkEntity entity = new TaskChunkEntity();
entity.setTaskId(taskId);
entity.setModuleType(MODULE_TYPE);
@@ -139,7 +139,7 @@ public class BrandTaskStorageService {
entity.setScopeHash(scopeHash);
entity.setChunkIndex(file.getChunkIndex());
entity.setChunkTotal(file.getChunkTotal());
entity.setPayloadJson(payloadJson);
entity.setPayloadJson(storedPayload);
entity.setPayloadHash(payloadHash);
entity.setCreatedAt(now);
entity.setUpdatedAt(now);
@@ -153,9 +153,11 @@ public class BrandTaskStorageService {
.eq(TaskChunkEntity::getChunkIndex, file.getChunkIndex())
.last("limit 1"));
if (concurrentChunk != null && payloadHash.equals(concurrentChunk.getPayloadHash())) {
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
BrandFileAggregateCacheDto aggregate = rebuildAggregate(taskId, scopeKey, scopeHash);
return new ChunkStoreResult(false, false, countCompletedFiles(taskId), aggregate);
}
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
throw new BusinessException("Brand task chunk conflict: " + scopeKey + "#" + file.getChunkIndex());
}
@@ -171,7 +173,8 @@ public class BrandTaskStorageService {
return null;
}
try {
return objectMapper.readValue(state.getStateJson(), BrandFileAggregateCacheDto.class);
String aggregateJson = transientPayloadStorageService.resolvePayload(state.getStateJson(), "read brand task aggregate failed");
return objectMapper.readValue(aggregateJson, BrandFileAggregateCacheDto.class);
} catch (Exception ex) {
throw new BusinessException("Read brand task aggregate failed");
}
@@ -195,7 +198,8 @@ public class BrandTaskStorageService {
continue;
}
try {
result.put(state.getScopeKey(), objectMapper.readValue(state.getStateJson(), BrandFileAggregateCacheDto.class));
String aggregateJson = transientPayloadStorageService.resolvePayload(state.getStateJson(), "read brand task aggregate failed");
result.put(state.getScopeKey(), objectMapper.readValue(aggregateJson, BrandFileAggregateCacheDto.class));
} catch (Exception ex) {
throw new BusinessException("Read brand task aggregate failed");
}
@@ -220,12 +224,22 @@ public class BrandTaskStorageService {
return;
}
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
.select(TaskScopeStateEntity::getParsedPayloadJson)
.select(TaskScopeStateEntity::getParsedPayloadJson, TaskScopeStateEntity::getStateJson)
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
if (states != null) {
for (TaskScopeStateEntity state : states) {
deleteParsedPayloadIfNeeded(state.getParsedPayloadJson(), null);
transientPayloadStorageService.deletePayloadIfPresent(state.getParsedPayloadJson());
transientPayloadStorageService.deletePayloadIfPresent(state.getStateJson());
}
}
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
.select(TaskChunkEntity::getPayloadJson)
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
if (chunks != null) {
for (TaskChunkEntity chunk : chunks) {
transientPayloadStorageService.deletePayloadIfPresent(chunk.getPayloadJson());
}
}
taskChunkMapper.delete(new LambdaQueryWrapper<TaskChunkEntity>()
@@ -246,12 +260,14 @@ public class BrandTaskStorageService {
if (state == null) {
throw new BusinessException("Brand task aggregate state missing: " + scopeKey);
}
String storedAggregate = transientPayloadStorageService.storeScopePayload(MODULE_TYPE, taskId, scopeHash, aggregateJson, true);
int chunkTotal = aggregate.getChunkTotal() == null ? 0 : aggregate.getChunkTotal();
int receivedCount = aggregate.getReceivedChunkCount() == null ? 0 : aggregate.getReceivedChunkCount();
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(state.getStateJson(), storedAggregate);
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getId, state.getId())
.set(TaskScopeStateEntity::getScopeKey, scopeKey)
.set(TaskScopeStateEntity::getStateJson, aggregateJson)
.set(TaskScopeStateEntity::getStateJson, storedAggregate)
.set(TaskScopeStateEntity::getChunkTotal, chunkTotal)
.set(TaskScopeStateEntity::getReceivedChunkCount, receivedCount)
.set(TaskScopeStateEntity::getCompleted, Boolean.TRUE.equals(aggregate.getCompleted()) ? 1 : 0)
@@ -362,7 +378,8 @@ public class BrandTaskStorageService {
private BrandCrawlResultFileDto readChunkPayload(String payloadJson) {
try {
return objectMapper.readValue(payloadJson, BrandCrawlResultFileDto.class);
String resolvedPayload = transientPayloadStorageService.resolvePayload(payloadJson, "read brand task chunk failed");
return objectMapper.readValue(resolvedPayload, BrandCrawlResultFileDto.class);
} catch (Exception ex) {
throw new BusinessException("Read brand task chunk failed");
}
@@ -382,56 +399,17 @@ public class BrandTaskStorageService {
}
private String storeParsedPayload(Long taskId, String scopeHash, String payloadJson) {
try {
String pointer = OSS_POINTER_PREFIX + ossStorageService.uploadTaskParsedPayload(MODULE_TYPE, taskId, scopeHash, payloadJson);
return objectMapper.writeValueAsString(pointer);
} catch (Exception ex) {
throw new BusinessException("Save brand task parsed payload failed");
}
return transientPayloadStorageService.storeParsedPayload(MODULE_TYPE, taskId, scopeHash, payloadJson, true);
}
private String resolveParsedPayload(String value) {
if (isBlank(value)) {
return value;
}
String ossPointer = extractOssPointer(value);
if (ossPointer == null) {
return value;
}
try {
return ossStorageService.readObjectAsString(ossPointer.substring(OSS_POINTER_PREFIX.length()));
return transientPayloadStorageService.resolvePayload(value, "Read brand task parsed payload failed");
} catch (Exception ex) {
throw new BusinessException("Read brand task parsed payload failed");
}
}
private void deleteParsedPayloadIfNeeded(String oldValue, String newValue) {
String oldPointer = extractOssPointer(oldValue);
String newPointer = extractOssPointer(newValue);
if (oldPointer == null || oldPointer.equals(newPointer)) {
return;
}
try {
ossStorageService.deleteObject(oldPointer.substring(OSS_POINTER_PREFIX.length()));
} catch (Exception ignored) {
}
}
private String extractOssPointer(String value) {
if (isBlank(value)) {
return null;
}
if (value.startsWith(OSS_POINTER_PREFIX)) {
return value;
}
try {
String decoded = objectMapper.readValue(value, String.class);
return decoded != null && decoded.startsWith(OSS_POINTER_PREFIX) ? decoded : null;
} catch (Exception ignored) {
return null;
}
}
private String writeJson(Object value, String message) {
try {
return objectMapper.writeValueAsString(value);

4
backend-java/src/main/java/com/nanri/aiimage/modules/convert/service/ConvertRunService.java
View File

@@ -205,9 +205,7 @@ public class ConvertRunService {
vo.setResultId(entity.getId());
vo.setSourceFilename(entity.getSourceFilename());
vo.setOutputFilename(entity.getResultFilename());
vo.setDownloadUrl(entity.getSuccess() != null && entity.getSuccess() == 1
? ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl())
: null);
vo.setDownloadUrl(null);
vo.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1);
vo.setError(entity.getErrorMessage());
return vo;

4
backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeRunService.java
View File

@@ -229,9 +229,7 @@ public class DedupeRunService {
vo.setResultId(entity.getId());
vo.setSourceFilename(entity.getSourceFilename());
vo.setOutputFilename(entity.getResultFilename());
vo.setDownloadUrl(entity.getSuccess() != null && entity.getSuccess() == 1
? ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl())
: null);
vo.setDownloadUrl(null);
vo.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1);
vo.setError(entity.getErrorMessage());
return vo;

4
backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/model/vo/DeleteBrandResultItemVo.java
View File

@@ -50,6 +50,10 @@ public class DeleteBrandResultItemVo {
@Schema(description = "下载地址")
private String downloadUrl;
private Long fileJobId;
private String fileStatus;
private String fileError;
private Boolean fileReady;
@Schema(description = "任务ID")
private Long taskId;

244
backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandRunService.java
View File

@@ -34,6 +34,8 @@ import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoShopMatchResultVo;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopIndexService;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
@@ -80,6 +82,7 @@ public class DeleteBrandRunService {
private final ObjectMapper objectMapper;
private final DeleteBrandProgressProperties deleteBrandProgressProperties;
private final TaskPressureProperties taskPressureProperties;
private final TaskFileJobService taskFileJobService;
public DeleteBrandRunVo run(DeleteBrandRunRequest request) {
if (request.getUserId() == null || request.getUserId() <= 0) {
@@ -271,7 +274,8 @@ public class DeleteBrandRunService {
item.setSourceFilename(entity.getSourceFilename());
item.setShopName(entity.getSourceFilename() == null ? null : FileUtil.mainName(entity.getSourceFilename()));
item.setOutputFilename(entity.getResultFilename());
item.setDownloadUrl(entity.getResultFileUrl() == null ? null : ossStorageService.getPublicUrl(entity.getResultFileUrl()));
item.setDownloadUrl(null);
attachFileJobState(item, entity);
item.setTotalRows(entity.getRowCount());
item.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1);
item.setError(entity.getErrorMessage());
@@ -320,6 +324,18 @@ public class DeleteBrandRunService {
return vo;
}
private void attachFileJobState(DeleteBrandResultItemVo item, FileResultEntity entity) {
TaskFileJobEntity job = taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId());
item.setFileReady(entity.getResultFileUrl() != null && !entity.getResultFileUrl().isBlank());
if (job == null) {
item.setFileStatus(Boolean.TRUE.equals(item.getFileReady()) ? "SUCCESS" : null);
return;
}
item.setFileJobId(job.getId());
item.setFileStatus(job.getStatus());
item.setFileError(job.getErrorMessage());
}
public DeleteBrandTaskDeletionStatusVo getTaskDeletionStatus(Long taskId, Long userId) {
FileTaskEntity task = loadTaskForExecution(taskId);
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !userId.equals(task.getUserId())) {
@@ -373,12 +389,19 @@ public class DeleteBrandRunService {
Map<String, Map<String, DeleteBrandCountryAsinVo>> grouped = new LinkedHashMap<>();
Map<String, String> displayCountryNames = new LinkedHashMap<>();
for (int rowNum = 2; rowNum <= sheet.getLastRowNum(); rowNum++) {
int firstDataRow = pairs.stream()
.mapToInt(CountryColumnPair::dataStartRowIndex)
.min()
.orElse(2);
for (int rowNum = firstDataRow; rowNum <= sheet.getLastRowNum(); rowNum++) {
Row row = sheet.getRow(rowNum);
if (row == null) {
continue;
}
for (CountryColumnPair pair : pairs) {
if (rowNum < pair.dataStartRowIndex()) {
continue;
}
String country = pair.country();
String asin = normalizeCellText(formatter.formatCellValue(row.getCell(pair.asinColumnIndex())));
String status = normalizeCellText(formatter.formatCellValue(row.getCell(pair.statusColumnIndex())));
@@ -457,7 +480,10 @@ public class DeleteBrandRunService {
continue;
}
if ("删除ASIN".equals(firstHeader) && "状态".equals(secondHeader)) {
pairs.add(new CountryColumnPair(title, i, i + 1));
pairs.add(new CountryColumnPair(title, i, i + 1, 2));
i++;
} else if (!"店铺名".equals(title) && "状态".equals(secondHeader)) {
pairs.add(new CountryColumnPair(title, i, i + 1, 1));
i++;
}
}
@@ -486,7 +512,7 @@ public class DeleteBrandRunService {
.replaceAll("\\s+", " ");
}
private record CountryColumnPair(String country, int asinColumnIndex, int statusColumnIndex) {
private record CountryColumnPair(String country, int asinColumnIndex, int statusColumnIndex, int dataStartRowIndex) {
}
private String resolveFileIdentity(DeleteBrandSourceFileDto sourceFile) {
@@ -1085,58 +1111,111 @@ public class DeleteBrandRunService {
return;
}
Map<String, DeleteBrandParsedFileCacheDto> parsedPayload = deleteBrandTaskStorageService.loadParsedPayload(taskId);
if (parsedPayload == null || parsedPayload.isEmpty()) {
int expectedFiles = deleteBrandTaskStorageService.countParsedPayloadScopes(taskId);
if (expectedFiles <= 0) {
tryFinalizeTaskFromTerminalResults(task);
return;
}
int expectedFiles = parsedPayload.size();
// 移除 finishedFilesHint 的强依赖，因为 Redis 里的 finished_files 可能是旧值
// 既然进入 tryFinalizeTask，就说明有新分片到达，应该以 loadMergedChunks 为准
Map<String, List<DeleteBrandResultFileDto>> mergedByFile = deleteBrandTaskStorageService.loadMergedChunks(taskId);
int finishedFiles = countCompletedFiles(mergedByFile);
int finishedFiles = deleteBrandTaskStorageService.countCompletedScopes(taskId);
if (finishedFiles < expectedFiles) {
Map<String, String> progress = new LinkedHashMap<>();
progress.put("finished_files", String.valueOf(finishedFiles));
// 定时补偿只负责“再试一次 finalize”，不能把缺片任务重新续命
// 否则 stale-check 会一直看不到超时任务
if (!fromCompensation) {
progress.put("updated_at", String.valueOf(System.currentTimeMillis()));
}
deleteBrandTaskCacheService.saveProgress(taskId, progress);
task.setSuccessFileCount(finishedFiles);
if (!fromCompensation) {
task.setUpdatedAt(LocalDateTime.now());
}
fileTaskMapper.updateById(task);
deleteBrandTaskCacheService.saveTaskCache(task);
updateTaskFinalizeProgress(task, finishedFiles, fromCompensation);
log.info("[DeleteBrand] tryFinalizeTask skipping -> finishedFiles ({}) < expectedFiles ({})", finishedFiles, expectedFiles);
return;
}
Map<String, String> progress = new LinkedHashMap<>();
progress.put("finished_files", String.valueOf(finishedFiles));
progress.put("updated_at", String.valueOf(System.currentTimeMillis()));
if (fromCompensation) {
progress.put("last_heartbeat_at", String.valueOf(System.currentTimeMillis()));
updateTaskFinalizeProgress(task, finishedFiles, true);
Map<String, DeleteBrandParsedFileCacheDto> parsedPayload = deleteBrandTaskStorageService.loadParsedPayload(taskId);
if (parsedPayload == null || parsedPayload.isEmpty()) {
tryFinalizeTaskFromTerminalResults(task);
return;
}
deleteBrandTaskCacheService.saveProgress(taskId, progress, true);
task.setSuccessFileCount(finishedFiles);
task.setUpdatedAt(LocalDateTime.now());
fileTaskMapper.updateById(task);
deleteBrandTaskCacheService.saveTaskCache(task);
Map<String, List<DeleteBrandResultFileDto>> mergedByFile = deleteBrandTaskStorageService.loadMergedChunks(taskId);
log.info("[DeleteBrand] tryFinalizeTask calling finalizeTask -> finishedFiles: {}, expectedFiles: {}", finishedFiles, expectedFiles);
finalizeTask(task, parsedPayload, mergedByFile);
}
private void updateTaskFinalizeProgress(FileTaskEntity task, int finishedFiles, boolean fromCompensation) {
Map<String, String> progress = new LinkedHashMap<>();
progress.put("finished_files", String.valueOf(finishedFiles));
if (!fromCompensation) {
progress.put("updated_at", String.valueOf(System.currentTimeMillis()));
}
deleteBrandTaskCacheService.saveProgress(task.getId(), progress);
Integer currentFinished = task.getSuccessFileCount();
if (fromCompensation && currentFinished != null && currentFinished == finishedFiles) {
return;
}
task.setSuccessFileCount(finishedFiles);
if (!fromCompensation) {
task.setUpdatedAt(LocalDateTime.now());
}
fileTaskMapper.updateById(task);
deleteBrandTaskCacheService.saveTaskCache(task);
}
private void tryFinalizeTaskFromTerminalResults(FileTaskEntity task) {
if (task == null || task.getId() == null) {
return;
}
List<FileResultEntity> resultEntities = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getModuleType, MODULE_TYPE)
.eq(FileResultEntity::getTaskId, task.getId())
.orderByAsc(FileResultEntity::getId));
if (resultEntities == null || resultEntities.isEmpty()) {
return;
}
if (taskFileJobService.countUnfinishedAssembleJobs(task.getId(), MODULE_TYPE) > 0L) {
return;
}
int successCount = 0;
int failedCount = 0;
for (FileResultEntity entity : resultEntities) {
boolean success = (entity.getSuccess() != null && entity.getSuccess() == 1)
|| (entity.getResultFileUrl() != null && !entity.getResultFileUrl().isBlank());
boolean failed = !success && entity.getErrorMessage() != null && !entity.getErrorMessage().isBlank();
if (!success && !failed) {
return;
}
if (success) {
successCount++;
} else {
failedCount++;
}
}
if (task.getSourceFileCount() != null
&& task.getSourceFileCount() > 0
&& successCount + failedCount < task.getSourceFileCount()) {
return;
}
task.setStatus(successCount > 0 ? "SUCCESS" : "FAILED");
task.setSuccessFileCount(successCount);
task.setFailedFileCount(failedCount);
task.setErrorMessage(successCount > 0 ? null : "任务中间数据缺失，已按结果行状态自动收尾");
task.setUpdatedAt(LocalDateTime.now());
task.setFinishedAt(LocalDateTime.now());
fileTaskMapper.updateById(task);
deleteBrandTaskCacheService.saveTaskCache(task);
deleteBrandTaskCacheService.saveProgress(task.getId(), Map.of(
"phase", successCount > 0 ? "success" : DeleteBrandTaskCacheService.PHASE_FAILED,
"finished_files", String.valueOf(successCount),
"updated_at", String.valueOf(System.currentTimeMillis())
), true);
log.info("[DeleteBrand] finalized orphan running task from terminal results -> taskId: {}, status: {}, successFiles: {}, failedFiles: {}",
task.getId(), task.getStatus(), successCount, failedCount);
}
private int countCompletedFiles(Map<String, List<DeleteBrandResultFileDto>> mergedByFile) {
int finishedFiles = 0;
for (List<DeleteBrandResultFileDto> chunks : mergedByFile.values()) {
@@ -1205,28 +1284,22 @@ public class DeleteBrandRunService {
if (!isMergedFileCompleted(chunks)) {
throw new BusinessException("存在未完成的结果分片: " + parsedFile.getSourceFilename());
}
MergedDeleteBrandFile mergedFile = mergeChunks(parsedFile, chunks);
File outputFile = buildResultWorkbookPreserveLayout(task.getId(), parsedFile, mergedFile);
mergeChunks(parsedFile, chunks);
deleteBrandTaskCacheService.saveProgress(task.getId(), Map.of(
"phase", DeleteBrandTaskCacheService.PHASE_UPLOADING,
"file_name", blankToEmpty(parsedFile.getSourceFilename()),
"updated_at", String.valueOf(System.currentTimeMillis())
));
String objectKey = ossStorageService.uploadResultFile(outputFile, MODULE_TYPE);
FileResultEntity resultEntity = resultEntityByIdentity.get(fileIdentity);
if (resultEntity == null) {
throw new BusinessException("结果记录不存在: " + parsedFile.getSourceFilename());
}
resultEntity.setResultFilename(outputFile.getName());
resultEntity.setResultFileUrl(objectKey);
resultEntity.setResultFileSize(outputFile.length());
String outputFilename = buildResultFilename(parsedFile);
resultEntity.setResultFilename(outputFilename);
resultEntity.setResultFileUrl(null);
resultEntity.setResultFileSize(0L);
resultEntity.setResultContentType(CONTENT_TYPE_XLSX);
resultEntity.setRowCount(parsedFile.getTotalRows());
resultEntity.setSuccess(1);
resultEntity.setErrorMessage(null);
fileResultMapper.updateById(resultEntity);
taskFileJobService.enqueueAssembleResult(task.getId(), MODULE_TYPE, resultEntity.getId(), fileIdentity);
DeleteBrandResultItemVo item = new DeleteBrandResultItemVo();
item.setResultId(resultEntity.getId());
@@ -1235,8 +1308,8 @@ public class DeleteBrandRunService {
item.setSourceFilename(parsedFile.getSourceFilename());
item.setShopName(parsedFile.getShopName());
item.setCompanyName(parsedFile.getCompanyName());
item.setOutputFilename(outputFile.getName());
item.setDownloadUrl(ossStorageService.getPublicUrl(objectKey));
item.setOutputFilename(outputFilename);
item.setDownloadUrl(null);
item.setTotalRows(parsedFile.getTotalRows());
item.setCountryCount(parsedFile.getCountries() == null ? 0 : parsedFile.getCountries().size());
item.setCountries(parsedFile.getCountries());
@@ -1263,8 +1336,6 @@ public class DeleteBrandRunService {
task.setFinishedAt(LocalDateTime.now());
fileTaskMapper.updateById(task);
deleteBrandTaskCacheService.saveTaskCache(task);
deleteBrandTaskStorageService.deleteTaskData(task.getId());
deleteBrandTaskCacheService.saveProgress(task.getId(), Map.of(
"phase", "success",
"file_index", String.valueOf(parsedPayload.size()),
@@ -1293,6 +1364,75 @@ public class DeleteBrandRunService {
}
}
public void processResultFileJob(TaskFileJobEntity job) {
if (job == null || job.getTaskId() == null || job.getResultId() == null) {
throw new BusinessException("结果文件任务参数不完整");
}
FileTaskEntity task = fileTaskMapper.selectById(job.getTaskId());
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
throw new BusinessException("任务不存在");
}
FileResultEntity resultEntity = fileResultMapper.selectById(job.getResultId());
if (resultEntity == null || !MODULE_TYPE.equals(resultEntity.getModuleType())) {
throw new BusinessException("结果记录不存在");
}
Map<String, DeleteBrandParsedFileCacheDto> parsedPayload = deleteBrandTaskStorageService.loadParsedPayload(job.getTaskId());
Map<String, List<DeleteBrandResultFileDto>> mergedByFile = deleteBrandTaskStorageService.loadMergedChunks(job.getTaskId());
String fileIdentity = blankToEmpty(job.getScopeKey());
if (fileIdentity.isBlank()) {
fileIdentity = blankToEmpty(resultEntity.getSourceFileUrl());
}
if (fileIdentity.isBlank()) {
fileIdentity = blankToEmpty(resultEntity.getSourceFilename());
}
DeleteBrandParsedFileCacheDto parsedFile = parsedPayload.get(fileIdentity);
if (parsedFile == null) {
throw new BusinessException("任务原始数据不存在: " + fileIdentity);
}
List<DeleteBrandResultFileDto> chunks = mergedByFile.get(fileIdentity);
if (!isMergedFileCompleted(chunks)) {
throw new BusinessException("结果分片未完整: " + fileIdentity);
}
MergedDeleteBrandFile mergedFile = mergeChunks(parsedFile, chunks);
File outputFile = buildResultWorkbookPreserveLayout(job.getTaskId(), parsedFile, mergedFile);
try {
deleteBrandTaskCacheService.saveProgress(job.getTaskId(), Map.of(
"phase", DeleteBrandTaskCacheService.PHASE_UPLOADING,
"file_name", blankToEmpty(parsedFile.getSourceFilename()),
"updated_at", String.valueOf(System.currentTimeMillis())
));
String objectKey = ossStorageService.uploadResultFile(outputFile, MODULE_TYPE);
resultEntity.setResultFilename(outputFile.getName());
resultEntity.setResultFileUrl(objectKey);
resultEntity.setResultFileSize(outputFile.length());
resultEntity.setResultContentType(CONTENT_TYPE_XLSX);
resultEntity.setRowCount(parsedFile.getTotalRows());
resultEntity.setSuccess(1);
resultEntity.setErrorMessage(null);
fileResultMapper.updateById(resultEntity);
} finally {
deleteQuietly(outputFile);
}
}
public void cleanupResultFileJob(TaskFileJobEntity job) {
if (job == null || job.getTaskId() == null) {
return;
}
if (taskFileJobService.countUnfinishedAssembleJobs(job.getTaskId(), MODULE_TYPE) == 0L) {
deleteBrandTaskStorageService.deleteTaskData(job.getTaskId());
}
}
private String buildResultFilename(DeleteBrandParsedFileCacheDto parsedFile) {
String filename = blankToDefault(parsedFile.getSourceFilename(), "delete-brand-result.xlsx");
String lower = filename.toLowerCase(Locale.ROOT);
if (!lower.endsWith(".xlsx") && !lower.endsWith(".xls")) {
return filename + ".xlsx";
}
return filename;
}
private MergedDeleteBrandFile mergeChunks(DeleteBrandParsedFileCacheDto parsedFile, List<DeleteBrandResultFileDto> chunks) {
Integer chunkTotal = chunks.get(0).getChunkTotal();
for (DeleteBrandResultFileDto chunk : chunks) {

44
backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandStaleTaskService.java
View File

@@ -185,20 +185,24 @@ public class DeleteBrandStaleTaskService {
if (runningTasks.isEmpty()) {
return stats;
}
Map<Long, Long> heartbeatByTaskId = productRiskTaskCacheService.getTaskHeartbeatMillisBatch(
runningTasks.stream().map(FileTaskEntity::getId).toList());
log.info("[stale-check] product-risk candidates={} threshold={} timeoutMinutes={} initialThreshold={} initialTimeoutMinutes={}",
runningTasks.size(), threshold, minutes, initialThreshold, initialMinutes);
for (FileTaskEntity task : runningTasks) {
boolean hasUploadedPayload = productRiskTaskCacheService.hasAnyShopMergedPayload(task.getId());
long lastPayloadHeartbeatMillis = productRiskTaskCacheService.getTaskHeartbeatMillis(task.getId());
long lastPayloadHeartbeatMillis = heartbeatByTaskId.getOrDefault(task.getId(), 0L);
boolean hasFinishedRows = (task.getSuccessFileCount() != null && task.getSuccessFileCount() > 0)
|| (task.getFailedFileCount() != null && task.getFailedFileCount() > 0);
boolean hasStartedProgress = hasUploadedPayload || hasFinishedRows;
if (lastPayloadHeartbeatMillis > 0L && nowMillis - lastPayloadHeartbeatMillis < staleTimeoutMillis) {
stats.skippedTaskCount++;
log.info("[stale-check] product-risk skip recent-task-heartbeat taskId={} lastPayloadHeartbeatMillis={} timeoutMinutes={} updatedAt={}",
task.getId(), lastPayloadHeartbeatMillis, minutes, task.getUpdatedAt());
continue;
}
boolean hasStartedProgress = lastPayloadHeartbeatMillis > 0L || hasFinishedRows;
if (!hasStartedProgress) {
hasStartedProgress = productRiskTaskCacheService.hasAnyShopMergedPayload(task.getId());
}
if (!hasStartedProgress && task.getCreatedAt() != null && task.getCreatedAt().isAfter(initialThreshold)) {
stats.skippedTaskCount++;
log.info("[stale-check] product-risk skip initial-grace taskId={} createdAt={} initialThreshold={}",
@@ -250,18 +254,22 @@ public class DeleteBrandStaleTaskService {
if (runningTasks.isEmpty()) {
return stats;
}
Map<Long, Long> heartbeatByTaskId = priceTrackTaskCacheService.getTaskHeartbeatMillisBatch(
runningTasks.stream().map(FileTaskEntity::getId).toList());
log.info("[stale-check] price-track candidates={} threshold={} timeoutMinutes={} initialThreshold={} initialTimeoutMinutes={}",
runningTasks.size(), threshold, minutes, initialThreshold, initialMinutes);
for (FileTaskEntity task : runningTasks) {
long lastHeartbeatMillis = priceTrackTaskCacheService.getTaskHeartbeatMillis(task.getId());
boolean hasUploadedPayload = priceTrackTaskCacheService.hasAnyShopMergedPayload(task.getId());
long lastHeartbeatMillis = heartbeatByTaskId.getOrDefault(task.getId(), 0L);
boolean hasFinishedRows = (task.getSuccessFileCount() != null && task.getSuccessFileCount() > 0)
|| (task.getFailedFileCount() != null && task.getFailedFileCount() > 0);
boolean hasStartedProgress = lastHeartbeatMillis > 0L || hasUploadedPayload || hasFinishedRows;
if (lastHeartbeatMillis > 0L && nowMillis - lastHeartbeatMillis < staleTimeoutMillis) {
stats.skippedTaskCount++;
continue;
}
boolean hasStartedProgress = lastHeartbeatMillis > 0L || hasFinishedRows;
if (!hasStartedProgress) {
hasStartedProgress = priceTrackTaskCacheService.hasAnyShopMergedPayload(task.getId());
}
if (!hasStartedProgress && task.getCreatedAt() != null && task.getCreatedAt().isAfter(initialThreshold)) {
stats.skippedTaskCount++;
continue;
@@ -312,20 +320,24 @@ public class DeleteBrandStaleTaskService {
if (runningTasks.isEmpty()) {
return stats;
}
Map<Long, Long> heartbeatByTaskId = shopMatchTaskCacheService.getTaskHeartbeatMillisBatch(
runningTasks.stream().map(FileTaskEntity::getId).toList());
log.info("[stale-check] shop-match candidates={} threshold={} timeoutMinutes={} initialThreshold={} initialTimeoutMinutes={}",
runningTasks.size(), threshold, minutes, initialThreshold, initialMinutes);
for (FileTaskEntity task : runningTasks) {
long lastHeartbeatMillis = shopMatchTaskCacheService.getTaskHeartbeatMillis(task.getId());
boolean hasUploadedPayload = shopMatchTaskCacheService.hasAnyShopMergedPayload(task.getId());
long lastHeartbeatMillis = heartbeatByTaskId.getOrDefault(task.getId(), 0L);
boolean hasFinishedRows = (task.getSuccessFileCount() != null && task.getSuccessFileCount() > 0)
|| (task.getFailedFileCount() != null && task.getFailedFileCount() > 0);
boolean hasStartedProgress = lastHeartbeatMillis > 0L || hasUploadedPayload || hasFinishedRows;
if (lastHeartbeatMillis > 0L && nowMillis - lastHeartbeatMillis < staleTimeoutMillis) {
stats.skippedTaskCount++;
log.info("[stale-check] shop-match skip recent-task-heartbeat taskId={} lastHeartbeatMillis={} timeoutMinutes={} updatedAt={}",
task.getId(), lastHeartbeatMillis, minutes, task.getUpdatedAt());
continue;
}
boolean hasStartedProgress = lastHeartbeatMillis > 0L || hasFinishedRows;
if (!hasStartedProgress) {
hasStartedProgress = shopMatchTaskCacheService.hasAnyShopMergedPayload(task.getId());
}
if (!hasStartedProgress && task.getCreatedAt() != null && task.getCreatedAt().isAfter(initialThreshold)) {
stats.skippedTaskCount++;
log.info("[stale-check] shop-match skip initial-grace taskId={} createdAt={} initialThreshold={}",
@@ -378,16 +390,20 @@ public class DeleteBrandStaleTaskService {
if (runningTasks.isEmpty()) {
return stats;
}
Map<Long, Long> heartbeatByTaskId = patrolDeleteTaskCacheService.getTaskHeartbeatMillisBatch(
runningTasks.stream().map(FileTaskEntity::getId).toList());
for (FileTaskEntity task : runningTasks) {
long lastHeartbeatMillis = patrolDeleteTaskCacheService.getTaskHeartbeatMillis(task.getId());
boolean hasUploadedPayload = patrolDeleteTaskCacheService.hasAnyShopMergedPayload(task.getId());
long lastHeartbeatMillis = heartbeatByTaskId.getOrDefault(task.getId(), 0L);
boolean hasFinishedRows = (task.getSuccessFileCount() != null && task.getSuccessFileCount() > 0)
|| (task.getFailedFileCount() != null && task.getFailedFileCount() > 0);
boolean hasStartedProgress = lastHeartbeatMillis > 0L || hasUploadedPayload || hasFinishedRows;
if (lastHeartbeatMillis > 0L && nowMillis - lastHeartbeatMillis < staleTimeoutMillis) {
stats.skippedTaskCount++;
continue;
}
boolean hasStartedProgress = lastHeartbeatMillis > 0L || hasFinishedRows;
if (!hasStartedProgress) {
hasStartedProgress = patrolDeleteTaskCacheService.hasAnyShopMergedPayload(task.getId());
}
if (!hasStartedProgress && task.getCreatedAt() != null && task.getCreatedAt().isAfter(initialThreshold)) {
stats.skippedTaskCount++;
continue;
@@ -431,6 +447,10 @@ public class DeleteBrandStaleTaskService {
java.util.List<FileTaskEntity> runningTasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getModuleType, MODULE_TYPE_DELETE_BRAND)
.eq(FileTaskEntity::getStatus, "RUNNING")
.and(wrapper -> wrapper
.gt(FileTaskEntity::getSuccessFileCount, 0)
.or()
.gt(FileTaskEntity::getFailedFileCount, 0))
.orderByAsc(FileTaskEntity::getUpdatedAt)
.last("limit 100"));

37
backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandTaskCacheService.java
View File

@@ -3,6 +3,7 @@ package com.nanri.aiimage.modules.deletebrand.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.TaskPressureProperties;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
@@ -11,6 +12,7 @@ import java.util.concurrent.ConcurrentHashMap;
@Service
@RequiredArgsConstructor
@Slf4j
public class DeleteBrandTaskCacheService {
public static final String PHASE_CRAWLING = "crawling";
@@ -52,9 +54,13 @@ public class DeleteBrandTaskCacheService {
}
String key = buildProgressKey(taskId);
try {
stringRedisTemplate.opsForHash().putAll(key, merged);
stringRedisTemplate.expire(key, Duration.ofHours(PAYLOAD_TTL_HOURS));
progressRedisFlushAt.put(taskId, now);
} catch (Exception ex) {
log.warn("[delete-brand-cache] save progress degraded taskId={} msg={}", taskId, ex.getMessage());
}
}
public java.util.Map<Object, Object> getProgress(Long taskId) {
@@ -63,7 +69,13 @@ public class DeleteBrandTaskCacheService {
if (cached != null && cached.isFresh(now)) {
return new java.util.LinkedHashMap<>(cached.values());
}
java.util.Map<Object, Object> values = stringRedisTemplate.opsForHash().entries(buildProgressKey(taskId));
java.util.Map<Object, Object> values;
try {
values = stringRedisTemplate.opsForHash().entries(buildProgressKey(taskId));
} catch (Exception ex) {
log.warn("[delete-brand-cache] load progress degraded taskId={} msg={}", taskId, ex.getMessage());
return java.util.Collections.emptyMap();
}
if (!values.isEmpty()) {
progressLocalCache.put(taskId, new LocalProgressCacheEntry(now, toStringMap(values)));
}
@@ -98,7 +110,9 @@ public class DeleteBrandTaskCacheService {
return result;
}
java.util.List<Object> pipelineResults = stringRedisTemplate.executePipelined(
java.util.List<Object> pipelineResults;
try {
pipelineResults = stringRedisTemplate.executePipelined(
(org.springframework.data.redis.core.RedisCallback<Object>) connection -> {
org.springframework.data.redis.serializer.RedisSerializer<String> serializer =
stringRedisTemplate.getStringSerializer();
@@ -107,6 +121,13 @@ public class DeleteBrandTaskCacheService {
}
return null;
});
} catch (Exception ex) {
log.warn("[delete-brand-cache] batch load progress degraded taskIds={} msg={}", missingTaskIds, ex.getMessage());
for (Long taskId : missingTaskIds) {
result.put(taskId, java.util.Collections.emptyMap());
}
return result;
}
for (int i = 0; i < missingTaskIds.size(); i++) {
Long taskId = missingTaskIds.get(i);
@@ -129,8 +150,12 @@ public class DeleteBrandTaskCacheService {
progressLocalCache.remove(taskId);
progressRedisFlushAt.remove(taskId);
taskEntityLocalCache.remove(taskId);
try {
stringRedisTemplate.delete(buildProgressKey(taskId));
stringRedisTemplate.delete(buildTaskEntityKey(taskId));
} catch (Exception ex) {
log.warn("[delete-brand-cache] delete degraded taskId={} msg={}", taskId, ex.getMessage());
}
}
public void saveTaskCache(com.nanri.aiimage.modules.task.model.entity.FileTaskEntity task) {
@@ -180,7 +205,13 @@ public class DeleteBrandTaskCacheService {
}
java.util.List<String> keys = missingIds.stream().map(this::buildTaskEntityKey).toList();
java.util.List<String> values = stringRedisTemplate.opsForValue().multiGet(keys);
java.util.List<String> values;
try {
values = stringRedisTemplate.opsForValue().multiGet(keys);
} catch (Exception ex) {
log.warn("[delete-brand-cache] batch load task cache degraded taskIds={} msg={}", missingIds, ex.getMessage());
return result;
}
for (int i = 0; i < missingIds.size(); i++) {
Long taskId = missingIds.get(i);

100
backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandTaskStorageService.java
View File

@@ -6,11 +6,11 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.deletebrand.model.cache.DeleteBrandParsedFileCacheDto;
import com.nanri.aiimage.modules.deletebrand.model.dto.DeleteBrandResultFileDto;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.dao.DuplicateKeyException;
@@ -32,12 +32,11 @@ import java.util.Map;
public class DeleteBrandTaskStorageService {
private static final String MODULE_TYPE = "DELETE_BRAND";
private static final String OSS_POINTER_PREFIX = "oss:";
private final TaskChunkMapper taskChunkMapper;
private final TaskScopeStateMapper taskScopeStateMapper;
private final ObjectMapper objectMapper;
private final OssStorageService ossStorageService;
private final TransientPayloadStorageService transientPayloadStorageService;
@Transactional
public void saveParsedPayload(Long taskId, Map<String, DeleteBrandParsedFileCacheDto> parsedPayloadByScope) {
@@ -81,7 +80,7 @@ public class DeleteBrandTaskStorageService {
if (existing == null) {
throw new BusinessException("failed to save delete-brand parsed payload");
}
deleteParsedPayloadIfNeeded(existing.getParsedPayloadJson(), parsedPayloadPointer);
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(existing.getParsedPayloadJson(), parsedPayloadPointer);
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getId, existing.getId())
.set(TaskScopeStateEntity::getScopeKey, scopeKey)
@@ -120,6 +119,28 @@ public class DeleteBrandTaskStorageService {
return result;
}
public int countParsedPayloadScopes(Long taskId) {
if (taskId == null || taskId <= 0) {
return 0;
}
Long count = taskScopeStateMapper.selectCount(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.isNotNull(TaskScopeStateEntity::getParsedPayloadJson));
return count == null ? 0 : count.intValue();
}
public int countCompletedScopes(Long taskId) {
if (taskId == null || taskId <= 0) {
return 0;
}
Long count = taskScopeStateMapper.selectCount(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.eq(TaskScopeStateEntity::getCompleted, 1));
return count == null ? 0 : count.intValue();
}
@Transactional
public boolean storeResultChunkIfChanged(Long taskId, String scopeKey, Integer chunkIndex, DeleteBrandResultFileDto chunk) {
if (taskId == null || taskId <= 0) {
@@ -145,6 +166,7 @@ public class DeleteBrandTaskStorageService {
.last("limit 1"));
boolean changed = true;
if (existingChunk == null) {
String storedPayload = transientPayloadStorageService.storeChunkPayload(MODULE_TYPE, taskId, scopeHash, chunkIndex, payloadJson);
TaskChunkEntity entity = new TaskChunkEntity();
entity.setTaskId(taskId);
entity.setModuleType(MODULE_TYPE);
@@ -152,7 +174,7 @@ public class DeleteBrandTaskStorageService {
entity.setScopeHash(scopeHash);
entity.setChunkIndex(chunkIndex);
entity.setChunkTotal(chunk.getChunkTotal() == null ? 0 : chunk.getChunkTotal());
entity.setPayloadJson(payloadJson);
entity.setPayloadJson(storedPayload);
entity.setPayloadHash(payloadHash);
entity.setCreatedAt(now);
entity.setUpdatedAt(now);
@@ -169,11 +191,13 @@ public class DeleteBrandTaskStorageService {
changed = !payloadHash.equals(concurrentChunk.getPayloadHash());
concurrentChunk.setScopeKey(normalizedScopeKey);
concurrentChunk.setChunkTotal(chunk.getChunkTotal() == null ? 0 : chunk.getChunkTotal());
concurrentChunk.setPayloadJson(payloadJson);
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(concurrentChunk.getPayloadJson(), storedPayload);
concurrentChunk.setPayloadJson(storedPayload);
concurrentChunk.setPayloadHash(payloadHash);
concurrentChunk.setUpdatedAt(now);
taskChunkMapper.updateById(concurrentChunk);
} else {
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
throw ex;
}
}
@@ -181,7 +205,9 @@ public class DeleteBrandTaskStorageService {
changed = !payloadHash.equals(existingChunk.getPayloadHash());
existingChunk.setScopeKey(normalizedScopeKey);
existingChunk.setChunkTotal(chunk.getChunkTotal() == null ? 0 : chunk.getChunkTotal());
existingChunk.setPayloadJson(payloadJson);
String storedPayload = transientPayloadStorageService.storeChunkPayload(MODULE_TYPE, taskId, scopeHash, chunkIndex, payloadJson);
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(existingChunk.getPayloadJson(), storedPayload);
existingChunk.setPayloadJson(storedPayload);
existingChunk.setPayloadHash(payloadHash);
existingChunk.setUpdatedAt(now);
taskChunkMapper.updateById(existingChunk);
@@ -210,7 +236,8 @@ public class DeleteBrandTaskStorageService {
continue;
}
try {
DeleteBrandResultFileDto dto = objectMapper.readValue(entity.getPayloadJson(), DeleteBrandResultFileDto.class);
String payloadJson = transientPayloadStorageService.resolvePayload(entity.getPayloadJson(), "failed to load delete-brand result chunks");
DeleteBrandResultFileDto dto = objectMapper.readValue(payloadJson, DeleteBrandResultFileDto.class);
grouped.computeIfAbsent(entity.getScopeKey(), ignored -> new ArrayList<>()).add(dto);
} catch (Exception ex) {
throw new BusinessException("failed to load delete-brand result chunks");
@@ -228,12 +255,22 @@ public class DeleteBrandTaskStorageService {
return;
}
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
.select(TaskScopeStateEntity::getParsedPayloadJson)
.select(TaskScopeStateEntity::getParsedPayloadJson, TaskScopeStateEntity::getStateJson)
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
if (states != null) {
for (TaskScopeStateEntity state : states) {
deleteParsedPayloadIfNeeded(state.getParsedPayloadJson(), null);
transientPayloadStorageService.deletePayloadIfPresent(state.getParsedPayloadJson());
transientPayloadStorageService.deletePayloadIfPresent(state.getStateJson());
}
}
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
.select(TaskChunkEntity::getPayloadJson)
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
if (chunks != null) {
for (TaskChunkEntity chunk : chunks) {
transientPayloadStorageService.deletePayloadIfPresent(chunk.getPayloadJson());
}
}
taskChunkMapper.delete(new LambdaQueryWrapper<TaskChunkEntity>()
@@ -322,56 +359,17 @@ public class DeleteBrandTaskStorageService {
}
private String storeParsedPayload(Long taskId, String scopeHash, String payloadJson) {
try {
String pointer = OSS_POINTER_PREFIX + ossStorageService.uploadTaskParsedPayload(MODULE_TYPE, taskId, scopeHash, payloadJson);
return objectMapper.writeValueAsString(pointer);
} catch (Exception ex) {
throw new BusinessException("failed to save delete-brand parsed payload");
}
return transientPayloadStorageService.storeParsedPayload(MODULE_TYPE, taskId, scopeHash, payloadJson, true);
}
private String resolveParsedPayload(String value) {
if (isBlank(value)) {
return value;
}
String ossPointer = extractOssPointer(value);
if (ossPointer == null) {
return value;
}
try {
return ossStorageService.readObjectAsString(ossPointer.substring(OSS_POINTER_PREFIX.length()));
return transientPayloadStorageService.resolvePayload(value, "failed to load delete-brand parsed payload");
} catch (Exception ex) {
throw new BusinessException("failed to load delete-brand parsed payload");
}
}
private void deleteParsedPayloadIfNeeded(String oldValue, String newValue) {
String oldPointer = extractOssPointer(oldValue);
String newPointer = extractOssPointer(newValue);
if (oldPointer == null || oldPointer.equals(newPointer)) {
return;
}
try {
ossStorageService.deleteObject(oldPointer.substring(OSS_POINTER_PREFIX.length()));
} catch (Exception ignored) {
}
}
private String extractOssPointer(String value) {
if (isBlank(value)) {
return null;
}
if (value.startsWith(OSS_POINTER_PREFIX)) {
return value;
}
try {
String decoded = objectMapper.readValue(value, String.class);
return decoded != null && decoded.startsWith(OSS_POINTER_PREFIX) ? decoded : null;
} catch (Exception ignored) {
return null;
}
}
private String writeJson(Object value, String message) {
try {
return objectMapper.writeValueAsString(value);

85
backend-java/src/main/java/com/nanri/aiimage/modules/file/service/object/RustfsObjectStorageService.java Normal file
View File

@@ -0,0 +1,85 @@
package com.nanri.aiimage.modules.file.service.object;
import com.nanri.aiimage.config.TransientStorageProperties;
import io.minio.GetObjectArgs;
import io.minio.MinioClient;
import io.minio.PutObjectArgs;
import io.minio.RemoveObjectArgs;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import java.io.ByteArrayInputStream;
import java.nio.charset.StandardCharsets;
import java.util.Objects;
@Service
@RequiredArgsConstructor
public class RustfsObjectStorageService {
private final TransientStorageProperties properties;
public boolean isConfigured() {
return notBlank(properties.getEndpoint())
&& notBlank(properties.getBucket())
&& notBlank(properties.getAccessKeyId())
&& notBlank(properties.getAccessKeySecret());
}
public String uploadText(String objectKey, String content) {
if (!isConfigured()) {
throw new IllegalStateException("transient storage is not configured");
}
try (ByteArrayInputStream stream = new ByteArrayInputStream(
Objects.requireNonNullElse(content, "").getBytes(StandardCharsets.UTF_8))) {
buildClient().putObject(PutObjectArgs.builder()
.bucket(properties.getBucket())
.object(objectKey)
.stream(stream, stream.available(), -1)
.contentType("application/json")
.build());
return objectKey;
} catch (Exception ex) {
throw new IllegalStateException("failed to upload payload to transient storage", ex);
}
}
public String readObjectAsString(String objectKey) {
if (!isConfigured()) {
throw new IllegalStateException("transient storage is not configured");
}
try (var stream = buildClient().getObject(GetObjectArgs.builder()
.bucket(properties.getBucket())
.object(objectKey)
.build())) {
return new String(stream.readAllBytes(), StandardCharsets.UTF_8);
} catch (Exception ex) {
throw new IllegalStateException("failed to read payload from transient storage", ex);
}
}
public void deleteObject(String objectKey) {
if (!isConfigured() || !notBlank(objectKey)) {
return;
}
try {
buildClient().removeObject(RemoveObjectArgs.builder()
.bucket(properties.getBucket())
.object(objectKey)
.build());
} catch (Exception ex) {
throw new IllegalStateException("failed to delete payload from transient storage", ex);
}
}
private MinioClient buildClient() {
return MinioClient.builder()
.endpoint(properties.getEndpoint())
.credentials(properties.getAccessKeyId(), properties.getAccessKeySecret())
.region(properties.getRegion())
.build();
}
private boolean notBlank(String value) {
return value != null && !value.isBlank();
}
}

4
backend-java/src/main/java/com/nanri/aiimage/modules/patroldelete/model/vo/PatrolDeleteResultItemVo.java
View File

@@ -43,6 +43,10 @@ public class PatrolDeleteResultItemVo {
private LocalDateTime finishedAt;
private String outputFilename;
private String downloadUrl;
private Long fileJobId;
private String fileStatus;
private String fileError;
private Boolean fileReady;
@JsonProperty("countrySections")
private List<PatrolDeleteCountrySectionDto> countrySections = new ArrayList<>();

62
backend-java/src/main/java/com/nanri/aiimage/modules/patroldelete/service/PatrolDeleteTaskCacheService.java
View File

@@ -6,6 +6,7 @@ import com.nanri.aiimage.modules.patroldelete.model.dto.PatrolDeleteShopPayloadD
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.service.TaskScopePayloadStorageService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
@@ -13,11 +14,13 @@ import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@Service
@RequiredArgsConstructor
@Slf4j
public class PatrolDeleteTaskCacheService {
private static final String MODULE_TYPE = "PATROL_DELETE";
@@ -60,7 +63,13 @@ public class PatrolDeleteTaskCacheService {
if (taskId == null || taskId <= 0) {
return 0L;
}
String raw = stringRedisTemplate.opsForValue().get(buildTaskHeartbeatKey(taskId));
String raw;
try {
raw = stringRedisTemplate.opsForValue().get(buildTaskHeartbeatKey(taskId));
} catch (Exception ex) {
log.warn("[patrol-delete-cache] get heartbeat degraded taskId={} msg={}", taskId, ex.getMessage());
return 0L;
}
if (raw == null || raw.isBlank()) {
return 0L;
}
@@ -71,11 +80,50 @@ public class PatrolDeleteTaskCacheService {
}
}
public Map<Long, Long> getTaskHeartbeatMillisBatch(List<Long> taskIds) {
Map<Long, Long> result = new LinkedHashMap<>();
if (taskIds == null || taskIds.isEmpty()) {
return result;
}
List<Long> normalized = taskIds.stream()
.filter(id -> id != null && id > 0)
.distinct()
.toList();
if (normalized.isEmpty()) {
return result;
}
List<String> keys = normalized.stream().map(this::buildTaskHeartbeatKey).toList();
List<String> values;
try {
values = stringRedisTemplate.opsForValue().multiGet(keys);
} catch (Exception ex) {
log.warn("[patrol-delete-cache] batch get heartbeat degraded taskIds={} msg={}", normalized, ex.getMessage());
return result;
}
for (int i = 0; i < normalized.size(); i++) {
String raw = values != null && i < values.size() ? values.get(i) : null;
if (raw == null || raw.isBlank()) {
result.put(normalized.get(i), 0L);
continue;
}
try {
result.put(normalized.get(i), Long.parseLong(raw));
} catch (NumberFormatException ignored) {
result.put(normalized.get(i), 0L);
}
}
return result;
}
public void touchTaskHeartbeat(Long taskId) {
try {
stringRedisTemplate.opsForValue().set(
buildTaskHeartbeatKey(taskId),
String.valueOf(Instant.now().toEpochMilli()),
Duration.ofHours(PAYLOAD_TTL_HOURS));
} catch (Exception ex) {
log.warn("[patrol-delete-cache] touch heartbeat degraded taskId={} msg={}", taskId, ex.getMessage());
}
}
public void deleteTaskCache(Long taskId) {
@@ -83,8 +131,12 @@ public class PatrolDeleteTaskCacheService {
return;
}
taskEntityLocalCache.remove(taskId);
try {
stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId));
stringRedisTemplate.delete(buildTaskEntityKey(taskId));
} catch (Exception ex) {
log.warn("[patrol-delete-cache] delete cache degraded taskId={} msg={}", taskId, ex.getMessage());
}
taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE);
}
@@ -133,7 +185,13 @@ public class PatrolDeleteTaskCacheService {
return result;
}
java.util.List<String> keys = missingIds.stream().map(this::buildTaskEntityKey).toList();
java.util.List<String> values = stringRedisTemplate.opsForValue().multiGet(keys);
java.util.List<String> values;
try {
values = stringRedisTemplate.opsForValue().multiGet(keys);
} catch (Exception ex) {
log.warn("[patrol-delete-cache] batch get task cache degraded taskIds={} msg={}", missingIds, ex.getMessage());
return result;
}
for (int i = 0; i < missingIds.size(); i++) {
Long taskId = missingIds.get(i);
String val = values != null && i < values.size() ? values.get(i) : null;

137
backend-java/src/main/java/com/nanri/aiimage/modules/patroldelete/service/PatrolDeleteTaskService.java
View File

@@ -24,6 +24,10 @@ import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import com.nanri.aiimage.modules.task.service.TaskResultItemService;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@@ -56,6 +60,9 @@ public class PatrolDeleteTaskService {
private final ZiniaoShopSwitchService ziniaoShopSwitchService;
private final ObjectMapper objectMapper;
private final TaskPressureProperties taskPressureProperties;
private final TaskFileJobService taskFileJobService;
private final TaskResultItemService taskResultItemService;
private final TaskProgressSnapshotService taskProgressSnapshotService;
private FileTaskEntity loadTaskForExecution(Long taskId) {
Map<Long, FileTaskEntity> cachedTasks = taskCacheService.getTaskCacheBatch(List.of(taskId));
@@ -185,8 +192,7 @@ public class PatrolDeleteTaskService {
batch.getMissingTaskIds().add(taskId);
continue;
}
List<PatrolDeleteResultItemVo> snapshots = buildSnapshotFromDb(task, taskRows);
batch.getItems().addAll(snapshots);
batch.getItems().addAll(buildProgressItems(task, taskRows));
}
return batch;
}
@@ -496,7 +502,11 @@ public class PatrolDeleteTaskService {
private Map<Long, Map<Long, PatrolDeleteResultItemVo>> buildSnapshotMap(Map<Long, FileTaskEntity> taskMap) {
Map<Long, Map<Long, PatrolDeleteResultItemVo>> out = new LinkedHashMap<>();
for (Map.Entry<Long, FileTaskEntity> entry : taskMap.entrySet()) {
out.put(entry.getKey(), indexSnapshotByResultId(parseTaskSnapshots(entry.getValue().getResultJson())));
List<PatrolDeleteResultItemVo> snapshots = taskResultItemService.listResultSnapshots(entry.getKey(), MODULE_TYPE, PatrolDeleteResultItemVo.class);
if (snapshots.isEmpty()) {
snapshots = parseTaskSnapshots(entry.getValue().getResultJson());
}
out.put(entry.getKey(), indexSnapshotByResultId(snapshots));
}
return out;
}
@@ -513,9 +523,8 @@ public class PatrolDeleteTaskService {
item.setCreatedAt(entity.getCreatedAt());
item.setFinishedAt(task != null ? task.getFinishedAt() : item.getFinishedAt());
item.setOutputFilename(firstNonBlank(item.getOutputFilename(), entity.getResultFilename()));
item.setDownloadUrl(blank(entity.getResultFileUrl())
? item.getDownloadUrl()
: ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl()));
item.setDownloadUrl(null);
attachFileJobState(item, entity);
if (item.getCountrySections() == null) {
item.setCountrySections(new ArrayList<>());
}
@@ -525,6 +534,18 @@ public class PatrolDeleteTaskService {
return item;
}
private void attachFileJobState(PatrolDeleteResultItemVo item, FileResultEntity entity) {
TaskFileJobEntity job = taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId());
item.setFileReady(!blank(entity.getResultFileUrl()));
if (job == null) {
item.setFileStatus(Boolean.TRUE.equals(item.getFileReady()) ? "SUCCESS" : null);
return;
}
item.setFileJobId(job.getId());
item.setFileStatus(job.getStatus());
item.setFileError(job.getErrorMessage());
}
private List<PatrolDeleteTaskItemDto> dedupeItems(List<PatrolDeleteTaskItemDto> items) {
LinkedHashMap<String, PatrolDeleteTaskItemDto> map = new LinkedHashMap<>();
for (PatrolDeleteTaskItemDto item : items) {
@@ -569,6 +590,7 @@ public class PatrolDeleteTaskService {
try {
task.setRequestJson(objectMapper.writeValueAsString(requestItems));
task.setResultJson(objectMapper.writeValueAsString(snapshots));
syncSnapshotTables(task, snapshots);
fileTaskMapper.updateById(task);
} catch (Exception ex) {
throw new BusinessException("巡店删除任务快照保存失败");
@@ -585,6 +607,27 @@ public class PatrolDeleteTaskService {
return list;
}
private List<PatrolDeleteResultItemVo> buildProgressItems(FileTaskEntity task, List<FileResultEntity> rows) {
List<PatrolDeleteResultItemVo> list = new ArrayList<>();
for (FileResultEntity row : rows) {
PatrolDeleteResultItemVo item = new PatrolDeleteResultItemVo();
item.setResultId(row.getId());
item.setTaskId(row.getTaskId());
item.setShopName(row.getSourceFilename());
item.setShopId(row.getSourceFileUrl());
item.setTaskStatus(task == null ? null : task.getStatus());
item.setSuccess(row.getSuccess() == null ? null : row.getSuccess() == 1);
item.setError(row.getErrorMessage());
item.setCreatedAt(row.getCreatedAt());
item.setFinishedAt(task == null ? null : task.getFinishedAt());
item.setOutputFilename(row.getResultFilename());
item.setDownloadUrl(null);
attachFileJobState(item, row);
list.add(item);
}
return list;
}
private Map<Long, PatrolDeleteResultItemVo> indexSnapshotByResultId(List<PatrolDeleteResultItemVo> snapshots) {
Map<Long, PatrolDeleteResultItemVo> map = new LinkedHashMap<>();
if (snapshots == null) {
@@ -753,8 +796,54 @@ public class PatrolDeleteTaskService {
.filter(item -> Boolean.TRUE.equals(item.getSuccess()))
.toList();
if (!successItems.isEmpty()) {
String filename = safeFileStem("patrol-delete-" + task.getId()) + ".xlsx";
int rowCount = excelAssemblyService.countRows(successItems);
FileResultEntity firstSuccessRow = null;
for (FileResultEntity row : rows) {
if (Integer.valueOf(1).equals(row.getSuccess())) {
row.setResultFilename(filename);
row.setResultFileUrl(null);
row.setResultFileSize(0L);
row.setResultContentType(CONTENT_TYPE_XLSX);
row.setRowCount(rowCount);
fileResultMapper.updateById(row);
if (firstSuccessRow == null) {
firstSuccessRow = row;
}
}
}
if (firstSuccessRow != null) {
taskFileJobService.enqueueAssembleResult(task.getId(), MODULE_TYPE, firstSuccessRow.getId(), "task:" + task.getId());
snapshots = buildSnapshotFromDb(task, rows);
}
}
persistSnapshotJson(task, snapshots);
fileTaskMapper.updateById(task);
taskCacheService.deleteTaskCache(task.getId());
}
public void processResultFileJob(TaskFileJobEntity job) {
if (job == null || job.getTaskId() == null) {
throw new BusinessException("结果文件任务参数不完整");
}
FileTaskEntity task = fileTaskMapper.selectById(job.getTaskId());
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
throw new BusinessException("任务不存在");
}
List<FileResultEntity> rows = listTaskRows(job.getTaskId());
List<PatrolDeleteResultItemVo> snapshots = taskResultItemService.listResultSnapshots(job.getTaskId(), MODULE_TYPE, PatrolDeleteResultItemVo.class);
if (snapshots.isEmpty()) {
snapshots = buildSnapshotFromDb(task, rows);
}
List<PatrolDeleteResultItemVo> successItems = snapshots.stream()
.filter(item -> Boolean.TRUE.equals(item.getSuccess()))
.toList();
if (successItems.isEmpty()) {
throw new BusinessException("没有可生成的巡店删除结果");
}
File workRoot = FileUtil.mkdir(FileUtil.file(System.getProperty("java.io.tmpdir"), "patrol-delete-result", String.valueOf(task.getId())));
String filename = safeFileStem("巡店删除-" + task.getId()) + ".xlsx";
String filename = safeFileStem("patrol-delete-" + task.getId()) + ".xlsx";
File xlsx = FileUtil.file(workRoot, filename);
try {
excelAssemblyService.writeWorkbook(xlsx, successItems);
@@ -771,17 +860,11 @@ public class PatrolDeleteTaskService {
fileResultMapper.updateById(row);
}
}
snapshots = buildSnapshotFromDb(task, rows);
} finally {
FileUtil.del(xlsx);
}
}
persistSnapshotJson(task, snapshots);
fileTaskMapper.updateById(task);
taskCacheService.deleteTaskCache(task.getId());
}
private void markResultSuccess(FileResultEntity row) {
row.setSuccess(1);
row.setErrorMessage(null);
@@ -818,11 +901,39 @@ public class PatrolDeleteTaskService {
private void persistSnapshotJson(FileTaskEntity task, List<PatrolDeleteResultItemVo> snapshots) {
try {
task.setResultJson(objectMapper.writeValueAsString(snapshots == null ? List.of() : snapshots));
syncSnapshotTables(task, snapshots);
} catch (Exception ex) {
throw new BusinessException("巡店删除任务快照保存失败");
}
}
private void syncSnapshotTables(FileTaskEntity task, List<PatrolDeleteResultItemVo> snapshots) {
List<PatrolDeleteResultItemVo> safe = snapshots == null ? List.of() : snapshots;
taskResultItemService.replaceTaskSnapshots(task.getId(), MODULE_TYPE, safe, new TaskResultItemService.SnapshotKeyResolver() {
@Override
public Long resultId(Object snapshot) {
return ((PatrolDeleteResultItemVo) snapshot).getResultId();
}
@Override
public String scopeKey(Object snapshot) {
PatrolDeleteResultItemVo item = (PatrolDeleteResultItemVo) snapshot;
return firstNonBlank(item.getShopName(), "result:" + item.getResultId());
}
});
int successCount = 0;
int failedCount = 0;
for (PatrolDeleteResultItemVo item : safe) {
if (Boolean.TRUE.equals(item.getSuccess())) {
successCount++;
} else if (Boolean.FALSE.equals(item.getSuccess())) {
failedCount++;
}
}
taskProgressSnapshotService.save(task.getId(), MODULE_TYPE, firstNonBlank(task.getStatus(), "RUNNING"),
safe.size(), successCount, failedCount, null, task.getErrorMessage(), null);
}
private List<PatrolDeleteCountrySectionDto> copyCountrySections(List<PatrolDeleteCountrySectionDto> sections) {
List<PatrolDeleteCountrySectionDto> copy = new ArrayList<>();
if (sections == null) {

12
backend-java/src/main/java/com/nanri/aiimage/modules/permission/service/PermissionMenuSchemaInitializer.java
View File

@@ -1,16 +1,20 @@
package com.nanri.aiimage.modules.permission.service;
import jakarta.annotation.PostConstruct;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.event.EventListener;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Component;
import java.util.List;
import java.util.concurrent.CompletableFuture;
@Component
@RequiredArgsConstructor
@Slf4j
@ConditionalOnProperty(prefix = "aiimage.permission-schema-init", name = "enabled", havingValue = "true")
public class PermissionMenuSchemaInitializer {
private final JdbcTemplate jdbcTemplate;
@@ -33,8 +37,12 @@ public class PermissionMenuSchemaInitializer {
new DefaultAdminMenu("版本管理", "admin_version", "version", 80)
);
@PostConstruct
@EventListener(ApplicationReadyEvent.class)
public void initialize() {
CompletableFuture.runAsync(this::initializeInternal);
}
private void initializeInternal() {
executeQuietly("""
CREATE TABLE IF NOT EXISTS columns (
id INT AUTO_INCREMENT PRIMARY KEY,

2
backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/controller/PriceTrackController.java
View File

@@ -215,7 +215,7 @@ public class PriceTrackController {
+ "未完成/处理中：本次只传增量行数据，shop.success 不传或传 null，后端仅合并缓存继续等待。"
+ "已完成：传最后一批数据并设置 shop.success=true，后端会使用该店铺累计后的完整数据出结果。"
+ "失败：传 shop.success=false 或传非空 shop.error。"
+ "行级数据只保留表头字段：shopMallName、asin、price、recommendedPrice、minimumPrice、firstPlace、secondPlace、cartShopName、priceChangeStatus、modifyCount、status。"
+ "行级数据只保留表头字段：shopMallName、asin、price、recommendedPrice、shippingFee、minimumPrice、firstPlace、secondPlace、cartShopName、priceChangeStatus、modifyCount、status。"
)
public ApiResponse<Void> submitResult(
@Parameter(

3
backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/model/dto/PriceTrackSubmitResultRequest.java
View File

@@ -73,6 +73,9 @@ public class PriceTrackSubmitResultRequest {
@Schema(description = "推荐价", example = "18.99")
private String recommendedPrice;
@Schema(description = "运费", example = "2.50")
private String shippingFee;
@Schema(description = "最低价", example = "18.50")
private String minimumPrice;

4
backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/model/vo/PriceTrackResultItemVo.java
View File

@@ -52,6 +52,10 @@ public class PriceTrackResultItemVo {
@Schema(description = "预签名下载 URL，列表可能带回，也可通过下载接口获取")
private String downloadUrl;
private Long fileJobId;
private String fileStatus;
private String fileError;
private Boolean fileReady;
@Schema(description = "所属循环任务 ID", example = "1")
private Long loopRunId;

18
backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackExcelAssemblyService.java
View File

@@ -24,6 +24,7 @@ public class PriceTrackExcelAssemblyService {
"ASIN",
"价格",
"推荐价",
"运费",
"最低价",
"第一名",
"第二名",
@@ -56,13 +57,14 @@ public class PriceTrackExcelAssemblyService {
row.createCell(1).setCellValue(valueOf(item.getAsin()));
row.createCell(2).setCellValue(valueOf(item.getPrice()));
row.createCell(3).setCellValue(valueOf(item.getRecommendedPrice()));
row.createCell(4).setCellValue(valueOf(item.getMinimumPrice()));
row.createCell(5).setCellValue(valueOf(item.getFirstPlace()));
row.createCell(6).setCellValue(valueOf(item.getSecondPlace()));
row.createCell(7).setCellValue(valueOf(item.getCartShopName()));
row.createCell(8).setCellValue(valueOf(item.getPriceChangeStatus()));
row.createCell(9).setCellValue(valueOf(item.getModifyCount()));
row.createCell(10).setCellValue(valueOf(item.getStatus()));
row.createCell(4).setCellValue(valueOf(item.getShippingFee()));
row.createCell(5).setCellValue(valueOf(item.getMinimumPrice()));
row.createCell(6).setCellValue(valueOf(item.getFirstPlace()));
row.createCell(7).setCellValue(valueOf(item.getSecondPlace()));
row.createCell(8).setCellValue(valueOf(item.getCartShopName()));
row.createCell(9).setCellValue(valueOf(item.getPriceChangeStatus()));
row.createCell(10).setCellValue(valueOf(item.getModifyCount()));
row.createCell(11).setCellValue(valueOf(item.getStatus()));
}
applyDefaultColumnWidths(sheet);
}
@@ -112,7 +114,7 @@ public class PriceTrackExcelAssemblyService {
}
private void applyDefaultColumnWidths(Sheet sheet) {
int[] widths = {22, 18, 12, 12, 12, 20, 20, 22, 18, 12, 12};
int[] widths = {22, 18, 12, 12, 12, 12, 20, 20, 22, 18, 12, 12};
for (int i = 0; i < widths.length; i++) {
sheet.setColumnWidth(i, widths[i] * 256);
}

62
backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskCacheService.java
View File

@@ -6,6 +6,7 @@ import com.nanri.aiimage.modules.pricetrack.model.dto.PriceTrackSubmitResultRequ
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.service.TaskScopePayloadStorageService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
@@ -13,11 +14,13 @@ import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@Service
@RequiredArgsConstructor
@Slf4j
public class PriceTrackTaskCacheService {
private static final String MODULE_TYPE = "PRICE_TRACK";
@@ -32,17 +35,27 @@ public class PriceTrackTaskCacheService {
if (taskId == null || taskId <= 0) {
return;
}
try {
stringRedisTemplate.opsForValue().set(
buildTaskHeartbeatKey(taskId),
String.valueOf(Instant.now().toEpochMilli()),
Duration.ofHours(HEARTBEAT_TTL_HOURS));
} catch (Exception ex) {
log.warn("[price-track-cache] touch heartbeat degraded taskId={} msg={}", taskId, ex.getMessage());
}
}
public long getTaskHeartbeatMillis(Long taskId) {
if (taskId == null || taskId <= 0) {
return 0L;
}
String raw = stringRedisTemplate.opsForValue().get(buildTaskHeartbeatKey(taskId));
String raw;
try {
raw = stringRedisTemplate.opsForValue().get(buildTaskHeartbeatKey(taskId));
} catch (Exception ex) {
log.warn("[price-track-cache] get heartbeat degraded taskId={} msg={}", taskId, ex.getMessage());
return 0L;
}
if (raw == null || raw.isBlank()) {
return 0L;
}
@@ -53,6 +66,41 @@ public class PriceTrackTaskCacheService {
}
}
public Map<Long, Long> getTaskHeartbeatMillisBatch(List<Long> taskIds) {
Map<Long, Long> result = new LinkedHashMap<>();
if (taskIds == null || taskIds.isEmpty()) {
return result;
}
List<Long> normalized = taskIds.stream()
.filter(id -> id != null && id > 0)
.distinct()
.toList();
if (normalized.isEmpty()) {
return result;
}
List<String> keys = normalized.stream().map(this::buildTaskHeartbeatKey).toList();
List<String> values;
try {
values = stringRedisTemplate.opsForValue().multiGet(keys);
} catch (Exception ex) {
log.warn("[price-track-cache] batch get heartbeat degraded taskIds={} msg={}", normalized, ex.getMessage());
return result;
}
for (int i = 0; i < normalized.size(); i++) {
String raw = values != null && i < values.size() ? values.get(i) : null;
if (raw == null || raw.isBlank()) {
result.put(normalized.get(i), 0L);
continue;
}
try {
result.put(normalized.get(i), Long.parseLong(raw));
} catch (NumberFormatException ignored) {
result.put(normalized.get(i), 0L);
}
}
return result;
}
public PriceTrackSubmitResultRequest.ShopResult getShopMergedPayload(Long taskId, String shopKey) {
return taskScopePayloadStorageService.getScopePayload(taskId, MODULE_TYPE, shopKey, PriceTrackSubmitResultRequest.ShopResult.class);
}
@@ -82,8 +130,12 @@ public class PriceTrackTaskCacheService {
return;
}
taskEntityLocalCache.remove(taskId);
try {
stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId));
stringRedisTemplate.delete(buildTaskEntityKey(taskId));
} catch (Exception ex) {
log.warn("[price-track-cache] delete cache degraded taskId={} msg={}", taskId, ex.getMessage());
}
taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE);
}
@@ -132,7 +184,13 @@ public class PriceTrackTaskCacheService {
return result;
}
java.util.List<String> keys = missingIds.stream().map(this::buildTaskEntityKey).toList();
java.util.List<String> values = stringRedisTemplate.opsForValue().multiGet(keys);
java.util.List<String> values;
try {
values = stringRedisTemplate.opsForValue().multiGet(keys);
} catch (Exception ex) {
log.warn("[price-track-cache] batch get task cache degraded taskIds={} msg={}", missingIds, ex.getMessage());
return result;
}
for (int i = 0; i < missingIds.size(); i++) {
Long taskId = missingIds.get(i);
String val = values != null && i < values.size() ? values.get(i) : null;

70
backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskService.java
View File

@@ -26,6 +26,9 @@ import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskResultPayloadService;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
import cn.hutool.core.util.IdUtil;
import lombok.RequiredArgsConstructor;
@@ -66,6 +69,8 @@ public class PriceTrackTaskService {
private final PriceTrackTaskCacheService priceTrackTaskCacheService;
private final PriceTrackLoopRunService priceTrackLoopRunService;
private final TaskPressureProperties taskPressureProperties;
private final TaskResultPayloadService taskResultPayloadService;
private final TaskFileJobService taskFileJobService;
private FileTaskEntity loadTaskForExecution(Long taskId) {
Map<Long, FileTaskEntity> cachedTasks = priceTrackTaskCacheService.getTaskCacheBatch(List.of(taskId));
@@ -435,7 +440,7 @@ public class PriceTrackTaskService {
continue;
}
try {
assembleShopResult(fr, shopKey, merged);
enqueueResultFileAssembly(fr, shopKey, merged);
priceTrackTaskCacheService.removeShopMergedPayload(taskId, shopKey);
} catch (Exception ex) {
markResultFailed(fr, ex.getMessage() == null ? "assemble price-track result failed" : ex.getMessage());
@@ -509,7 +514,7 @@ public class PriceTrackTaskService {
continue;
}
try {
assembleShopResult(fr, shopKey, cachedPayload);
enqueueResultFileAssembly(fr, shopKey, cachedPayload);
priceTrackTaskCacheService.removeShopMergedPayload(taskId, shopKey);
changed = true;
log.warn("[price-track] stale finalize assembled taskId={} shop={} fromCompensation={} completedFlag={} rowCount={}",
@@ -618,6 +623,49 @@ public class PriceTrackTaskService {
}
}
public void processResultFileJob(TaskFileJobEntity job) {
if (job == null || job.getTaskId() == null || job.getResultId() == null || job.getScopeKey() == null) {
throw new BusinessException("结果文件任务参数不完整");
}
FileResultEntity result = fileResultMapper.selectById(job.getResultId());
if (result == null || !MODULE_TYPE.equals(result.getModuleType())) {
throw new BusinessException("结果记录不存在");
}
PriceTrackSubmitResultRequest.ShopResult payload = taskResultPayloadService.getLatest(
job.getTaskId(), MODULE_TYPE, job.getScopeKey(), PriceTrackSubmitResultRequest.ShopResult.class);
if (payload == null) {
throw new BusinessException("结果文件载荷不存在");
}
assembleShopResult(result, job.getScopeKey(), payload);
}
private void enqueueResultFileAssembly(FileResultEntity result,
String shopKey,
PriceTrackSubmitResultRequest.ShopResult payload) {
taskResultPayloadService.saveLatest(result.getTaskId(), MODULE_TYPE, shopKey, payload);
markResultFilePending(result, shopKey, payload);
taskFileJobService.enqueueAssembleResult(result.getTaskId(), MODULE_TYPE, result.getId(), shopKey);
}
private void markResultFilePending(FileResultEntity result,
String shopKey,
PriceTrackSubmitResultRequest.ShopResult payload) {
Map<String, List<PriceTrackSubmitResultRequest.AsinResult>> countries =
excelAssemblyService.normalizeCountriesMap(payload.getCountries());
String displayName = payload.getShopName() != null && !payload.getShopName().isBlank()
? payload.getShopName().trim()
: shopKey;
String stem = safeFileStem(displayName);
result.setSuccess(1);
result.setErrorMessage(null);
result.setResultFilename(stem + ".xlsx");
result.setResultFileUrl(null);
result.setResultFileSize(0L);
result.setResultContentType(CONTENT_TYPE_XLSX);
result.setRowCount(excelAssemblyService.countRows(countries));
fileResultMapper.updateById(result);
}
public Map<String, List<Map<String, String>>> parseMatchAsinRowsByCountry(List<String> asinFiles, List<String> countryCodes) {
return parseAsinRowsByCountry(asinFiles, countryCodes);
}
@@ -901,8 +949,8 @@ public class PriceTrackTaskService {
vo.setSuccess(ok);
vo.setError(entity.getErrorMessage());
vo.setOutputFilename(entity.getResultFilename());
vo.setDownloadUrl(entity.getResultFileUrl() == null || entity.getResultFileUrl().isBlank()
? null : ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl()));
vo.setDownloadUrl(null);
attachFileJobState(vo, entity);
vo.setMatched(true);
if (entity.getTaskId() != null) {
mergeQueueFieldsFromRequest(vo, entity.getTaskId());
@@ -910,6 +958,18 @@ public class PriceTrackTaskService {
return vo;
}
private void attachFileJobState(PriceTrackResultItemVo vo, FileResultEntity entity) {
TaskFileJobEntity job = taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId());
vo.setFileReady(entity.getResultFileUrl() != null && !entity.getResultFileUrl().isBlank());
if (job == null) {
vo.setFileStatus(Boolean.TRUE.equals(vo.getFileReady()) ? "SUCCESS" : null);
return;
}
vo.setFileJobId(job.getId());
vo.setFileStatus(job.getStatus());
vo.setFileError(job.getErrorMessage());
}
private PriceTrackTaskDetailVo buildTaskDetail(FileTaskEntity task) {
PriceTrackTaskDetailVo detail = new PriceTrackTaskDetailVo();
detail.setTask(toTaskItemVo(task));
@@ -1074,6 +1134,7 @@ public class PriceTrackTaskService {
merged.setAsin(firstNonBlank(incoming.getAsin(), merged.getAsin()));
merged.setPrice(firstNonBlank(incoming.getPrice(), merged.getPrice()));
merged.setRecommendedPrice(firstNonBlank(incoming.getRecommendedPrice(), merged.getRecommendedPrice()));
merged.setShippingFee(firstNonBlank(incoming.getShippingFee(), merged.getShippingFee()));
merged.setMinimumPrice(firstNonBlank(incoming.getMinimumPrice(), merged.getMinimumPrice()));
merged.setFirstPlace(firstNonBlank(incoming.getFirstPlace(), merged.getFirstPlace()));
merged.setSecondPlace(firstNonBlank(incoming.getSecondPlace(), merged.getSecondPlace()));
@@ -1097,6 +1158,7 @@ public class PriceTrackTaskService {
out.setAsin(row.getAsin());
out.setPrice(row.getPrice());
out.setRecommendedPrice(row.getRecommendedPrice());
out.setShippingFee(row.getShippingFee());
out.setMinimumPrice(row.getMinimumPrice());
out.setFirstPlace(row.getFirstPlace());
out.setSecondPlace(row.getSecondPlace());

4
backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/model/vo/ProductRiskResultItemVo.java
View File

@@ -57,6 +57,10 @@ public class ProductRiskResultItemVo {
@JsonProperty("downloadUrl")
@Schema(description = "预签名下载 URL（列表里可能带；直链下载也可用 GET /results/{resultId}/download）")
private String downloadUrl;
private Long fileJobId;
private String fileStatus;
private String fileError;
private Boolean fileReady;
@JsonProperty("scheduledAt")
@Schema(description = "定时执行时间，未设置时为空")

62
backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskTaskCacheService.java
View File

@@ -6,6 +6,7 @@ import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskShopPayloadDto
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.service.TaskScopePayloadStorageService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
@@ -13,11 +14,13 @@ import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@Service
@RequiredArgsConstructor
@Slf4j
public class ProductRiskTaskCacheService {
private static final String MODULE_TYPE = "PRODUCT_RISK_RESOLVE";
@@ -60,7 +63,13 @@ public class ProductRiskTaskCacheService {
if (taskId == null || taskId <= 0) {
return 0L;
}
String raw = stringRedisTemplate.opsForValue().get(buildTaskHeartbeatKey(taskId));
String raw;
try {
raw = stringRedisTemplate.opsForValue().get(buildTaskHeartbeatKey(taskId));
} catch (Exception ex) {
log.warn("[product-risk-cache] get heartbeat degraded taskId={} msg={}", taskId, ex.getMessage());
return 0L;
}
if (raw == null || raw.isBlank()) {
return 0L;
}
@@ -71,13 +80,52 @@ public class ProductRiskTaskCacheService {
}
}
public Map<Long, Long> getTaskHeartbeatMillisBatch(List<Long> taskIds) {
Map<Long, Long> result = new LinkedHashMap<>();
if (taskIds == null || taskIds.isEmpty()) {
return result;
}
List<Long> normalized = taskIds.stream()
.filter(id -> id != null && id > 0)
.distinct()
.toList();
if (normalized.isEmpty()) {
return result;
}
List<String> keys = normalized.stream().map(this::buildTaskHeartbeatKey).toList();
List<String> values;
try {
values = stringRedisTemplate.opsForValue().multiGet(keys);
} catch (Exception ex) {
log.warn("[product-risk-cache] batch get heartbeat degraded taskIds={} msg={}", normalized, ex.getMessage());
return result;
}
for (int i = 0; i < normalized.size(); i++) {
String raw = values != null && i < values.size() ? values.get(i) : null;
if (raw == null || raw.isBlank()) {
result.put(normalized.get(i), 0L);
continue;
}
try {
result.put(normalized.get(i), Long.parseLong(raw));
} catch (NumberFormatException ignored) {
result.put(normalized.get(i), 0L);
}
}
return result;
}
public void deleteTaskCache(Long taskId) {
if (taskId == null || taskId <= 0) {
return;
}
taskEntityLocalCache.remove(taskId);
try {
stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId));
stringRedisTemplate.delete(buildTaskEntityKey(taskId));
} catch (Exception ex) {
log.warn("[product-risk-cache] delete cache degraded taskId={} msg={}", taskId, ex.getMessage());
}
taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE);
}
@@ -126,7 +174,13 @@ public class ProductRiskTaskCacheService {
return result;
}
java.util.List<String> keys = missingIds.stream().map(this::buildTaskEntityKey).toList();
java.util.List<String> values = stringRedisTemplate.opsForValue().multiGet(keys);
java.util.List<String> values;
try {
values = stringRedisTemplate.opsForValue().multiGet(keys);
} catch (Exception ex) {
log.warn("[product-risk-cache] batch get task cache degraded taskIds={} msg={}", missingIds, ex.getMessage());
return result;
}
for (int i = 0; i < missingIds.size(); i++) {
Long taskId = missingIds.get(i);
String val = values != null && i < values.size() ? values.get(i) : null;
@@ -144,10 +198,14 @@ public class ProductRiskTaskCacheService {
}
public void touchTaskHeartbeat(Long taskId) {
try {
stringRedisTemplate.opsForValue().set(
buildTaskHeartbeatKey(taskId),
String.valueOf(Instant.now().toEpochMilli()),
Duration.ofHours(PAYLOAD_TTL_HOURS));
} catch (Exception ex) {
log.warn("[product-risk-cache] touch heartbeat degraded taskId={} msg={}", taskId, ex.getMessage());
}
}
private String buildTaskHeartbeatKey(Long taskId) {

75
backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskTaskService.java
View File

@@ -24,6 +24,11 @@ import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import com.nanri.aiimage.modules.task.service.TaskResultItemService;
import com.nanri.aiimage.modules.task.service.TaskResultPayloadService;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.util.IdUtil;
@@ -59,6 +64,10 @@ public class ProductRiskTaskService {
private final ObjectMapper objectMapper;
private final ProductRiskTaskCacheService productRiskTaskCacheService;
private final TaskPressureProperties taskPressureProperties;
private final TaskResultPayloadService taskResultPayloadService;
private final TaskFileJobService taskFileJobService;
private final TaskResultItemService taskResultItemService;
private final TaskProgressSnapshotService taskProgressSnapshotService;
private FileTaskEntity loadTaskForExecution(Long taskId) {
Map<Long, FileTaskEntity> cachedTasks = productRiskTaskCacheService.getTaskCacheBatch(List.of(taskId));
@@ -553,7 +562,7 @@ public class ProductRiskTaskService {
}
try {
assembleShopResult(fr, shopKey, mergedPayload, workRoot);
enqueueResultFileAssembly(fr, shopKey, mergedPayload);
assembledCount++;
log.info("[product-risk] shop assembled taskId={} shop={} rowCount={} file={}",
taskId, shopKey, fr.getRowCount(), fr.getResultFilename());
@@ -698,7 +707,7 @@ public class ProductRiskTaskService {
}
try {
assembleShopResult(fr, shopKey, cachedPayload, workRoot);
enqueueResultFileAssembly(fr, shopKey, cachedPayload);
productRiskTaskCacheService.removeShopMergedPayload(taskId, shopKey);
changed = true;
log.warn("[product-risk] stale finalize assembled taskId={} shop={} fromCompensation={} completedFlag={} rowCount={}",
@@ -778,6 +787,49 @@ public class ProductRiskTaskService {
}
}
public void processResultFileJob(TaskFileJobEntity job) {
if (job == null || job.getTaskId() == null || job.getResultId() == null || job.getScopeKey() == null) {
throw new BusinessException("结果文件任务参数不完整");
}
FileResultEntity result = fileResultMapper.selectById(job.getResultId());
if (result == null || !MODULE_TYPE.equals(result.getModuleType())) {
throw new BusinessException("结果记录不存在");
}
ProductRiskShopPayloadDto payload = taskResultPayloadService.getLatest(
job.getTaskId(), MODULE_TYPE, job.getScopeKey(), ProductRiskShopPayloadDto.class);
if (payload == null) {
payload = taskResultItemService.getResultSnapshot(job.getTaskId(), MODULE_TYPE, job.getResultId(), ProductRiskShopPayloadDto.class);
}
if (payload == null) {
throw new BusinessException("结果文件载荷不存在");
}
File workRoot = FileUtil.mkdir(FileUtil.file(System.getProperty("java.io.tmpdir"), "product-risk-result", String.valueOf(job.getTaskId())));
assembleShopResult(result, job.getScopeKey(), payload, workRoot);
}
private void enqueueResultFileAssembly(FileResultEntity result, String shopKey, ProductRiskShopPayloadDto payload) {
taskResultPayloadService.saveLatest(result.getTaskId(), MODULE_TYPE, shopKey, payload);
taskResultItemService.replaceResultSnapshot(result.getTaskId(), MODULE_TYPE, result.getId(), shopKey, payload);
markResultFilePending(result, shopKey, payload);
taskFileJobService.enqueueAssembleResult(result.getTaskId(), MODULE_TYPE, result.getId(), shopKey);
}
private void markResultFilePending(FileResultEntity result, String shopKey, ProductRiskShopPayloadDto payload) {
Map<String, List<ProductRiskRowDto>> countries = excelAssemblyService.normalizeCountriesMap(payload.getCountries());
String displayName = payload.getShopName() != null && !payload.getShopName().isBlank()
? payload.getShopName().trim()
: shopKey;
String stem = safeFileStem(displayName);
result.setResultFilename(stem + ".zip");
result.setResultFileUrl(null);
result.setResultFileSize(0L);
result.setResultContentType(CONTENT_TYPE_ZIP);
result.setRowCount(excelAssemblyService.countRows(countries));
result.setSuccess(1);
result.setErrorMessage(null);
fileResultMapper.updateById(result);
}
private void updateTaskStatusFromLatestRows(FileTaskEntity task,
List<FileResultEntity> latest,
List<String> batchErrors) {
@@ -826,6 +878,8 @@ public class ProductRiskTaskService {
log.warn("[product-risk] compact result json failed: {}", ex.getMessage());
}
fileTaskMapper.updateById(task);
taskProgressSnapshotService.save(task.getId(), MODULE_TYPE, task.getStatus(),
latest.size(), ok, fail, null, task.getErrorMessage(), null);
log.warn("[product-risk] status updated from latest rows taskId={} oldStatus={} newStatus={} ok={} fail={} allDone={} batchErrors={} errorMessage={}",
task.getId(), oldStatus, task.getStatus(), ok, fail, allDone, batchErrors, task.getErrorMessage());
cleanupTaskCacheIfTerminal(task.getId(), task.getStatus());
@@ -910,9 +964,8 @@ public class ProductRiskTaskService {
vo.setSuccess(ok);
vo.setError(entity.getErrorMessage());
vo.setOutputFilename(entity.getResultFilename());
vo.setDownloadUrl(entity.getResultFileUrl() == null || entity.getResultFileUrl().isBlank()
? null
: ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl()));
vo.setDownloadUrl(null);
attachFileJobState(vo, entity);
vo.setMatched(true);
if (entity.getTaskId() != null) {
mergeQueueFieldsFromRequest(vo, entity.getTaskId());
@@ -920,6 +973,18 @@ public class ProductRiskTaskService {
return vo;
}
private void attachFileJobState(ProductRiskResultItemVo vo, FileResultEntity entity) {
TaskFileJobEntity job = taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId());
vo.setFileReady(entity.getResultFileUrl() != null && !entity.getResultFileUrl().isBlank());
if (job == null) {
vo.setFileStatus(Boolean.TRUE.equals(vo.getFileReady()) ? "SUCCESS" : null);
return;
}
vo.setFileJobId(job.getId());
vo.setFileStatus(job.getStatus());
vo.setFileError(job.getErrorMessage());
}
private void mergeQueueFieldsFromRequest(ProductRiskResultItemVo vo, Long taskId) {
FileTaskEntity t = loadTaskForExecution(taskId);
if (t == null || t.getRequestJson() == null || t.getRequestJson().isBlank()) {

4
backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/model/vo/QueryAsinResultItemVo.java
View File

@@ -43,6 +43,10 @@ public class QueryAsinResultItemVo {
private LocalDateTime finishedAt;
private String outputFilename;
private String downloadUrl;
private Long fileJobId;
private String fileStatus;
private String fileError;
private Boolean fileReady;
@JsonProperty("queryAsins")
private List<QueryAsinCountryAsinsDto> queryAsins = new ArrayList<>();

26
backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/service/QueryAsinTaskCacheService.java
View File

@@ -6,6 +6,7 @@ import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinShopPayloadDto;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.service.TaskScopePayloadStorageService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
@@ -18,6 +19,7 @@ import java.util.concurrent.ConcurrentHashMap;
@Service
@RequiredArgsConstructor
@Slf4j
public class QueryAsinTaskCacheService {
private static final String MODULE_TYPE = "QUERY_ASIN";
@@ -60,7 +62,13 @@ public class QueryAsinTaskCacheService {
if (taskId == null || taskId <= 0) {
return 0L;
}
String raw = stringRedisTemplate.opsForValue().get(buildTaskHeartbeatKey(taskId));
String raw;
try {
raw = stringRedisTemplate.opsForValue().get(buildTaskHeartbeatKey(taskId));
} catch (Exception ex) {
log.warn("[query-asin-cache] get heartbeat degraded taskId={} msg={}", taskId, ex.getMessage());
return 0L;
}
if (raw == null || raw.isBlank()) {
return 0L;
}
@@ -72,10 +80,14 @@ public class QueryAsinTaskCacheService {
}
public void touchTaskHeartbeat(Long taskId) {
try {
stringRedisTemplate.opsForValue().set(
buildTaskHeartbeatKey(taskId),
String.valueOf(Instant.now().toEpochMilli()),
Duration.ofHours(PAYLOAD_TTL_HOURS));
} catch (Exception ex) {
log.warn("[query-asin-cache] touch heartbeat degraded taskId={} msg={}", taskId, ex.getMessage());
}
}
public void deleteTaskCache(Long taskId) {
@@ -83,8 +95,12 @@ public class QueryAsinTaskCacheService {
return;
}
taskEntityLocalCache.remove(taskId);
try {
stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId));
stringRedisTemplate.delete(buildTaskEntityKey(taskId));
} catch (Exception ex) {
log.warn("[query-asin-cache] delete cache degraded taskId={} msg={}", taskId, ex.getMessage());
}
taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE);
}
@@ -133,7 +149,13 @@ public class QueryAsinTaskCacheService {
return result;
}
java.util.List<String> keys = missingIds.stream().map(this::buildTaskEntityKey).toList();
java.util.List<String> values = stringRedisTemplate.opsForValue().multiGet(keys);
java.util.List<String> values;
try {
values = stringRedisTemplate.opsForValue().multiGet(keys);
} catch (Exception ex) {
log.warn("[query-asin-cache] batch get task cache degraded taskIds={} msg={}", missingIds, ex.getMessage());
return result;
}
for (int i = 0; i < missingIds.size(); i++) {
Long taskId = missingIds.get(i);
String val = values != null && i < values.size() ? values.get(i) : null;

137
backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/service/QueryAsinTaskService.java
View File

@@ -24,6 +24,10 @@ import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import com.nanri.aiimage.modules.task.service.TaskResultItemService;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@@ -56,6 +60,9 @@ public class QueryAsinTaskService {
private final ZiniaoShopSwitchService ziniaoShopSwitchService;
private final ObjectMapper objectMapper;
private final TaskPressureProperties taskPressureProperties;
private final TaskFileJobService taskFileJobService;
private final TaskResultItemService taskResultItemService;
private final TaskProgressSnapshotService taskProgressSnapshotService;
private FileTaskEntity loadTaskForExecution(Long taskId) {
Map<Long, FileTaskEntity> cachedTasks = taskCacheService.getTaskCacheBatch(List.of(taskId));
@@ -185,8 +192,7 @@ public class QueryAsinTaskService {
batch.getMissingTaskIds().add(taskId);
continue;
}
List<QueryAsinResultItemVo> snapshots = buildSnapshotFromDb(task, taskRows);
batch.getItems().addAll(snapshots);
batch.getItems().addAll(buildProgressItems(task, taskRows));
}
return batch;
}
@@ -502,7 +508,11 @@ public class QueryAsinTaskService {
private Map<Long, Map<Long, QueryAsinResultItemVo>> buildSnapshotMap(Map<Long, FileTaskEntity> taskMap) {
Map<Long, Map<Long, QueryAsinResultItemVo>> out = new LinkedHashMap<>();
for (Map.Entry<Long, FileTaskEntity> entry : taskMap.entrySet()) {
out.put(entry.getKey(), indexSnapshotByResultId(parseTaskSnapshots(entry.getValue().getResultJson())));
List<QueryAsinResultItemVo> snapshots = taskResultItemService.listResultSnapshots(entry.getKey(), MODULE_TYPE, QueryAsinResultItemVo.class);
if (snapshots.isEmpty()) {
snapshots = parseTaskSnapshots(entry.getValue().getResultJson());
}
out.put(entry.getKey(), indexSnapshotByResultId(snapshots));
}
return out;
}
@@ -519,9 +529,8 @@ public class QueryAsinTaskService {
item.setCreatedAt(entity.getCreatedAt());
item.setFinishedAt(task != null ? task.getFinishedAt() : item.getFinishedAt());
item.setOutputFilename(firstNonBlank(item.getOutputFilename(), entity.getResultFilename()));
item.setDownloadUrl(blank(entity.getResultFileUrl())
? item.getDownloadUrl()
: ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl()));
item.setDownloadUrl(null);
attachFileJobState(item, entity);
if (item.getCountryResults() == null) {
item.setCountryResults(new ArrayList<>());
}
@@ -531,6 +540,18 @@ public class QueryAsinTaskService {
return item;
}
private void attachFileJobState(QueryAsinResultItemVo item, FileResultEntity entity) {
TaskFileJobEntity job = taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId());
item.setFileReady(!blank(entity.getResultFileUrl()));
if (job == null) {
item.setFileStatus(Boolean.TRUE.equals(item.getFileReady()) ? "SUCCESS" : null);
return;
}
item.setFileJobId(job.getId());
item.setFileStatus(job.getStatus());
item.setFileError(job.getErrorMessage());
}
private List<QueryAsinTaskItemDto> dedupeItems(List<QueryAsinTaskItemDto> items) {
LinkedHashMap<String, QueryAsinTaskItemDto> map = new LinkedHashMap<>();
for (QueryAsinTaskItemDto item : items) {
@@ -575,6 +596,7 @@ public class QueryAsinTaskService {
try {
task.setRequestJson(objectMapper.writeValueAsString(requestItems));
task.setResultJson(objectMapper.writeValueAsString(snapshots));
syncSnapshotTables(task, snapshots);
fileTaskMapper.updateById(task);
} catch (Exception ex) {
throw new BusinessException("查询ASIN任务快照保存失败");
@@ -591,6 +613,27 @@ public class QueryAsinTaskService {
return list;
}
private List<QueryAsinResultItemVo> buildProgressItems(FileTaskEntity task, List<FileResultEntity> rows) {
List<QueryAsinResultItemVo> list = new ArrayList<>();
for (FileResultEntity row : rows) {
QueryAsinResultItemVo item = new QueryAsinResultItemVo();
item.setResultId(row.getId());
item.setTaskId(row.getTaskId());
item.setShopName(row.getSourceFilename());
item.setShopId(row.getSourceFileUrl());
item.setTaskStatus(task == null ? null : task.getStatus());
item.setSuccess(row.getSuccess() == null ? null : row.getSuccess() == 1);
item.setError(row.getErrorMessage());
item.setCreatedAt(row.getCreatedAt());
item.setFinishedAt(task == null ? null : task.getFinishedAt());
item.setOutputFilename(row.getResultFilename());
item.setDownloadUrl(null);
attachFileJobState(item, row);
list.add(item);
}
return list;
}
private Map<Long, QueryAsinResultItemVo> indexSnapshotByResultId(List<QueryAsinResultItemVo> snapshots) {
Map<Long, QueryAsinResultItemVo> map = new LinkedHashMap<>();
if (snapshots == null) {
@@ -773,8 +816,54 @@ public class QueryAsinTaskService {
.filter(item -> Boolean.TRUE.equals(item.getSuccess()))
.toList();
if (!successItems.isEmpty()) {
String filename = safeFileStem("query-asin-" + task.getId()) + ".xlsx";
int rowCount = excelAssemblyService.countRows(successItems);
FileResultEntity firstSuccessRow = null;
for (FileResultEntity row : rows) {
if (Integer.valueOf(1).equals(row.getSuccess())) {
row.setResultFilename(filename);
row.setResultFileUrl(null);
row.setResultFileSize(0L);
row.setResultContentType(CONTENT_TYPE_XLSX);
row.setRowCount(rowCount);
fileResultMapper.updateById(row);
if (firstSuccessRow == null) {
firstSuccessRow = row;
}
}
}
if (firstSuccessRow != null) {
taskFileJobService.enqueueAssembleResult(task.getId(), MODULE_TYPE, firstSuccessRow.getId(), "task:" + task.getId());
snapshots = buildSnapshotFromDb(task, rows);
}
}
persistSnapshotJson(task, snapshots);
fileTaskMapper.updateById(task);
taskCacheService.deleteTaskCache(task.getId());
}
public void processResultFileJob(TaskFileJobEntity job) {
if (job == null || job.getTaskId() == null) {
throw new BusinessException("结果文件任务参数不完整");
}
FileTaskEntity task = fileTaskMapper.selectById(job.getTaskId());
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
throw new BusinessException("任务不存在");
}
List<FileResultEntity> rows = listTaskRows(job.getTaskId());
List<QueryAsinResultItemVo> snapshots = taskResultItemService.listResultSnapshots(job.getTaskId(), MODULE_TYPE, QueryAsinResultItemVo.class);
if (snapshots.isEmpty()) {
snapshots = buildSnapshotFromDb(task, rows);
}
List<QueryAsinResultItemVo> successItems = snapshots.stream()
.filter(item -> Boolean.TRUE.equals(item.getSuccess()))
.toList();
if (successItems.isEmpty()) {
throw new BusinessException("没有可生成的查询ASIN结果");
}
File workRoot = FileUtil.mkdir(FileUtil.file(System.getProperty("java.io.tmpdir"), "query-asin-result", String.valueOf(task.getId())));
String filename = safeFileStem("查询ASIN-" + task.getId()) + ".xlsx";
String filename = safeFileStem("query-asin-" + task.getId()) + ".xlsx";
File xlsx = FileUtil.file(workRoot, filename);
try {
excelAssemblyService.writeWorkbook(xlsx, successItems);
@@ -791,17 +880,11 @@ public class QueryAsinTaskService {
fileResultMapper.updateById(row);
}
}
snapshots = buildSnapshotFromDb(task, rows);
} finally {
FileUtil.del(xlsx);
}
}
persistSnapshotJson(task, snapshots);
fileTaskMapper.updateById(task);
taskCacheService.deleteTaskCache(task.getId());
}
private void markResultSuccess(FileResultEntity row) {
row.setSuccess(1);
row.setErrorMessage(null);
@@ -838,11 +921,39 @@ public class QueryAsinTaskService {
private void persistSnapshotJson(FileTaskEntity task, List<QueryAsinResultItemVo> snapshots) {
try {
task.setResultJson(objectMapper.writeValueAsString(snapshots == null ? List.of() : snapshots));
syncSnapshotTables(task, snapshots);
} catch (Exception ex) {
throw new BusinessException("查询ASIN任务快照保存失败");
}
}
private void syncSnapshotTables(FileTaskEntity task, List<QueryAsinResultItemVo> snapshots) {
List<QueryAsinResultItemVo> safe = snapshots == null ? List.of() : snapshots;
taskResultItemService.replaceTaskSnapshots(task.getId(), MODULE_TYPE, safe, new TaskResultItemService.SnapshotKeyResolver() {
@Override
public Long resultId(Object snapshot) {
return ((QueryAsinResultItemVo) snapshot).getResultId();
}
@Override
public String scopeKey(Object snapshot) {
QueryAsinResultItemVo item = (QueryAsinResultItemVo) snapshot;
return firstNonBlank(item.getShopName(), "result:" + item.getResultId());
}
});
int successCount = 0;
int failedCount = 0;
for (QueryAsinResultItemVo item : safe) {
if (Boolean.TRUE.equals(item.getSuccess())) {
successCount++;
} else if (Boolean.FALSE.equals(item.getSuccess())) {
failedCount++;
}
}
taskProgressSnapshotService.save(task.getId(), MODULE_TYPE, firstNonBlank(task.getStatus(), "RUNNING"),
safe.size(), successCount, failedCount, null, task.getErrorMessage(), null);
}
private List<QueryAsinCountryResultDto> copyCountryResults(List<QueryAsinCountryResultDto> results) {
List<QueryAsinCountryResultDto> copy = new ArrayList<>();
if (results == null) {

52
backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchTaskCacheService.java
View File

@@ -18,6 +18,7 @@ import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@@ -38,17 +39,27 @@ public class ShopMatchTaskCacheService {
if (taskId == null || taskId <= 0) {
return;
}
try {
stringRedisTemplate.opsForValue().set(
buildTaskHeartbeatKey(taskId),
String.valueOf(Instant.now().toEpochMilli()),
Duration.ofHours(PAYLOAD_TTL_HOURS));
} catch (Exception ex) {
log.warn("[shop-match-cache] touch heartbeat degraded taskId={} msg={}", taskId, ex.getMessage());
}
}
public long getTaskHeartbeatMillis(Long taskId) {
if (taskId == null || taskId <= 0) {
return 0L;
}
String raw = stringRedisTemplate.opsForValue().get(buildTaskHeartbeatKey(taskId));
String raw;
try {
raw = stringRedisTemplate.opsForValue().get(buildTaskHeartbeatKey(taskId));
} catch (Exception ex) {
log.warn("[shop-match-cache] get heartbeat degraded taskId={} msg={}", taskId, ex.getMessage());
return 0L;
}
if (raw == null || raw.isBlank()) {
return 0L;
}
@@ -59,6 +70,41 @@ public class ShopMatchTaskCacheService {
}
}
public Map<Long, Long> getTaskHeartbeatMillisBatch(List<Long> taskIds) {
Map<Long, Long> result = new LinkedHashMap<>();
if (taskIds == null || taskIds.isEmpty()) {
return result;
}
List<Long> normalized = taskIds.stream()
.filter(id -> id != null && id > 0)
.distinct()
.toList();
if (normalized.isEmpty()) {
return result;
}
List<String> keys = normalized.stream().map(this::buildTaskHeartbeatKey).toList();
List<String> values;
try {
values = stringRedisTemplate.opsForValue().multiGet(keys);
} catch (Exception ex) {
log.warn("[shop-match-cache] batch get heartbeat degraded taskIds={} msg={}", normalized, ex.getMessage());
return result;
}
for (int i = 0; i < normalized.size(); i++) {
String raw = values != null && i < values.size() ? values.get(i) : null;
if (raw == null || raw.isBlank()) {
result.put(normalized.get(i), 0L);
continue;
}
try {
result.put(normalized.get(i), Long.parseLong(raw));
} catch (NumberFormatException ignored) {
result.put(normalized.get(i), 0L);
}
}
return result;
}
public ShopMatchShopPayloadDto getShopMergedPayload(Long taskId, String shopKey) {
return taskScopePayloadStorageService.getScopePayload(taskId, MODULE_TYPE, shopKey, ShopMatchShopPayloadDto.class);
}
@@ -88,7 +134,11 @@ public class ShopMatchTaskCacheService {
return;
}
taskEntityLocalCache.remove(taskId);
try {
stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId));
} catch (Exception ex) {
log.warn("[shop-match-cache] delete heartbeat degraded taskId={} msg={}", taskId, ex.getMessage());
}
taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE);
try {
Path taskDir = buildTaskDir(taskId);

82
backend-java/src/main/java/com/nanri/aiimage/modules/shopmatch/service/ShopMatchTaskService.java
View File

@@ -28,6 +28,9 @@ import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskResultPayloadService;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@@ -62,6 +65,8 @@ public class ShopMatchTaskService {
private final ObjectMapper objectMapper;
private final ShopMatchTaskCacheService shopMatchTaskCacheService;
private final TaskPressureProperties taskPressureProperties;
private final TaskResultPayloadService taskResultPayloadService;
private final TaskFileJobService taskFileJobService;
private FileTaskEntity loadTaskForExecution(Long taskId) {
Map<Long, FileTaskEntity> cachedTasks = shopMatchTaskCacheService.getTaskCacheBatch(List.of(taskId));
@@ -409,6 +414,12 @@ public class ShopMatchTaskService {
if (state.getScheduleTimes() == null || stageIndex >= state.getScheduleTimes().size()) {
throw new BusinessException("轮次配置缺失");
}
FileTaskEntity runningTask = findOtherRunningTask(userId, taskId);
if (runningTask != null) {
log.warn("[shop-match] activate serialized taskId={} stageIndex={} blockedByTaskId={} blockedUpdatedAt={}",
taskId, stageIndex, runningTask.getId(), runningTask.getUpdatedAt());
throw new BusinessException("当前已有定时匹配任务执行中，请等待上一任务执行结束后再重试");
}
state.setActiveStageIndex(stageIndex);
task.setStatus("RUNNING");
task.setUpdatedAt(now);
@@ -511,7 +522,7 @@ public class ShopMatchTaskService {
continue;
}
try {
assembleShopResult(result, shopKey, merged, workRoot);
enqueueResultFileAssembly(result, shopKey, merged);
shopMatchTaskCacheService.removeShopMergedPayload(taskId, shopKey);
} catch (Exception ex) {
markResultFailed(result, ex.getMessage() == null ? "结果组装失败" : ex.getMessage());
@@ -603,7 +614,7 @@ public class ShopMatchTaskService {
}
try {
assembleShopResult(result, shopKey, cachedPayload, workRoot);
enqueueResultFileAssembly(result, shopKey, cachedPayload);
shopMatchTaskCacheService.removeShopMergedPayload(taskId, shopKey);
changed = true;
log.warn("[shop-match] stale finalize assembled taskId={} shop={} fromCompensation={} completedFlag={} rowCount={}",
@@ -651,6 +662,43 @@ public class ShopMatchTaskService {
}
}
public void processResultFileJob(TaskFileJobEntity job) {
if (job == null || job.getTaskId() == null || job.getResultId() == null || job.getScopeKey() == null) {
throw new BusinessException("结果文件任务参数不完整");
}
FileResultEntity result = fileResultMapper.selectById(job.getResultId());
if (result == null || !MODULE_TYPE.equals(result.getModuleType())) {
throw new BusinessException("结果记录不存在");
}
ShopMatchShopPayloadDto payload = taskResultPayloadService.getLatest(
job.getTaskId(), MODULE_TYPE, job.getScopeKey(), ShopMatchShopPayloadDto.class);
if (payload == null) {
throw new BusinessException("结果文件载荷不存在");
}
File workRoot = FileUtil.mkdir(FileUtil.file(System.getProperty("java.io.tmpdir"), "shop-match-result", String.valueOf(job.getTaskId())));
assembleShopResult(result, job.getScopeKey(), payload, workRoot);
}
private void enqueueResultFileAssembly(FileResultEntity result, String shopKey, ShopMatchShopPayloadDto payload) {
taskResultPayloadService.saveLatest(result.getTaskId(), MODULE_TYPE, shopKey, payload);
markResultFilePending(result, shopKey, payload);
taskFileJobService.enqueueAssembleResult(result.getTaskId(), MODULE_TYPE, result.getId(), shopKey);
}
private void markResultFilePending(FileResultEntity result, String shopKey, ShopMatchShopPayloadDto payload) {
Map<String, List<ShopMatchRowDto>> countries = excelAssemblyService.normalizeCountriesMap(payload.getCountries());
String displayName = payload.getShopName() != null && !payload.getShopName().isBlank() ? payload.getShopName().trim() : shopKey;
String stem = safeFileStem(displayName);
result.setResultFilename(stem + ".xlsx");
result.setResultFileUrl(null);
result.setResultFileSize(0L);
result.setResultContentType(CONTENT_TYPE_XLSX);
result.setRowCount(excelAssemblyService.countRows(countries));
result.setSuccess(1);
result.setErrorMessage(null);
fileResultMapper.updateById(result);
}
private void markResultFailed(FileResultEntity result, String message) {
result.setSuccess(0);
result.setErrorMessage(message);
@@ -816,9 +864,8 @@ public class ShopMatchTaskService {
vo.setSuccess(success);
vo.setError(entity.getErrorMessage());
vo.setOutputFilename(entity.getResultFilename());
vo.setDownloadUrl(entity.getResultFileUrl() == null || entity.getResultFileUrl().isBlank()
? null
: ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl()));
vo.setDownloadUrl(null);
attachFileJobState(vo, entity);
vo.setMatched(true);
if (entity.getTaskId() != null) {
mergeQueueFieldsFromRequest(vo, entity.getTaskId());
@@ -826,6 +873,18 @@ public class ShopMatchTaskService {
return vo;
}
private void attachFileJobState(ProductRiskResultItemVo vo, FileResultEntity entity) {
TaskFileJobEntity job = taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId());
vo.setFileReady(entity.getResultFileUrl() != null && !entity.getResultFileUrl().isBlank());
if (job == null) {
vo.setFileStatus(Boolean.TRUE.equals(vo.getFileReady()) ? "SUCCESS" : null);
return;
}
vo.setFileJobId(job.getId());
vo.setFileStatus(job.getStatus());
vo.setFileError(job.getErrorMessage());
}
private List<ShopMatchTaskStageVo> buildStageVos(FileTaskEntity task, ShopMatchCreateTaskRequest request) {
List<ShopMatchTaskStageVo> stages = new ArrayList<>();
List<LocalDateTime> scheduleTimes = request.getScheduleTimes() == null ? List.of() : request.getScheduleTimes();
@@ -942,6 +1001,19 @@ public class ShopMatchTaskService {
shopMatchTaskCacheService.saveTaskCache(task);
}
private FileTaskEntity findOtherRunningTask(Long userId, Long taskId) {
if (userId == null || userId <= 0) {
return null;
}
return fileTaskMapper.selectOne(new LambdaQueryWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
.eq(FileTaskEntity::getUserId, userId)
.eq(FileTaskEntity::getStatus, "RUNNING")
.ne(taskId != null, FileTaskEntity::getId, taskId)
.orderByAsc(FileTaskEntity::getUpdatedAt)
.last("limit 1"));
}
private static String fmt(LocalDateTime time) {
return time == null ? null : time.toString();
}

4
backend-java/src/main/java/com/nanri/aiimage/modules/split/service/SplitRunService.java
View File

@@ -173,9 +173,7 @@ public class SplitRunService {
vo.setResultId(entity.getId());
vo.setSourceFilename(entity.getSourceFilename());
vo.setOutputFilename(entity.getResultFilename());
vo.setDownloadUrl(entity.getSuccess() != null && entity.getSuccess() == 1
? ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl())
: null);
vo.setDownloadUrl(null);
vo.setRowCount(entity.getRowCount());
vo.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1);
vo.setError(entity.getErrorMessage());

52
backend-java/src/main/java/com/nanri/aiimage/modules/task/controller/TaskFileJobController.java Normal file
View File

@@ -0,0 +1,52 @@
package com.nanri.aiimage.modules.task.controller;
import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import lombok.RequiredArgsConstructor;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import java.util.List;
import java.util.Map;
@RestController
@RequiredArgsConstructor
@RequestMapping("/api/task-file-jobs")
@Tag(name = "任务结果文件 Job")
public class TaskFileJobController {
private final TaskFileJobService taskFileJobService;
@GetMapping
@Operation(summary = "查询结果文件生成 Job")
public ApiResponse<List<TaskFileJobEntity>> list(
@RequestParam(value = "status", required = false) String status,
@RequestParam(value = "module_type", required = false) String moduleType,
@RequestParam(value = "task_id", required = false) Long taskId,
@RequestParam(value = "limit", defaultValue = "100") int limit) {
return ApiResponse.success(taskFileJobService.listJobs(status, moduleType, taskId, limit));
}
@PostMapping("/{jobId}/retry")
@Operation(summary = "手动重试结果文件生成 Job")
public ApiResponse<Map<String, Object>> retry(@PathVariable Long jobId) {
boolean updated = taskFileJobService.retry(jobId);
return ApiResponse.success(Map.of("jobId", jobId, "updated", updated));
}
@PostMapping("/stuck/reset")
@Operation(summary = "手动扫描并复位卡住的 RUNNING Job")
public ApiResponse<Map<String, Object>> resetStuck(
@RequestParam(value = "timeout_minutes", defaultValue = "30") int timeoutMinutes,
@RequestParam(value = "limit", defaultValue = "100") int limit) {
int reset = taskFileJobService.resetStuckRunningJobs(timeoutMinutes, limit);
return ApiResponse.success(Map.of("reset", reset));
}
}

9
backend-java/src/main/java/com/nanri/aiimage/modules/task/mapper/TaskFileJobMapper.java Normal file
View File

@@ -0,0 +1,9 @@
package com.nanri.aiimage.modules.task.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface TaskFileJobMapper extends BaseMapper<TaskFileJobEntity> {
}

9
backend-java/src/main/java/com/nanri/aiimage/modules/task/mapper/TaskProgressSnapshotMapper.java Normal file
View File

@@ -0,0 +1,9 @@
package com.nanri.aiimage.modules.task.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskProgressSnapshotEntity;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface TaskProgressSnapshotMapper extends BaseMapper<TaskProgressSnapshotEntity> {
}

9
backend-java/src/main/java/com/nanri/aiimage/modules/task/mapper/TaskResultItemMapper.java Normal file
View File

@@ -0,0 +1,9 @@
package com.nanri.aiimage.modules.task.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskResultItemEntity;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface TaskResultItemMapper extends BaseMapper<TaskResultItemEntity> {
}

9
backend-java/src/main/java/com/nanri/aiimage/modules/task/mapper/TaskResultPayloadMapper.java Normal file
View File

@@ -0,0 +1,9 @@
package com.nanri.aiimage.modules.task.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskResultPayloadEntity;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface TaskResultPayloadMapper extends BaseMapper<TaskResultPayloadEntity> {
}

14
backend-java/src/main/java/com/nanri/aiimage/modules/task/model/dto/TaskFileJobDispatchEvent.java Normal file
View File

@@ -0,0 +1,14 @@
package com.nanri.aiimage.modules.task.model.dto;
import java.time.LocalDateTime;
public record TaskFileJobDispatchEvent(
Long jobId,
Long taskId,
String moduleType,
Long resultId,
String scopeKey,
String jobType,
LocalDateTime createdAt
) {
}

16
backend-java/src/main/java/com/nanri/aiimage/modules/task/model/dto/TaskFileJobMessage.java Normal file
View File

@@ -0,0 +1,16 @@
package com.nanri.aiimage.modules.task.model.dto;
import lombok.Data;
import java.time.LocalDateTime;
@Data
public class TaskFileJobMessage {
private Long jobId;
private Long taskId;
private String moduleType;
private Long resultId;
private String scopeKey;
private String jobType;
private LocalDateTime createdAt;
}

28
backend-java/src/main/java/com/nanri/aiimage/modules/task/model/entity/TaskFileJobEntity.java Normal file
View File

@@ -0,0 +1,28 @@
package com.nanri.aiimage.modules.task.model.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.time.LocalDateTime;
@Data
@TableName("biz_task_file_job")
public class TaskFileJobEntity {
@TableId(type = IdType.AUTO)
private Long id;
private Long taskId;
private String moduleType;
private Long resultId;
private String scopeKey;
private String jobType;
private String status;
private Integer retryCount;
private String errorMessage;
private String resultFileUrl;
private LocalDateTime createdAt;
private LocalDateTime updatedAt;
private LocalDateTime finishedAt;
}

28
backend-java/src/main/java/com/nanri/aiimage/modules/task/model/entity/TaskProgressSnapshotEntity.java Normal file
View File

@@ -0,0 +1,28 @@
package com.nanri.aiimage.modules.task.model.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.time.LocalDateTime;
@Data
@TableName("biz_task_progress_snapshot")
public class TaskProgressSnapshotEntity {
@TableId(type = IdType.AUTO)
private Long id;
private Long taskId;
private String moduleType;
private String status;
private Integer totalCount;
private Integer successCount;
private Integer failedCount;
private Integer pendingCount;
private String currentScopeKey;
private String message;
private String snapshotJson;
private LocalDateTime createdAt;
private LocalDateTime updatedAt;
}

29
backend-java/src/main/java/com/nanri/aiimage/modules/task/model/entity/TaskResultItemEntity.java Normal file
View File

@@ -0,0 +1,29 @@
package com.nanri.aiimage.modules.task.model.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.time.LocalDateTime;
@Data
@TableName("biz_task_result_item")
public class TaskResultItemEntity {
@TableId(type = IdType.AUTO)
private Long id;
private Long taskId;
private String moduleType;
private Long resultId;
private String scopeKey;
private String scopeHash;
private String itemKey;
private String countryCode;
private String asin;
private String status;
private String payloadJson;
private String payloadHash;
private LocalDateTime createdAt;
private LocalDateTime updatedAt;
}

26
backend-java/src/main/java/com/nanri/aiimage/modules/task/model/entity/TaskResultPayloadEntity.java Normal file
View File

@@ -0,0 +1,26 @@
package com.nanri.aiimage.modules.task.model.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.time.LocalDateTime;
@Data
@TableName("biz_task_result_payload")
public class TaskResultPayloadEntity {
@TableId(type = IdType.AUTO)
private Long id;
private Long taskId;
private String moduleType;
private String scopeKey;
private String scopeHash;
private String submissionId;
private String payloadJson;
private String payloadHash;
private Long version;
private LocalDateTime createdAt;
private LocalDateTime updatedAt;
}

39
backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobConsumer.java Normal file
View File

@@ -0,0 +1,39 @@
package com.nanri.aiimage.modules.task.service;
import com.nanri.aiimage.modules.task.mapper.TaskFileJobMapper;
import com.nanri.aiimage.modules.task.model.dto.TaskFileJobMessage;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.rocketmq.spring.annotation.RocketMQMessageListener;
import org.apache.rocketmq.spring.core.RocketMQListener;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
@Slf4j
@Service
@RequiredArgsConstructor
@ConditionalOnProperty(prefix = "aiimage.result-file-job", name = "mq-enabled", havingValue = "true")
@RocketMQMessageListener(
topic = "${aiimage.result-file-job.topic:aiimage-result-file-job}",
consumerGroup = "${aiimage.result-file-job.consumer-group:aiimage-result-file-job-consumer}"
)
public class TaskFileJobConsumer implements RocketMQListener<TaskFileJobMessage> {
private final TaskFileJobMapper taskFileJobMapper;
private final TaskResultFileJobWorker taskResultFileJobWorker;
@Override
public void onMessage(TaskFileJobMessage message) {
if (message == null || message.getJobId() == null) {
return;
}
TaskFileJobEntity job = taskFileJobMapper.selectById(message.getJobId());
if (job == null) {
log.warn("[task-file-job] mq message ignored, job not found jobId={} taskId={} moduleType={}",
message.getJobId(), message.getTaskId(), message.getModuleType());
return;
}
taskResultFileJobWorker.process(job);
}
}

42
backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobDispatchCoordinator.java Normal file
View File

@@ -0,0 +1,42 @@
package com.nanri.aiimage.modules.task.service;
import com.nanri.aiimage.modules.task.model.dto.TaskFileJobDispatchEvent;
import com.nanri.aiimage.modules.task.model.dto.TaskFileJobMessage;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.transaction.event.TransactionPhase;
import org.springframework.transaction.event.TransactionalEventListener;
@Slf4j
@Component
@RequiredArgsConstructor
public class TaskFileJobDispatchCoordinator {
private final TaskFileJobPublisher taskFileJobPublisher;
private final TaskFileJobLocalDispatcher taskFileJobLocalDispatcher;
@TransactionalEventListener(phase = TransactionPhase.AFTER_COMMIT, fallbackExecution = true)
public void onTaskFileJobDispatch(TaskFileJobDispatchEvent event) {
if (event == null || event.jobId() == null) {
return;
}
TaskFileJobMessage message = new TaskFileJobMessage();
message.setJobId(event.jobId());
message.setTaskId(event.taskId());
message.setModuleType(event.moduleType());
message.setResultId(event.resultId());
message.setScopeKey(event.scopeKey());
message.setJobType(event.jobType());
message.setCreatedAt(event.createdAt());
boolean published = taskFileJobPublisher.publish(message);
if (published) {
return;
}
boolean dispatched = taskFileJobLocalDispatcher.dispatch(event.jobId(), event.taskId(), event.moduleType());
if (dispatched) {
log.info("[task-file-job] local async dispatch scheduled jobId={} taskId={} moduleType={}",
event.jobId(), event.taskId(), event.moduleType());
}
}
}

53
backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobLocalDispatcher.java Normal file
View File

@@ -0,0 +1,53 @@
package com.nanri.aiimage.modules.task.service;
import com.nanri.aiimage.modules.task.mapper.TaskFileJobMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.ObjectProvider;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.task.TaskExecutor;
import org.springframework.stereotype.Service;
@Slf4j
@Service
@RequiredArgsConstructor
public class TaskFileJobLocalDispatcher {
private final TaskFileJobMapper taskFileJobMapper;
private final ObjectProvider<TaskResultFileJobWorker> taskResultFileJobWorkerProvider;
@Qualifier("taskFileJobDispatchExecutor")
private final TaskExecutor taskFileJobDispatchExecutor;
@Value("${aiimage.result-file-job.local-dispatch-enabled:true}")
private boolean localDispatchEnabled;
public boolean dispatch(Long jobId, Long taskId, String moduleType) {
if (!localDispatchEnabled || jobId == null || jobId <= 0) {
return false;
}
taskFileJobDispatchExecutor.execute(() -> {
try {
TaskFileJobEntity job = taskFileJobMapper.selectById(jobId);
if (job == null) {
log.warn("[task-file-job] local dispatch ignored, job not found jobId={} taskId={} moduleType={}",
jobId, taskId, moduleType);
return;
}
TaskResultFileJobWorker taskResultFileJobWorker = taskResultFileJobWorkerProvider.getIfAvailable();
if (taskResultFileJobWorker == null) {
log.warn("[task-file-job] local dispatch ignored, worker unavailable jobId={} taskId={} moduleType={}",
jobId, taskId, moduleType);
return;
}
taskResultFileJobWorker.process(job);
} catch (Exception ex) {
log.warn("[task-file-job] local dispatch failed jobId={} taskId={} moduleType={} msg={}",
jobId, taskId, moduleType, ex.getMessage(), ex);
}
});
return true;
}
}

50
backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobPublisher.java Normal file
View File

@@ -0,0 +1,50 @@
package com.nanri.aiimage.modules.task.service;
import com.nanri.aiimage.modules.task.model.dto.TaskFileJobMessage;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.rocketmq.spring.core.RocketMQTemplate;
import org.springframework.beans.factory.ObjectProvider;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
@Slf4j
@Service
@RequiredArgsConstructor
public class TaskFileJobPublisher {
private final ObjectProvider<RocketMQTemplate> rocketMQTemplateProvider;
@Value("${aiimage.result-file-job.mq-enabled:false}")
private boolean mqEnabled;
@Value("${aiimage.result-file-job.topic:aiimage-result-file-job}")
private String topic;
public boolean publish(TaskFileJobMessage message) {
if (message == null || message.getJobId() == null) {
return false;
}
if (!mqEnabled) {
log.info("[task-file-job] mq disabled, fallback to local dispatcher/scheduled worker jobId={} taskId={} moduleType={}",
message.getJobId(), message.getTaskId(), message.getModuleType());
return false;
}
RocketMQTemplate template = rocketMQTemplateProvider.getIfAvailable();
if (template == null) {
log.warn("[task-file-job] RocketMQTemplate unavailable, jobId={} taskId={} moduleType={}",
message.getJobId(), message.getTaskId(), message.getModuleType());
return false;
}
try {
template.convertAndSend(topic, message);
log.info("[task-file-job] mq published jobId={} taskId={} moduleType={} topic={}",
message.getJobId(), message.getTaskId(), message.getModuleType(), topic);
return true;
} catch (Exception ex) {
log.warn("[task-file-job] publish RocketMQ failed, fallback to local dispatcher/scheduled worker jobId={} taskId={} moduleType={} msg={}",
message.getJobId(), message.getTaskId(), message.getModuleType(), ex.getMessage());
return false;
}
}
}

225
backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobService.java Normal file
View File

@@ -0,0 +1,225 @@
package com.nanri.aiimage.modules.task.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.nanri.aiimage.modules.task.mapper.TaskFileJobMapper;
import com.nanri.aiimage.modules.task.model.dto.TaskFileJobDispatchEvent;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import lombok.RequiredArgsConstructor;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.time.LocalDateTime;
import java.util.List;
@Service
@RequiredArgsConstructor
public class TaskFileJobService {
public static final String JOB_TYPE_ASSEMBLE_RESULT = "ASSEMBLE_RESULT";
private final TaskFileJobMapper taskFileJobMapper;
private final ApplicationEventPublisher applicationEventPublisher;
@Transactional
public TaskFileJobEntity enqueueAssembleResult(Long taskId, String moduleType, Long resultId, String scopeKey) {
if (taskId == null || taskId <= 0 || moduleType == null || moduleType.isBlank() || resultId == null || resultId <= 0) {
return null;
}
LocalDateTime now = LocalDateTime.now();
TaskFileJobEntity existing = findJob(taskId, moduleType, resultId, JOB_TYPE_ASSEMBLE_RESULT);
if (existing == null) {
TaskFileJobEntity entity = new TaskFileJobEntity();
entity.setTaskId(taskId);
entity.setModuleType(moduleType);
entity.setResultId(resultId);
entity.setScopeKey(scopeKey);
entity.setJobType(JOB_TYPE_ASSEMBLE_RESULT);
entity.setStatus("PENDING");
entity.setRetryCount(0);
entity.setCreatedAt(now);
entity.setUpdatedAt(now);
try {
taskFileJobMapper.insert(entity);
publishDispatchEvent(entity);
return entity;
} catch (DuplicateKeyException ignored) {
existing = findJob(taskId, moduleType, resultId, JOB_TYPE_ASSEMBLE_RESULT);
}
}
if (existing == null) {
return null;
}
if ("SUCCESS".equals(existing.getStatus()) || "RUNNING".equals(existing.getStatus())) {
return existing;
}
taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getId, existing.getId())
.set(TaskFileJobEntity::getScopeKey, scopeKey)
.set(TaskFileJobEntity::getStatus, "PENDING")
.set(TaskFileJobEntity::getErrorMessage, null)
.set(TaskFileJobEntity::getUpdatedAt, now)
.set(TaskFileJobEntity::getFinishedAt, null));
TaskFileJobEntity refreshed = taskFileJobMapper.selectById(existing.getId());
publishDispatchEvent(refreshed);
return refreshed;
}
public List<TaskFileJobEntity> listRunnableJobs(int limit) {
return taskFileJobMapper.selectList(new LambdaQueryWrapper<TaskFileJobEntity>()
.in(TaskFileJobEntity::getStatus, List.of("PENDING", "FAILED"))
.lt(TaskFileJobEntity::getRetryCount, 5)
.orderByAsc(TaskFileJobEntity::getUpdatedAt)
.last("limit " + Math.max(1, Math.min(limit, 100))));
}
public List<TaskFileJobEntity> listJobs(String status, String moduleType, Long taskId, int limit) {
LambdaQueryWrapper<TaskFileJobEntity> wrapper = new LambdaQueryWrapper<TaskFileJobEntity>()
.orderByDesc(TaskFileJobEntity::getUpdatedAt)
.last("limit " + Math.max(1, Math.min(limit, 500)));
if (status != null && !status.isBlank()) {
wrapper.eq(TaskFileJobEntity::getStatus, status.trim());
}
if (moduleType != null && !moduleType.isBlank()) {
wrapper.eq(TaskFileJobEntity::getModuleType, moduleType.trim());
}
if (taskId != null && taskId > 0) {
wrapper.eq(TaskFileJobEntity::getTaskId, taskId);
}
return taskFileJobMapper.selectList(wrapper);
}
@Transactional
public boolean retry(Long jobId) {
if (jobId == null || jobId <= 0) {
return false;
}
int updated = taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getId, jobId)
.ne(TaskFileJobEntity::getStatus, "SUCCESS")
.set(TaskFileJobEntity::getStatus, "PENDING")
.set(TaskFileJobEntity::getRetryCount, 0)
.set(TaskFileJobEntity::getErrorMessage, null)
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())
.set(TaskFileJobEntity::getFinishedAt, null));
if (updated > 0) {
TaskFileJobEntity refreshed = taskFileJobMapper.selectById(jobId);
publishDispatchEvent(refreshed);
return true;
}
return false;
}
@Transactional
public int resetStuckRunningJobs(int stuckMinutes, int limit) {
LocalDateTime threshold = LocalDateTime.now().minusMinutes(Math.max(5, stuckMinutes));
List<TaskFileJobEntity> jobs = taskFileJobMapper.selectList(new LambdaQueryWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getStatus, "RUNNING")
.lt(TaskFileJobEntity::getUpdatedAt, threshold)
.orderByAsc(TaskFileJobEntity::getUpdatedAt)
.last("limit " + Math.max(1, Math.min(limit, 200))));
int reset = 0;
for (TaskFileJobEntity job : jobs) {
int retryCount = job.getRetryCount() == null ? 0 : job.getRetryCount();
if (retryCount >= 5) {
taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getId, job.getId())
.eq(TaskFileJobEntity::getStatus, "RUNNING")
.set(TaskFileJobEntity::getStatus, "FAILED")
.set(TaskFileJobEntity::getErrorMessage, "文件生成任务运行超时")
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now()));
continue;
}
int updated = taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getId, job.getId())
.eq(TaskFileJobEntity::getStatus, "RUNNING")
.set(TaskFileJobEntity::getStatus, "PENDING")
.set(TaskFileJobEntity::getErrorMessage, "文件生成任务运行超时，已重新排队")
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now()));
if (updated > 0) {
reset++;
publishDispatchEvent(taskFileJobMapper.selectById(job.getId()));
}
}
return reset;
}
@Transactional
public boolean markRunning(Long jobId) {
if (jobId == null || jobId <= 0) {
return false;
}
return taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getId, jobId)
.in(TaskFileJobEntity::getStatus, List.of("PENDING", "FAILED"))
.set(TaskFileJobEntity::getStatus, "RUNNING")
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())) > 0;
}
@Transactional
public void markSuccess(TaskFileJobEntity job, String resultFileUrl) {
taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getId, job.getId())
.set(TaskFileJobEntity::getStatus, "SUCCESS")
.set(TaskFileJobEntity::getErrorMessage, null)
.set(TaskFileJobEntity::getResultFileUrl, resultFileUrl)
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())
.set(TaskFileJobEntity::getFinishedAt, LocalDateTime.now()));
}
@Transactional
public void markFailed(TaskFileJobEntity job, String message) {
int retryCount = job.getRetryCount() == null ? 0 : job.getRetryCount() + 1;
taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getId, job.getId())
.set(TaskFileJobEntity::getStatus, "FAILED")
.set(TaskFileJobEntity::getRetryCount, retryCount)
.set(TaskFileJobEntity::getErrorMessage, message == null ? "结果文件生成失败" : message)
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now()));
}
public TaskFileJobEntity findAssembleJob(Long taskId, String moduleType, Long resultId) {
return findJob(taskId, moduleType, resultId, JOB_TYPE_ASSEMBLE_RESULT);
}
public long countUnfinishedAssembleJobs(Long taskId, String moduleType) {
if (taskId == null || taskId <= 0 || moduleType == null || moduleType.isBlank()) {
return 0L;
}
Long count = taskFileJobMapper.selectCount(new LambdaQueryWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getTaskId, taskId)
.eq(TaskFileJobEntity::getModuleType, moduleType)
.eq(TaskFileJobEntity::getJobType, JOB_TYPE_ASSEMBLE_RESULT)
.ne(TaskFileJobEntity::getStatus, "SUCCESS"));
return count == null ? 0L : count;
}
private TaskFileJobEntity findJob(Long taskId, String moduleType, Long resultId, String jobType) {
if (taskId == null || moduleType == null || moduleType.isBlank() || resultId == null || jobType == null || jobType.isBlank()) {
return null;
}
return taskFileJobMapper.selectOne(new LambdaQueryWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getTaskId, taskId)
.eq(TaskFileJobEntity::getModuleType, moduleType)
.eq(TaskFileJobEntity::getResultId, resultId)
.eq(TaskFileJobEntity::getJobType, jobType)
.last("limit 1"));
}
private void publishDispatchEvent(TaskFileJobEntity entity) {
if (entity == null || entity.getId() == null) {
return;
}
applicationEventPublisher.publishEvent(new TaskFileJobDispatchEvent(
entity.getId(),
entity.getTaskId(),
entity.getModuleType(),
entity.getResultId(),
entity.getScopeKey(),
entity.getJobType(),
LocalDateTime.now()
));
}
}

98
backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskProgressSnapshotService.java Normal file
View File

@@ -0,0 +1,98 @@
package com.nanri.aiimage.modules.task.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.task.mapper.TaskProgressSnapshotMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskProgressSnapshotEntity;
import lombok.RequiredArgsConstructor;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.time.LocalDateTime;
@Service
@RequiredArgsConstructor
public class TaskProgressSnapshotService {
private final TaskProgressSnapshotMapper taskProgressSnapshotMapper;
private final ObjectMapper objectMapper;
@Transactional
public void save(Long taskId,
String moduleType,
String status,
int totalCount,
int successCount,
int failedCount,
String currentScopeKey,
String message,
Object snapshot) {
if (taskId == null || taskId <= 0 || isBlank(moduleType) || isBlank(status)) {
return;
}
int pendingCount = Math.max(0, totalCount - successCount - failedCount);
String snapshotJson = snapshot == null ? null : writeJson(snapshot);
LocalDateTime now = LocalDateTime.now();
TaskProgressSnapshotEntity existing = find(taskId, moduleType);
if (existing == null) {
TaskProgressSnapshotEntity entity = new TaskProgressSnapshotEntity();
entity.setTaskId(taskId);
entity.setModuleType(moduleType);
entity.setStatus(status);
entity.setTotalCount(totalCount);
entity.setSuccessCount(successCount);
entity.setFailedCount(failedCount);
entity.setPendingCount(pendingCount);
entity.setCurrentScopeKey(currentScopeKey);
entity.setMessage(message);
entity.setSnapshotJson(snapshotJson);
entity.setCreatedAt(now);
entity.setUpdatedAt(now);
try {
taskProgressSnapshotMapper.insert(entity);
return;
} catch (DuplicateKeyException ignored) {
existing = find(taskId, moduleType);
}
}
if (existing == null) {
throw new BusinessException("保存任务进度快照失败");
}
taskProgressSnapshotMapper.update(null, new LambdaUpdateWrapper<TaskProgressSnapshotEntity>()
.eq(TaskProgressSnapshotEntity::getId, existing.getId())
.set(TaskProgressSnapshotEntity::getStatus, status)
.set(TaskProgressSnapshotEntity::getTotalCount, totalCount)
.set(TaskProgressSnapshotEntity::getSuccessCount, successCount)
.set(TaskProgressSnapshotEntity::getFailedCount, failedCount)
.set(TaskProgressSnapshotEntity::getPendingCount, pendingCount)
.set(TaskProgressSnapshotEntity::getCurrentScopeKey, currentScopeKey)
.set(TaskProgressSnapshotEntity::getMessage, message)
.set(TaskProgressSnapshotEntity::getSnapshotJson, snapshotJson)
.set(TaskProgressSnapshotEntity::getUpdatedAt, now));
}
public TaskProgressSnapshotEntity find(Long taskId, String moduleType) {
if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
return null;
}
return taskProgressSnapshotMapper.selectOne(new LambdaQueryWrapper<TaskProgressSnapshotEntity>()
.eq(TaskProgressSnapshotEntity::getTaskId, taskId)
.eq(TaskProgressSnapshotEntity::getModuleType, moduleType)
.last("limit 1"));
}
private String writeJson(Object value) {
try {
return objectMapper.writeValueAsString(value);
} catch (Exception ex) {
throw new BusinessException("序列化任务进度快照失败");
}
}
private boolean isBlank(String value) {
return value == null || value.isBlank();
}
}

166
backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorker.java Normal file
View File

@@ -0,0 +1,166 @@
package com.nanri.aiimage.modules.task.service;
import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskService;
import com.nanri.aiimage.modules.brand.service.BrandTaskService;
import com.nanri.aiimage.modules.deletebrand.service.DeleteBrandRunService;
import com.nanri.aiimage.modules.patroldelete.service.PatrolDeleteTaskService;
import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskService;
import com.nanri.aiimage.modules.productrisk.service.ProductRiskTaskService;
import com.nanri.aiimage.modules.queryasin.service.QueryAsinTaskService;
import com.nanri.aiimage.modules.shopmatch.service.ShopMatchTaskService;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import java.util.List;
@Slf4j
@Service
@RequiredArgsConstructor
public class TaskResultFileJobWorker {
private final TaskFileJobService taskFileJobService;
private final TaskResultPayloadService taskResultPayloadService;
private final FileResultMapper fileResultMapper;
private final TaskFileJobLocalDispatcher taskFileJobLocalDispatcher;
private final ShopMatchTaskService shopMatchTaskService;
private final PriceTrackTaskService priceTrackTaskService;
private final ProductRiskTaskService productRiskTaskService;
private final QueryAsinTaskService queryAsinTaskService;
private final PatrolDeleteTaskService patrolDeleteTaskService;
private final AppearancePatentTaskService appearancePatentTaskService;
private final DeleteBrandRunService deleteBrandRunService;
private final BrandTaskService brandTaskService;
@Value("${aiimage.result-file-job.local-worker-enabled:true}")
private boolean localWorkerEnabled;
@Value("${aiimage.result-file-job.batch-size:20}")
private int batchSize;
@Value("${aiimage.result-file-job.stuck-timeout-minutes:30}")
private int stuckTimeoutMinutes;
@Scheduled(fixedDelayString = "${aiimage.result-file-job.local-worker-delay-ms:15000}")
public void runPendingJobs() {
if (!localWorkerEnabled) {
return;
}
List<TaskFileJobEntity> jobs = taskFileJobService.listRunnableJobs(batchSize);
if (jobs == null || jobs.isEmpty()) {
return;
}
log.info("[task-file-job] scheduled worker picked jobs count={}", jobs.size());
for (TaskFileJobEntity job : jobs) {
boolean dispatched = taskFileJobLocalDispatcher.dispatch(
job.getId(),
job.getTaskId(),
job.getModuleType()
);
if (!dispatched) {
process(job);
}
}
}
@Scheduled(fixedDelayString = "${aiimage.result-file-job.stuck-scan-delay-ms:60000}")
public void resetStuckJobs() {
int reset = taskFileJobService.resetStuckRunningJobs(stuckTimeoutMinutes, batchSize);
if (reset > 0) {
log.warn("[task-file-job] reset stuck running jobs count={} timeoutMinutes={}", reset, stuckTimeoutMinutes);
}
}
public void process(TaskFileJobEntity job) {
if (job == null || job.getId() == null || !taskFileJobService.markRunning(job.getId())) {
return;
}
long startedAt = System.currentTimeMillis();
try {
dispatch(job);
String resultFileUrl = resolveResultFileUrl(job);
taskFileJobService.markSuccess(job, resultFileUrl);
cleanupAfterSuccess(job);
log.info("[task-file-job] process success jobId={} taskId={} moduleType={} resultId={} elapsedMs={} resultFileUrl={}",
job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId(),
System.currentTimeMillis() - startedAt, resultFileUrl);
} catch (Exception ex) {
String message = ex.getMessage() == null ? "结果文件生成失败" : ex.getMessage();
log.warn("[task-file-job] process failed jobId={} taskId={} moduleType={} resultId={} msg={}",
job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId(), message);
taskFileJobService.markFailed(job, message);
}
}
private String resolveResultFileUrl(TaskFileJobEntity job) {
if ("BRAND".equals(job.getModuleType())) {
return brandTaskService.resolveResultObjectKey(job.getTaskId());
}
if (job.getResultId() == null) {
return null;
}
FileResultEntity result = fileResultMapper.selectById(job.getResultId());
return result == null ? null : result.getResultFileUrl();
}
private void dispatch(TaskFileJobEntity job) {
String moduleType = job.getModuleType();
if ("SHOP_MATCH".equals(moduleType)) {
shopMatchTaskService.processResultFileJob(job);
return;
}
if ("PRICE_TRACK".equals(moduleType)) {
priceTrackTaskService.processResultFileJob(job);
return;
}
if ("PRODUCT_RISK_RESOLVE".equals(moduleType)) {
productRiskTaskService.processResultFileJob(job);
return;
}
if ("QUERY_ASIN".equals(moduleType)) {
queryAsinTaskService.processResultFileJob(job);
return;
}
if ("PATROL_DELETE".equals(moduleType)) {
patrolDeleteTaskService.processResultFileJob(job);
return;
}
if ("APPEARANCE_PATENT".equals(moduleType)) {
appearancePatentTaskService.processResultFileJob(job);
return;
}
if ("DELETE_BRAND".equals(moduleType)) {
deleteBrandRunService.processResultFileJob(job);
return;
}
if ("BRAND".equals(moduleType)) {
brandTaskService.processResultFileJob(job);
return;
}
throw new IllegalArgumentException("unsupported result file job module: " + moduleType);
}
private void cleanupAfterSuccess(TaskFileJobEntity job) {
String moduleType = job.getModuleType();
if ("SHOP_MATCH".equals(moduleType)
|| "PRICE_TRACK".equals(moduleType)
|| "PRODUCT_RISK_RESOLVE".equals(moduleType)
|| "QUERY_ASIN".equals(moduleType)
|| "PATROL_DELETE".equals(moduleType)) {
taskResultPayloadService.deleteLatest(job.getTaskId(), moduleType, job.getScopeKey());
return;
}
if ("APPEARANCE_PATENT".equals(moduleType)) {
appearancePatentTaskService.cleanupResultFileJob(job);
return;
}
if ("DELETE_BRAND".equals(moduleType)) {
deleteBrandRunService.cleanupResultFileJob(job);
}
}
}

233
backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultItemService.java Normal file
View File

@@ -0,0 +1,233 @@
package com.nanri.aiimage.modules.task.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.task.mapper.TaskResultItemMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskResultItemEntity;
import lombok.RequiredArgsConstructor;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
@Service
@RequiredArgsConstructor
public class TaskResultItemService {
private final TaskResultItemMapper taskResultItemMapper;
private final ObjectMapper objectMapper;
private final TransientPayloadStorageService transientPayloadStorageService;
@Transactional
public void replaceResultSnapshot(Long taskId, String moduleType, Long resultId, String scopeKey, Object snapshot) {
if (taskId == null || taskId <= 0 || isBlank(moduleType) || resultId == null || resultId <= 0 || snapshot == null) {
return;
}
String normalizedScopeKey = firstNonBlank(scopeKey, "result:" + resultId).trim();
String itemKey = "result:" + resultId;
upsert(taskId, moduleType, resultId, normalizedScopeKey, itemKey, null, null, null, snapshot);
}
@Transactional
public void replaceTaskSnapshots(Long taskId, String moduleType, List<? extends Object> snapshots, SnapshotKeyResolver resolver) {
if (taskId == null || taskId <= 0 || isBlank(moduleType) || snapshots == null) {
return;
}
for (Object snapshot : snapshots) {
if (snapshot == null) {
continue;
}
Long resultId = resolver == null ? null : resolver.resultId(snapshot);
if (resultId == null || resultId <= 0) {
continue;
}
replaceResultSnapshot(taskId, moduleType, resultId, resolver.scopeKey(snapshot), snapshot);
}
}
public <T> List<T> listResultSnapshots(Long taskId, String moduleType, Class<T> clazz) {
if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
return List.of();
}
List<TaskResultItemEntity> rows = taskResultItemMapper.selectList(new LambdaQueryWrapper<TaskResultItemEntity>()
.eq(TaskResultItemEntity::getTaskId, taskId)
.eq(TaskResultItemEntity::getModuleType, moduleType)
.likeRight(TaskResultItemEntity::getItemKey, "result:")
.orderByAsc(TaskResultItemEntity::getResultId)
.orderByAsc(TaskResultItemEntity::getId));
List<T> out = new ArrayList<>();
for (TaskResultItemEntity row : rows) {
T value = readPayload(row, clazz);
if (value != null) {
out.add(value);
}
}
return out;
}
public <T> T getResultSnapshot(Long taskId, String moduleType, Long resultId, Class<T> clazz) {
if (taskId == null || taskId <= 0 || isBlank(moduleType) || resultId == null || resultId <= 0) {
return null;
}
TaskResultItemEntity row = taskResultItemMapper.selectOne(new LambdaQueryWrapper<TaskResultItemEntity>()
.eq(TaskResultItemEntity::getTaskId, taskId)
.eq(TaskResultItemEntity::getModuleType, moduleType)
.eq(TaskResultItemEntity::getResultId, resultId)
.eq(TaskResultItemEntity::getItemKey, "result:" + resultId)
.last("limit 1"));
return readPayload(row, clazz);
}
public long countResultSnapshots(Long taskId, String moduleType) {
if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
return 0L;
}
Long count = taskResultItemMapper.selectCount(new LambdaQueryWrapper<TaskResultItemEntity>()
.eq(TaskResultItemEntity::getTaskId, taskId)
.eq(TaskResultItemEntity::getModuleType, moduleType)
.likeRight(TaskResultItemEntity::getItemKey, "result:"));
return count == null ? 0L : count;
}
@Transactional
public void deleteTaskItems(Long taskId, String moduleType) {
if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
return;
}
List<TaskResultItemEntity> rows = taskResultItemMapper.selectList(new LambdaQueryWrapper<TaskResultItemEntity>()
.select(TaskResultItemEntity::getId, TaskResultItemEntity::getPayloadJson)
.eq(TaskResultItemEntity::getTaskId, taskId)
.eq(TaskResultItemEntity::getModuleType, moduleType));
if (rows != null) {
for (TaskResultItemEntity row : rows) {
transientPayloadStorageService.deletePayloadIfPresent(row.getPayloadJson());
}
}
taskResultItemMapper.delete(new LambdaQueryWrapper<TaskResultItemEntity>()
.eq(TaskResultItemEntity::getTaskId, taskId)
.eq(TaskResultItemEntity::getModuleType, moduleType));
}
private void upsert(Long taskId,
String moduleType,
Long resultId,
String scopeKey,
String itemKey,
String countryCode,
String asin,
String status,
Object payload) {
String payloadJson = writeJson(payload);
String normalizedScopeKey = firstNonBlank(scopeKey, "task:" + taskId).trim();
String normalizedItemKey = firstNonBlank(itemKey, "result:" + resultId).trim();
String scopeHash = hash(normalizedScopeKey);
String storedPayload = transientPayloadStorageService.storeResultItemPayload(
moduleType, taskId, scopeHash, normalizedItemKey, payloadJson);
String payloadHash = hash(payloadJson);
LocalDateTime now = LocalDateTime.now();
TaskResultItemEntity existing = find(taskId, moduleType, scopeHash, normalizedItemKey);
if (existing == null) {
TaskResultItemEntity entity = new TaskResultItemEntity();
entity.setTaskId(taskId);
entity.setModuleType(moduleType);
entity.setResultId(resultId);
entity.setScopeKey(normalizedScopeKey);
entity.setScopeHash(scopeHash);
entity.setItemKey(normalizedItemKey);
entity.setCountryCode(countryCode);
entity.setAsin(asin);
entity.setStatus(status);
entity.setPayloadJson(storedPayload);
entity.setPayloadHash(payloadHash);
entity.setCreatedAt(now);
entity.setUpdatedAt(now);
try {
taskResultItemMapper.insert(entity);
return;
} catch (DuplicateKeyException ignored) {
existing = find(taskId, moduleType, scopeHash, normalizedItemKey);
}
}
if (existing == null) {
throw new BusinessException("保存任务结果明细失败");
}
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(existing.getPayloadJson(), storedPayload);
taskResultItemMapper.update(null, new LambdaUpdateWrapper<TaskResultItemEntity>()
.eq(TaskResultItemEntity::getId, existing.getId())
.set(TaskResultItemEntity::getResultId, resultId)
.set(TaskResultItemEntity::getScopeKey, normalizedScopeKey)
.set(TaskResultItemEntity::getCountryCode, countryCode)
.set(TaskResultItemEntity::getAsin, asin)
.set(TaskResultItemEntity::getStatus, status)
.set(TaskResultItemEntity::getPayloadJson, storedPayload)
.set(TaskResultItemEntity::getPayloadHash, payloadHash)
.set(TaskResultItemEntity::getUpdatedAt, now));
}
private TaskResultItemEntity find(Long taskId, String moduleType, String scopeHash, String itemKey) {
return taskResultItemMapper.selectOne(new LambdaQueryWrapper<TaskResultItemEntity>()
.eq(TaskResultItemEntity::getTaskId, taskId)
.eq(TaskResultItemEntity::getModuleType, moduleType)
.eq(TaskResultItemEntity::getScopeHash, scopeHash)
.eq(TaskResultItemEntity::getItemKey, itemKey)
.last("limit 1"));
}
private <T> T readPayload(TaskResultItemEntity row, Class<T> clazz) {
if (row == null || isBlank(row.getPayloadJson())) {
return null;
}
try {
String payloadJson = transientPayloadStorageService.resolvePayload(row.getPayloadJson(), "read task result item failed");
return objectMapper.readValue(payloadJson, clazz);
} catch (BusinessException ex) {
throw ex;
} catch (Exception ex) {
throw new BusinessException("读取任务结果明细失败");
}
}
private String writeJson(Object value) {
try {
return objectMapper.writeValueAsString(value);
} catch (Exception ex) {
throw new BusinessException("序列化任务结果明细失败");
}
}
private String hash(String value) {
try {
MessageDigest digest = MessageDigest.getInstance("SHA-256");
byte[] bytes = digest.digest((value == null ? "" : value).getBytes(StandardCharsets.UTF_8));
StringBuilder sb = new StringBuilder(bytes.length * 2);
for (byte b : bytes) {
sb.append(String.format("%02x", b));
}
return sb.toString();
} catch (Exception ex) {
throw new IllegalStateException("failed to hash task result item", ex);
}
}
private String firstNonBlank(String primary, String fallback) {
return primary == null || primary.isBlank() ? fallback : primary;
}
private boolean isBlank(String value) {
return value == null || value.isBlank();
}
public interface SnapshotKeyResolver {
Long resultId(Object snapshot);
String scopeKey(Object snapshot);
}
}

139
backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultPayloadService.java Normal file
View File

@@ -0,0 +1,139 @@
package com.nanri.aiimage.modules.task.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.task.mapper.TaskResultPayloadMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskResultPayloadEntity;
import lombok.RequiredArgsConstructor;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.time.LocalDateTime;
@Service
@RequiredArgsConstructor
public class TaskResultPayloadService {
private static final String LATEST_SUBMISSION_ID = "latest";
private final TaskResultPayloadMapper taskResultPayloadMapper;
private final ObjectMapper objectMapper;
private final TransientPayloadStorageService transientPayloadStorageService;
@Transactional
public void saveLatest(Long taskId, String moduleType, String scopeKey, Object payload) {
if (taskId == null || taskId <= 0 || isBlank(moduleType) || isBlank(scopeKey) || payload == null) {
return;
}
String normalizedScopeKey = scopeKey.trim();
String scopeHash = hash(normalizedScopeKey);
String payloadJson = writeJson(payload);
String storedPayload = transientPayloadStorageService.storeResultPayload(
moduleType, taskId, scopeHash, LATEST_SUBMISSION_ID, payloadJson);
String payloadHash = hash(payloadJson);
LocalDateTime now = LocalDateTime.now();
TaskResultPayloadEntity existing = findLatest(taskId, moduleType, scopeHash);
if (existing == null) {
TaskResultPayloadEntity entity = new TaskResultPayloadEntity();
entity.setTaskId(taskId);
entity.setModuleType(moduleType);
entity.setScopeKey(normalizedScopeKey);
entity.setScopeHash(scopeHash);
entity.setSubmissionId(LATEST_SUBMISSION_ID);
entity.setPayloadJson(storedPayload);
entity.setPayloadHash(payloadHash);
entity.setVersion(1L);
entity.setCreatedAt(now);
entity.setUpdatedAt(now);
try {
taskResultPayloadMapper.insert(entity);
return;
} catch (DuplicateKeyException ignored) {
existing = findLatest(taskId, moduleType, scopeHash);
}
}
if (existing == null) {
throw new BusinessException("保存任务结果载荷失败");
}
long nextVersion = existing.getVersion() == null ? 1L : existing.getVersion() + 1L;
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(existing.getPayloadJson(), storedPayload);
taskResultPayloadMapper.update(null, new LambdaUpdateWrapper<TaskResultPayloadEntity>()
.eq(TaskResultPayloadEntity::getId, existing.getId())
.set(TaskResultPayloadEntity::getScopeKey, normalizedScopeKey)
.set(TaskResultPayloadEntity::getPayloadJson, storedPayload)
.set(TaskResultPayloadEntity::getPayloadHash, payloadHash)
.set(TaskResultPayloadEntity::getVersion, nextVersion)
.set(TaskResultPayloadEntity::getUpdatedAt, now));
}
public <T> T getLatest(Long taskId, String moduleType, String scopeKey, Class<T> clazz) {
if (taskId == null || taskId <= 0 || isBlank(moduleType) || isBlank(scopeKey)) {
return null;
}
TaskResultPayloadEntity entity = findLatest(taskId, moduleType, hash(scopeKey.trim()));
if (entity == null || isBlank(entity.getPayloadJson())) {
return null;
}
try {
String payloadJson = transientPayloadStorageService.resolvePayload(entity.getPayloadJson(), "read task result payload failed");
return objectMapper.readValue(payloadJson, clazz);
} catch (BusinessException ex) {
throw ex;
} catch (Exception ex) {
throw new BusinessException("读取任务结果载荷失败");
}
}
@Transactional
public void deleteLatest(Long taskId, String moduleType, String scopeKey) {
if (taskId == null || taskId <= 0 || isBlank(moduleType) || isBlank(scopeKey)) {
return;
}
TaskResultPayloadEntity existing = findLatest(taskId, moduleType, hash(scopeKey.trim()));
if (existing != null) {
transientPayloadStorageService.deletePayloadIfPresent(existing.getPayloadJson());
taskResultPayloadMapper.deleteById(existing.getId());
}
}
private TaskResultPayloadEntity findLatest(Long taskId, String moduleType, String scopeHash) {
return taskResultPayloadMapper.selectOne(new LambdaQueryWrapper<TaskResultPayloadEntity>()
.eq(TaskResultPayloadEntity::getTaskId, taskId)
.eq(TaskResultPayloadEntity::getModuleType, moduleType)
.eq(TaskResultPayloadEntity::getScopeHash, scopeHash)
.eq(TaskResultPayloadEntity::getSubmissionId, LATEST_SUBMISSION_ID)
.last("limit 1"));
}
private String writeJson(Object value) {
try {
return objectMapper.writeValueAsString(value);
} catch (Exception ex) {
throw new BusinessException("序列化任务结果载荷失败");
}
}
private String hash(String value) {
try {
MessageDigest digest = MessageDigest.getInstance("SHA-256");
byte[] bytes = digest.digest((value == null ? "" : value).getBytes(StandardCharsets.UTF_8));
StringBuilder sb = new StringBuilder(bytes.length * 2);
for (byte b : bytes) {
sb.append(String.format("%02x", b));
}
return sb.toString();
} catch (Exception ex) {
throw new IllegalStateException("failed to hash task result payload", ex);
}
}
private boolean isBlank(String value) {
return value == null || value.isBlank();
}
}

255
backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskScopePayloadStorageService.java
View File

@@ -3,9 +3,7 @@ package com.nanri.aiimage.modules.task.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.TaskPressureProperties;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import lombok.RequiredArgsConstructor;
@@ -13,37 +11,23 @@ import org.springframework.dao.DuplicateKeyException;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.security.MessageDigest;
import java.time.LocalDateTime;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
@Service
@RequiredArgsConstructor
public class TaskScopePayloadStorageService {
private static final Set<String> OSS_BACKED_STATE_MODULES = new HashSet<>(Set.of(
"SHOP_MATCH",
"PRODUCT_RISK_RESOLVE",
"PRICE_TRACK",
"PATROL_DELETE",
"QUERY_ASIN"
));
private static final String OSS_POINTER_PREFIX = "oss:";
private static final String LOCAL_BUFFER_DIR = "task-scope-buffer";
private final TaskScopeStateMapper taskScopeStateMapper;
private final ObjectMapper objectMapper;
private final OssStorageService ossStorageService;
private final TaskPressureProperties taskPressureProperties;
private final TransientPayloadStorageService transientPayloadStorageService;
@Transactional
public <T> void saveScopePayload(Long taskId, String moduleType, String scopeKey, T payload) {
@@ -53,13 +37,11 @@ public class TaskScopePayloadStorageService {
String normalizedScopeKey = normalize(scopeKey);
String scopeHash = hash(normalizedScopeKey);
String payloadJson = writeJson(payload, "写入任务范围载荷失败");
String stateJson = transientPayloadStorageService.storeScopePayload(
moduleType, taskId, scopeHash, payloadJson, true);
LocalDateTime now = LocalDateTime.now();
TaskScopeStateEntity state = getScopeState(taskId, moduleType, scopeHash);
if (shouldStoreStatePayloadInOss(moduleType)) {
saveOssBackedScopePayload(taskId, moduleType, normalizedScopeKey, scopeHash, payloadJson, now, state);
return;
}
String stateJson = storeStatePayload(moduleType, taskId, scopeHash, payloadJson);
if (state == null) {
TaskScopeStateEntity entity = new TaskScopeStateEntity();
entity.setTaskId(taskId);
@@ -84,7 +66,7 @@ public class TaskScopePayloadStorageService {
if (state == null) {
throw new BusinessException("写入任务范围载荷失败");
}
deleteReplacedStatePayloadIfNeeded(state.getStateJson(), stateJson);
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(state.getStateJson(), stateJson);
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getId, state.getId())
.set(TaskScopeStateEntity::getScopeKey, normalizedScopeKey)
@@ -145,8 +127,7 @@ public class TaskScopePayloadStorageService {
if (state == null) {
return;
}
deleteBufferedPayload(taskId, moduleType, state.getScopeHash());
deleteStatePayloadIfNeeded(state.getStateJson());
transientPayloadStorageService.deletePayloadIfPresent(state.getStateJson());
boolean hasParsedPayload = !isBlank(state.getParsedPayloadJson());
if (hasParsedPayload) {
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
@@ -195,8 +176,7 @@ public class TaskScopePayloadStorageService {
if (state == null || state.getId() == null) {
continue;
}
deleteBufferedPayload(taskId, moduleType, state.getScopeHash());
deleteStatePayloadIfNeeded(state.getStateJson());
transientPayloadStorageService.deletePayloadIfPresent(state.getStateJson());
if (!isBlank(state.getParsedPayloadJson())) {
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getId, state.getId())
@@ -221,6 +201,9 @@ public class TaskScopePayloadStorageService {
String scopeKey = normalize(entry.getKey());
String scopeHash = hash(scopeKey);
String parsedPayloadJson = writeJson(entry.getValue(), "写入任务解析载荷失败");
String storedParsedPayload = transientPayloadStorageService.storeParsedPayload(
moduleType, taskId, scopeHash, parsedPayloadJson, true);
TaskScopeStateEntity state = getScopeState(taskId, moduleType, scopeHash);
if (state == null) {
TaskScopeStateEntity entity = new TaskScopeStateEntity();
@@ -228,7 +211,7 @@ public class TaskScopePayloadStorageService {
entity.setModuleType(moduleType);
entity.setScopeKey(scopeKey);
entity.setScopeHash(scopeHash);
entity.setParsedPayloadJson(parsedPayloadJson);
entity.setParsedPayloadJson(storedParsedPayload);
entity.setChunkTotal(0);
entity.setReceivedChunkCount(0);
entity.setCompleted(0);
@@ -244,11 +227,11 @@ public class TaskScopePayloadStorageService {
if (state == null) {
throw new BusinessException("写入任务解析载荷失败");
}
deleteParsedPayloadIfNeeded(state.getParsedPayloadJson(), parsedPayloadJson);
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(state.getParsedPayloadJson(), storedParsedPayload);
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getId, state.getId())
.set(TaskScopeStateEntity::getScopeKey, scopeKey)
.set(TaskScopeStateEntity::getParsedPayloadJson, parsedPayloadJson)
.set(TaskScopeStateEntity::getParsedPayloadJson, storedParsedPayload)
.set(TaskScopeStateEntity::getUpdatedAt, now));
}
}
@@ -271,7 +254,9 @@ public class TaskScopePayloadStorageService {
continue;
}
try {
result.put(state.getScopeKey(), objectMapper.readValue(resolveParsedPayload(state.getParsedPayloadJson()), clazz));
String payloadJson = transientPayloadStorageService.resolvePayload(
state.getParsedPayloadJson(), "read parsed payload failed");
result.put(state.getScopeKey(), objectMapper.readValue(payloadJson, clazz));
} catch (Exception ex) {
throw new BusinessException("读取任务原始载荷失败");
}
@@ -281,33 +266,8 @@ public class TaskScopePayloadStorageService {
@Transactional
public boolean recoverBufferedScopePayload(Long taskId, String moduleType, String scopeHash) {
if (taskId == null || taskId <= 0 || isBlank(moduleType) || isBlank(scopeHash)) {
return false;
}
String payloadJson = readBufferedPayload(taskId, moduleType, scopeHash);
if (payloadJson == null) {
return false;
}
TaskScopeStateEntity state = getScopeState(taskId, moduleType, scopeHash);
if (state == null) {
deleteBufferedPayload(taskId, moduleType, scopeHash);
return false;
}
if (!shouldStoreStatePayloadInOss(moduleType)) {
deleteBufferedPayload(taskId, moduleType, scopeHash);
return false;
}
LocalDateTime now = LocalDateTime.now();
String stateJson = storeStatePayload(moduleType, taskId, scopeHash, payloadJson);
deleteReplacedStatePayloadIfNeeded(state.getStateJson(), stateJson);
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getId, state.getId())
.set(TaskScopeStateEntity::getStateJson, stateJson)
.set(TaskScopeStateEntity::getLastChunkAt, now)
.set(TaskScopeStateEntity::getUpdatedAt, now));
deleteBufferedPayload(taskId, moduleType, scopeHash);
return true;
}
public Path getBufferedPayloadRoot() {
return Path.of(System.getProperty("java.io.tmpdir"), LOCAL_BUFFER_DIR);
@@ -321,126 +281,17 @@ public class TaskScopePayloadStorageService {
.last("limit 1"));
}
private String storeStatePayload(String moduleType, Long taskId, String scopeHash, String payloadJson) {
if (!shouldStoreStatePayloadInOss(moduleType)) {
return payloadJson;
}
try {
String pointer = OSS_POINTER_PREFIX + ossStorageService.uploadTaskScopePayload(moduleType, taskId, scopeHash, payloadJson);
return objectMapper.writeValueAsString(pointer);
} catch (Exception ex) {
throw new BusinessException("写入任务范围载荷失败");
}
}
private String resolveStatePayload(TaskScopeStateEntity state) {
if (state == null || isBlank(state.getStateJson())) {
return null;
}
String bufferedPayload = readBufferedPayload(state.getTaskId(), state.getModuleType(), state.getScopeHash());
if (bufferedPayload != null) {
return bufferedPayload;
}
String ossPointer = extractOssPointer(state.getStateJson());
if (ossPointer == null) {
return state.getStateJson();
}
try {
return ossStorageService.readObjectAsString(stripOssPointerPrefix(ossPointer));
return transientPayloadStorageService.resolvePayload(state.getStateJson(), "read task scope payload failed");
} catch (Exception ex) {
throw new BusinessException("读取任务范围载荷失败");
}
}
private void deleteStatePayloadIfNeeded(String value) {
String ossPointer = extractOssPointer(value);
if (ossPointer == null) {
return;
}
try {
ossStorageService.deleteObject(stripOssPointerPrefix(ossPointer));
} catch (Exception ignored) {
}
}
private String resolveParsedPayload(String value) {
if (isBlank(value) || !isOssPointer(value)) {
return value;
}
try {
return ossStorageService.readObjectAsString(stripOssPointerPrefix(value));
} catch (Exception ex) {
throw new BusinessException("读取任务解析载荷失败");
}
}
private void deleteParsedPayloadIfNeeded(String oldValue, String newValue) {
if (isBlank(oldValue) || oldValue.equals(newValue) || !isOssPointer(oldValue)) {
return;
}
try {
ossStorageService.deleteObject(stripOssPointerPrefix(oldValue));
} catch (Exception ignored) {
}
}
private void deleteReplacedStatePayloadIfNeeded(String oldValue, String newValue) {
if (isBlank(oldValue) || oldValue.equals(newValue)) {
return;
}
deleteStatePayloadIfNeeded(oldValue);
}
private void saveOssBackedScopePayload(Long taskId,
String moduleType,
String normalizedScopeKey,
String scopeHash,
String payloadJson,
LocalDateTime now,
TaskScopeStateEntity state) {
if (state == null) {
String stateJson = storeStatePayload(moduleType, taskId, scopeHash, payloadJson);
TaskScopeStateEntity entity = new TaskScopeStateEntity();
entity.setTaskId(taskId);
entity.setModuleType(moduleType);
entity.setScopeKey(normalizedScopeKey);
entity.setScopeHash(scopeHash);
entity.setStateJson(stateJson);
entity.setChunkTotal(0);
entity.setReceivedChunkCount(0);
entity.setCompleted(0);
entity.setLastChunkAt(now);
entity.setLastError(null);
entity.setCreatedAt(now);
entity.setUpdatedAt(now);
try {
taskScopeStateMapper.insert(entity);
deleteBufferedPayload(taskId, moduleType, scopeHash);
return;
} catch (DuplicateKeyException ignored) {
state = getScopeState(taskId, moduleType, scopeHash);
}
}
if (state == null) {
throw new BusinessException("刷新任务范围缓冲载荷失败");
}
writeBufferedPayload(taskId, moduleType, scopeHash, payloadJson);
if (!shouldFlushBufferedPayload(state, now)) {
return;
}
String stateJson = storeStatePayload(moduleType, taskId, scopeHash, payloadJson);
deleteReplacedStatePayloadIfNeeded(state.getStateJson(), stateJson);
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getId, state.getId())
.set(TaskScopeStateEntity::getScopeKey, normalizedScopeKey)
.set(TaskScopeStateEntity::getStateJson, stateJson)
.set(TaskScopeStateEntity::getLastChunkAt, now)
.set(TaskScopeStateEntity::getUpdatedAt, now));
deleteBufferedPayload(taskId, moduleType, scopeHash);
}
private String writeJson(Object value, String message) {
try {
return objectMapper.writeValueAsString(value);
@@ -457,76 +308,6 @@ public class TaskScopePayloadStorageService {
return value == null || value.isBlank();
}
private boolean shouldStoreStatePayloadInOss(String moduleType) {
return OSS_BACKED_STATE_MODULES.contains(normalize(moduleType).toUpperCase());
}
private boolean isOssPointer(String value) {
return !isBlank(value) && value.startsWith(OSS_POINTER_PREFIX);
}
private String extractOssPointer(String value) {
if (isBlank(value)) {
return null;
}
if (isOssPointer(value)) {
return value;
}
try {
String decoded = objectMapper.readValue(value, String.class);
return isOssPointer(decoded) ? decoded : null;
} catch (Exception ignored) {
return null;
}
}
private String stripOssPointerPrefix(String value) {
return value.substring(OSS_POINTER_PREFIX.length());
}
private boolean shouldFlushBufferedPayload(TaskScopeStateEntity state, LocalDateTime now) {
long flushIntervalMillis = Math.max(0L, taskPressureProperties.getScopePayloadFlushIntervalMillis());
if (flushIntervalMillis <= 0L || state.getUpdatedAt() == null) {
return true;
}
return java.time.Duration.between(state.getUpdatedAt(), now).toMillis() >= flushIntervalMillis;
}
private void writeBufferedPayload(Long taskId, String moduleType, String scopeHash, String payloadJson) {
try {
Path path = buildBufferedPayloadPath(taskId, moduleType, scopeHash);
Files.createDirectories(path.getParent());
Files.writeString(path, payloadJson, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE);
} catch (IOException ex) {
throw new BusinessException("写入任务范围缓冲载荷失败");
}
}
private String readBufferedPayload(Long taskId, String moduleType, String scopeHash) {
try {
Path path = buildBufferedPayloadPath(taskId, moduleType, scopeHash);
if (!Files.isRegularFile(path)) {
return null;
}
return Files.readString(path);
} catch (IOException ex) {
return null;
}
}
private void deleteBufferedPayload(Long taskId, String moduleType, String scopeHash) {
try {
Path path = buildBufferedPayloadPath(taskId, moduleType, scopeHash);
Files.deleteIfExists(path);
} catch (IOException ignored) {
}
}
private Path buildBufferedPayloadPath(Long taskId, String moduleType, String scopeHash) {
String normalizedModuleType = normalize(moduleType).toLowerCase();
return Path.of(System.getProperty("java.io.tmpdir"), LOCAL_BUFFER_DIR, normalizedModuleType, String.valueOf(taskId), scopeHash + ".json");
}
private String hash(String value) {
try {
MessageDigest digest = MessageDigest.getInstance("SHA-256");

149
backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TransientPayloadStorageService.java Normal file
View File

@@ -0,0 +1,149 @@
package com.nanri.aiimage.modules.task.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.TransientStorageProperties;
import com.nanri.aiimage.modules.file.service.object.RustfsObjectStorageService;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import java.util.Locale;
import java.util.UUID;
@Service
@RequiredArgsConstructor
public class TransientPayloadStorageService {
private static final String RUSTFS_POINTER_PREFIX = "rustfs:";
private static final String OSS_POINTER_PREFIX = "oss:";
private final TransientStorageProperties properties;
private final RustfsObjectStorageService rustfsObjectStorageService;
private final OssStorageService ossStorageService;
private final ObjectMapper objectMapper;
public boolean isWriteEnabled() {
return properties.isEnabled() && rustfsObjectStorageService.isConfigured();
}
public String storeScopePayload(String moduleType, Long taskId, String scopeHash, String content, boolean encodeAsJsonString) {
return store("task-scope", moduleType, taskId, scopeHash, "latest", content, encodeAsJsonString);
}
public String storeParsedPayload(String moduleType, Long taskId, String scopeHash, String content, boolean encodeAsJsonString) {
return store("task-parsed", moduleType, taskId, scopeHash, "latest", content, encodeAsJsonString);
}
public String storeResultPayload(String moduleType, Long taskId, String scopeHash, String submissionId, String content) {
return store("task-result-payload", moduleType, taskId, scopeHash, submissionId, content, true);
}
public String storeResultItemPayload(String moduleType, Long taskId, String scopeHash, String itemKey, String content) {
return store("task-result-item", moduleType, taskId, scopeHash, itemKey, content, true);
}
public String storeChunkPayload(String moduleType, Long taskId, String scopeHash, Integer chunkIndex, String content) {
String entryKey = chunkIndex == null ? UUID.randomUUID().toString() : "chunk-" + chunkIndex;
return store("task-chunk", moduleType, taskId, scopeHash, entryKey, content, true);
}
public String resolvePayload(String value, String errorMessage) {
String pointer = extractPointer(value);
if (pointer == null) {
return value;
}
try {
if (pointer.startsWith(RUSTFS_POINTER_PREFIX)) {
return rustfsObjectStorageService.readObjectAsString(pointer.substring(RUSTFS_POINTER_PREFIX.length()));
}
if (pointer.startsWith(OSS_POINTER_PREFIX)) {
return ossStorageService.readObjectAsString(pointer.substring(OSS_POINTER_PREFIX.length()));
}
} catch (Exception ex) {
throw new IllegalStateException(errorMessage, ex);
}
return value;
}
public void deletePayloadIfPresent(String value) {
String pointer = extractPointer(value);
if (pointer == null) {
return;
}
if (pointer.startsWith(RUSTFS_POINTER_PREFIX)) {
rustfsObjectStorageService.deleteObject(pointer.substring(RUSTFS_POINTER_PREFIX.length()));
return;
}
if (pointer.startsWith(OSS_POINTER_PREFIX)) {
ossStorageService.deleteObject(pointer.substring(OSS_POINTER_PREFIX.length()));
}
}
public void deleteReplacedPayloadIfNeeded(String oldValue, String newValue) {
String oldPointer = extractPointer(oldValue);
String newPointer = extractPointer(newValue);
if (oldPointer == null || oldPointer.equals(newPointer)) {
return;
}
deletePayloadIfPresent(oldPointer);
}
public String extractPointer(String value) {
if (value == null || value.isBlank()) {
return null;
}
if (isPointer(value)) {
return value;
}
try {
String decoded = objectMapper.readValue(value, String.class);
return isPointer(decoded) ? decoded : null;
} catch (Exception ignored) {
return null;
}
}
private boolean isPointer(String value) {
return value != null && (value.startsWith(RUSTFS_POINTER_PREFIX) || value.startsWith(OSS_POINTER_PREFIX));
}
private String store(String category,
String moduleType,
Long taskId,
String scopeHash,
String entryKey,
String content,
boolean encodeAsJsonString) {
if (!isWriteEnabled()) {
return content;
}
String objectKey = buildObjectKey(category, moduleType, taskId, scopeHash, entryKey);
String pointer = RUSTFS_POINTER_PREFIX + rustfsObjectStorageService.uploadText(objectKey, content);
if (!encodeAsJsonString) {
return pointer;
}
try {
return objectMapper.writeValueAsString(pointer);
} catch (Exception ex) {
throw new IllegalStateException("failed to encode transient storage pointer", ex);
}
}
private String buildObjectKey(String category, String moduleType, Long taskId, String scopeHash, String entryKey) {
String normalizedCategory = normalize(category, "unknown");
String normalizedModuleType = normalize(moduleType, "unknown");
String normalizedScopeHash = normalize(scopeHash, UUID.randomUUID().toString());
String normalizedEntryKey = normalize(entryKey, "latest");
return String.format("%s/%s/%s/%s/%s.json",
normalizedCategory,
normalizedModuleType,
taskId == null ? 0L : taskId,
normalizedScopeHash,
normalizedEntryKey);
}
private String normalize(String value, String fallback) {
String normalized = value == null ? "" : value.trim().toLowerCase(Locale.ROOT);
return normalized.isBlank() ? fallback : normalized.replaceAll("[^a-z0-9._-]+", "_");
}
}

61
backend-java/src/main/java/com/nanri/aiimage/modules/ziniao/service/ZiniaoSessionCacheService.java
View File

@@ -2,11 +2,11 @@ package com.nanri.aiimage.modules.ziniao.service;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.ZiniaoProperties;
import com.nanri.aiimage.modules.ziniao.model.cache.ZiniaoSessionCacheDto;
import com.nanri.aiimage.modules.ziniao.model.cache.ZiniaoShopCacheDto;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
@@ -15,6 +15,7 @@ import java.util.List;
@Service
@RequiredArgsConstructor
@Slf4j
public class ZiniaoSessionCacheService {
private final StringRedisTemplate stringRedisTemplate;
@@ -22,62 +23,96 @@ public class ZiniaoSessionCacheService {
private final ZiniaoProperties ziniaoProperties;
public void saveSession(ZiniaoSessionCacheDto session) {
if (session == null || session.getSessionId() == null || session.getSessionId().isBlank()) {
return;
}
try {
stringRedisTemplate.opsForValue().set(buildSessionKey(session.getSessionId()), objectMapper.writeValueAsString(session), Duration.ofHours(ziniaoProperties.getSessionTtlHours()));
stringRedisTemplate.opsForValue().set(
buildSessionKey(session.getSessionId()),
objectMapper.writeValueAsString(session),
Duration.ofHours(ziniaoProperties.getSessionTtlHours()));
} catch (Exception ex) {
throw new BusinessException("保存紫鸟会话失败");
log.warn("[ziniao-session-cache] save session degraded sessionId={} msg={}",
session.getSessionId(), ex.getMessage());
}
}
public ZiniaoSessionCacheDto getSession(String sessionId) {
String raw = stringRedisTemplate.opsForValue().get(buildSessionKey(sessionId));
String raw;
try {
raw = stringRedisTemplate.opsForValue().get(buildSessionKey(sessionId));
} catch (Exception ex) {
log.warn("[ziniao-session-cache] get session degraded sessionId={} msg={}", sessionId, ex.getMessage());
return null;
}
if (raw == null || raw.isBlank()) {
return null;
}
try {
return objectMapper.readValue(raw, ZiniaoSessionCacheDto.class);
} catch (Exception ex) {
throw new BusinessException("读取紫鸟会话失败");
log.warn("[ziniao-session-cache] parse session degraded sessionId={} msg={}", sessionId, ex.getMessage());
return null;
}
}
public void saveShops(String sessionId, List<ZiniaoShopCacheDto> shops) {
try {
stringRedisTemplate.opsForValue().set(buildShopsKey(sessionId), objectMapper.writeValueAsString(shops), Duration.ofMinutes(ziniaoProperties.getShopsCacheMinutes()));
stringRedisTemplate.opsForValue().set(
buildShopsKey(sessionId),
objectMapper.writeValueAsString(shops),
Duration.ofMinutes(ziniaoProperties.getShopsCacheMinutes()));
} catch (Exception ex) {
throw new BusinessException("保存紫鸟店铺列表失败");
log.warn("[ziniao-session-cache] save shops degraded sessionId={} msg={}", sessionId, ex.getMessage());
}
}
public List<ZiniaoShopCacheDto> getShops(String sessionId) {
String raw = stringRedisTemplate.opsForValue().get(buildShopsKey(sessionId));
String raw;
try {
raw = stringRedisTemplate.opsForValue().get(buildShopsKey(sessionId));
} catch (Exception ex) {
log.warn("[ziniao-session-cache] get shops degraded sessionId={} msg={}", sessionId, ex.getMessage());
return List.of();
}
if (raw == null || raw.isBlank()) {
return List.of();
}
try {
return objectMapper.readValue(raw, new TypeReference<List<ZiniaoShopCacheDto>>() {});
} catch (Exception ex) {
throw new BusinessException("读取紫鸟店铺列表失败");
log.warn("[ziniao-session-cache] parse shops degraded sessionId={} msg={}", sessionId, ex.getMessage());
return List.of();
}
}
public void saveCurrentShop(String sessionId, ZiniaoShopCacheDto shop) {
try {
stringRedisTemplate.opsForValue().set(buildCurrentShopKey(sessionId), objectMapper.writeValueAsString(shop), Duration.ofHours(ziniaoProperties.getSessionTtlHours()));
stringRedisTemplate.opsForValue().set(
buildCurrentShopKey(sessionId),
objectMapper.writeValueAsString(shop),
Duration.ofHours(ziniaoProperties.getSessionTtlHours()));
} catch (Exception ex) {
throw new BusinessException("保存当前紫鸟店铺失败");
log.warn("[ziniao-session-cache] save current shop degraded sessionId={} msg={}", sessionId, ex.getMessage());
}
}
public ZiniaoShopCacheDto getCurrentShop(String sessionId) {
String raw = stringRedisTemplate.opsForValue().get(buildCurrentShopKey(sessionId));
String raw;
try {
raw = stringRedisTemplate.opsForValue().get(buildCurrentShopKey(sessionId));
} catch (Exception ex) {
log.warn("[ziniao-session-cache] get current shop degraded sessionId={} msg={}", sessionId, ex.getMessage());
return null;
}
if (raw == null || raw.isBlank()) {
return null;
}
try {
return objectMapper.readValue(raw, ZiniaoShopCacheDto.class);
} catch (Exception ex) {
throw new BusinessException("读取当前紫鸟店铺失败");
log.warn("[ziniao-session-cache] parse current shop degraded sessionId={} msg={}", sessionId, ex.getMessage());
return null;
}
}

43
backend-java/src/main/java/com/nanri/aiimage/modules/ziniao/service/ZiniaoShopIndexService.java
View File

@@ -101,6 +101,8 @@ public class ZiniaoShopIndexService {
return pendingResult("店铺索引信息不完整，请等待后台刷新");
}
boolean treatAsFresh = isFresh(entry) || shouldBypassFreshnessBecauseWhitelistFailure(entry);
if (!buildOpenStoreUrl) {
ZiniaoShopMatchResultVo vo = new ZiniaoShopMatchResultVo();
vo.setMatched(true);
@@ -110,8 +112,8 @@ public class ZiniaoShopIndexService {
vo.setPlatform(entry.getPlatform());
vo.setMatchedUserId(entry.getMatchedUserId());
vo.setOpenStoreUrl(null);
vo.setMatchStatus(isFresh(entry) ? MATCH_STATUS_MATCHED : MATCH_STATUS_STALE);
vo.setMatchMessage(isFresh(entry) ? null : "店铺索引已过保鲜期，请等待后台刷新");
vo.setMatchStatus(treatAsFresh ? MATCH_STATUS_MATCHED : MATCH_STATUS_STALE);
vo.setMatchMessage(treatAsFresh ? null : "店铺索引已过保鲜期，请等待后台刷新");
return vo;
}
@@ -145,8 +147,8 @@ public class ZiniaoShopIndexService {
vo.setMatchStatus(MATCH_STATUS_STALE);
vo.setMatchMessage("打开店铺链接暂时无法生成，请稍后重试");
} else {
vo.setMatchStatus(isFresh(entry) ? MATCH_STATUS_MATCHED : MATCH_STATUS_STALE);
vo.setMatchMessage(isFresh(entry) ? null : "店铺索引已过保鲜期，请等待后台刷新");
vo.setMatchStatus(treatAsFresh ? MATCH_STATUS_MATCHED : MATCH_STATUS_STALE);
vo.setMatchMessage(treatAsFresh ? null : "店铺索引已过保鲜期，请等待后台刷新");
}
return vo;
}
@@ -406,7 +408,19 @@ public class ZiniaoShopIndexService {
private ZiniaoShopMatchResultVo staleOrPendingResult(String defaultMessage, ZiniaoShopIndexEntryDto entry) {
if (entry != null && STATUS_STALE.equals(entry.getStatus())) {
ZiniaoShopMatchResultVo vo = new ZiniaoShopMatchResultVo();
vo.setMatched(false);
boolean hasReusableMatch = entry.getShopId() != null
&& !entry.getShopId().isBlank()
&& entry.getMatchedUserId() != null
&& entry.getMatchedUserId() > 0;
vo.setMatched(hasReusableMatch);
if (hasReusableMatch) {
vo.setShopId(entry.getShopId());
vo.setShopName(entry.getShopName());
vo.setCompanyName(entry.getCompanyName());
vo.setPlatform(entry.getPlatform());
vo.setMatchedUserId(entry.getMatchedUserId());
vo.setOpenStoreUrl(null);
}
vo.setMatchStatus(MATCH_STATUS_STALE);
vo.setMatchMessage(defaultMessage);
return vo;
@@ -538,6 +552,25 @@ public class ZiniaoShopIndexService {
return Instant.now().toEpochMilli() - entry.getLastRefreshedAt() <= freshWindowMillis;
}
private boolean shouldBypassFreshnessBecauseWhitelistFailure(ZiniaoShopIndexEntryDto entry) {
if (entry == null || !STATUS_ACTIVE.equals(entry.getStatus())) {
return false;
}
if (entry.getLastRefreshedAt() == null || entry.getLastRefreshedAt() <= 0) {
return false;
}
ZiniaoShopIndexRefreshCursorDto cursor = getRefreshCursor();
if (cursor == null || !"FAILED".equals(cursor.getStatus())) {
return false;
}
String message = cursor.getMessage();
if (message == null || !message.contains("白名单")) {
return false;
}
Long lastFinishedAt = cursor.getLastFinishedAt();
return lastFinishedAt != null && lastFinishedAt >= entry.getLastRefreshedAt();
}
private Duration resolveEntryTtl() {
Integer ttlHours = ziniaoProperties.getShopIndexEntryTtlHours();
if (ttlHours == null || ttlHours <= 0) {

35
backend-java/src/main/resources/application-local.example.yml
View File

@@ -3,15 +3,22 @@
AIIMAGE_SERVER_PORT=18080
AIIMAGE_DB_URL=jdbc:mysql://159.75.121.33:3306/aiimage?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai&useSSL=false
AIIMAGE_DB_USERNAME=AIimage
AIIMAGE_DB_PASSWORD=WTFrb5y6hNLz6hNy
AIIMAGE_DB_URL=jdbc:mysql://127.0.0.1:3306/aiimage?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai&useSSL=false
AIIMAGE_DB_USERNAME=change-me
AIIMAGE_DB_PASSWORD=change-me
AIIMAGE_OSS_REGION=cn-hangzhou
AIIMAGE_OSS_ENDPOINT=oss-cn-hangzhou.aliyuncs.com
AIIMAGE_OSS_BUCKET=nanri-ai-images
AIIMAGE_OSS_ACCESS_KEY_ID=LTAI5tNpyvzMNz9f2dHarsm8
AIIMAGE_OSS_ACCESS_KEY_SECRET=bQSZnFH455i8tzyOgeahJmUzwmhynz
AIIMAGE_OSS_BUCKET=change-me
AIIMAGE_OSS_ACCESS_KEY_ID=change-me
AIIMAGE_OSS_ACCESS_KEY_SECRET=change-me
AIIMAGE_TRANSIENT_STORAGE_ENABLED=true
AIIMAGE_TRANSIENT_STORAGE_ENDPOINT=http://change-me
AIIMAGE_TRANSIENT_STORAGE_BUCKET=change-me
AIIMAGE_TRANSIENT_STORAGE_ACCESS_KEY_ID=change-me
AIIMAGE_TRANSIENT_STORAGE_ACCESS_KEY_SECRET=change-me
AIIMAGE_TRANSIENT_STORAGE_REGION=us-east-1
AIIMAGE_STORAGE_LOCAL_TEMP_DIR=F:/project/crawler-plugin/backend-java/data/tmp
@@ -19,6 +26,22 @@ AIIMAGE_MODULE_CLEANUP_ENABLED=true
AIIMAGE_MODULE_CLEANUP_CRON=0 0 0 * * *
AIIMAGE_MODULE_CLEANUP_MODULE_TYPES=DEDUPE,SPLIT,CONVERT,DELETE_BRAND
AIIMAGE_APPEARANCE_PATENT_COZE_BASE_URL=https://api.coze.cn
AIIMAGE_APPEARANCE_PATENT_COZE_WORKFLOW_PATH=/v1/workflow/run
AIIMAGE_APPEARANCE_PATENT_COZE_WORKFLOW_ID=7632683471312355338
AIIMAGE_APPEARANCE_PATENT_COZE_TOKEN=
AIIMAGE_APPEARANCE_PATENT_COZE_BATCH_SIZE=10
AIIMAGE_APPEARANCE_PATENT_STALE_TIMEOUT_MINUTES=20
AIIMAGE_ROCKETMQ_NAME_SERVER=121.196.149.225:9876
AIIMAGE_RESULT_FILE_JOB_MQ_ENABLED=true
AIIMAGE_RESULT_FILE_JOB_LOCAL_DISPATCH_ENABLED=true
AIIMAGE_RESULT_FILE_JOB_LOCAL_DISPATCH_POOL_SIZE=2
AIIMAGE_RESULT_FILE_JOB_LOCAL_DISPATCH_QUEUE_CAPACITY=200
AIIMAGE_RESULT_FILE_JOB_LOCAL_WORKER_ENABLED=false
AIIMAGE_RESULT_FILE_JOB_LOCAL_WORKER_DELAY_MS=10000
AIIMAGE_RESULT_FILE_JOB_STUCK_TIMEOUT_MINUTES=30
AIIMAGE_ZINIAO_ENABLED=false
AIIMAGE_ZINIAO_BASE_URL=https://sbappstoreapi.ziniao.com/openapi-router
AIIMAGE_ZINIAO_API_KEY=change-me

35
backend-java/src/main/resources/application.yml
View File

@@ -28,6 +28,12 @@ spring:
database: ${AIIMAGE_REDIS_DATABASE:0}
timeout: ${AIIMAGE_REDIS_TIMEOUT:5s}
rocketmq:
name-server: ${AIIMAGE_ROCKETMQ_NAME_SERVER:121.196.149.225:9876}
producer:
group: ${AIIMAGE_ROCKETMQ_PRODUCER_GROUP:aiimage-backend-producer}
send-message-timeout: ${AIIMAGE_ROCKETMQ_SEND_TIMEOUT_MS:3000}
management:
health:
db:
@@ -67,6 +73,13 @@ aiimage:
bucket: ${AIIMAGE_OSS_BUCKET:change-me}
access-key-id: ${AIIMAGE_OSS_ACCESS_KEY_ID:change-me}
access-key-secret: ${AIIMAGE_OSS_ACCESS_KEY_SECRET:change-me}
transient-storage:
enabled: ${AIIMAGE_TRANSIENT_STORAGE_ENABLED:false}
endpoint: ${AIIMAGE_TRANSIENT_STORAGE_ENDPOINT:}
bucket: ${AIIMAGE_TRANSIENT_STORAGE_BUCKET:}
access-key-id: ${AIIMAGE_TRANSIENT_STORAGE_ACCESS_KEY_ID:}
access-key-secret: ${AIIMAGE_TRANSIENT_STORAGE_ACCESS_KEY_SECRET:}
region: ${AIIMAGE_TRANSIENT_STORAGE_REGION:us-east-1}
storage:
local-temp-dir: ${AIIMAGE_STORAGE_LOCAL_TEMP_DIR:./data/tmp}
cleanup-enabled: ${AIIMAGE_STORAGE_CLEANUP_ENABLED:true}
@@ -96,6 +109,8 @@ aiimage:
enabled: ${AIIMAGE_MODULE_CLEANUP_ENABLED:true}
cron: ${AIIMAGE_MODULE_CLEANUP_CRON:0 0 0 * * *}
module-types: ${AIIMAGE_MODULE_CLEANUP_MODULE_TYPES:DEDUPE,SPLIT,CONVERT,DELETE_BRAND,PRODUCT_RISK_RESOLVE,PRICE_TRACK,SHOP_MATCH,PATROL_DELETE}
permission-schema-init:
enabled: ${AIIMAGE_PERMISSION_SCHEMA_INIT_ENABLED:false}
task-pressure:
local-task-entity-cache-millis: ${AIIMAGE_TASK_LOCAL_ENTITY_CACHE_MILLIS:3000}
db-select-batch-size: ${AIIMAGE_TASK_DB_SELECT_BATCH_SIZE:200}
@@ -103,6 +118,26 @@ aiimage:
scope-payload-buffer-retention-hours: ${AIIMAGE_TASK_SCOPE_PAYLOAD_BUFFER_RETENTION_HOURS:24}
scope-payload-recovery-max-files: ${AIIMAGE_TASK_SCOPE_PAYLOAD_RECOVERY_MAX_FILES:200}
scope-payload-cleanup-cron: ${AIIMAGE_TASK_SCOPE_PAYLOAD_CLEANUP_CRON:15 */30 * * * *}
result-file-job:
mq-enabled: ${AIIMAGE_RESULT_FILE_JOB_MQ_ENABLED:false}
topic: ${AIIMAGE_RESULT_FILE_JOB_TOPIC:aiimage-result-file-job}
consumer-group: ${AIIMAGE_RESULT_FILE_JOB_CONSUMER_GROUP:aiimage-result-file-job-consumer}
local-dispatch-enabled: ${AIIMAGE_RESULT_FILE_JOB_LOCAL_DISPATCH_ENABLED:true}
local-dispatch-pool-size: ${AIIMAGE_RESULT_FILE_JOB_LOCAL_DISPATCH_POOL_SIZE:2}
local-dispatch-queue-capacity: ${AIIMAGE_RESULT_FILE_JOB_LOCAL_DISPATCH_QUEUE_CAPACITY:200}
local-worker-enabled: ${AIIMAGE_RESULT_FILE_JOB_LOCAL_WORKER_ENABLED:true}
local-worker-delay-ms: ${AIIMAGE_RESULT_FILE_JOB_LOCAL_WORKER_DELAY_MS:10000}
stuck-scan-delay-ms: ${AIIMAGE_RESULT_FILE_JOB_STUCK_SCAN_DELAY_MS:60000}
stuck-timeout-minutes: ${AIIMAGE_RESULT_FILE_JOB_STUCK_TIMEOUT_MINUTES:30}
batch-size: ${AIIMAGE_RESULT_FILE_JOB_BATCH_SIZE:20}
appearance-patent:
coze-base-url: ${AIIMAGE_APPEARANCE_PATENT_COZE_BASE_URL:https://api.coze.cn}
coze-workflow-path: ${AIIMAGE_APPEARANCE_PATENT_COZE_WORKFLOW_PATH:/v1/workflow/run}
coze-workflow-id: ${AIIMAGE_APPEARANCE_PATENT_COZE_WORKFLOW_ID:7632683471312355338}
coze-token: ${AIIMAGE_APPEARANCE_PATENT_COZE_TOKEN:}
coze-batch-size: ${AIIMAGE_APPEARANCE_PATENT_COZE_BATCH_SIZE:10}
stale-timeout-minutes: ${AIIMAGE_APPEARANCE_PATENT_STALE_TIMEOUT_MINUTES:20}
stale-finalize-cron: ${AIIMAGE_APPEARANCE_PATENT_STALE_FINALIZE_CRON:0 */2 * * * *}
security:
shop-credential-key: ${AIIMAGE_SHOP_CREDENTIAL_KEY:change-me-shop-credential-key}
internal-token: ${AIIMAGE_INTERNAL_TOKEN:}

85
backend-java/src/main/resources/db/V34__appearance_patent.sql Normal file
View File

@@ -0,0 +1,85 @@
-- 外观专利检测模块复用通用任务表：
-- biz_file_task 保存任务主记录，module_type = APPEARANCE_PATENT
-- biz_file_result 保存最终 xlsx 结果文件和历史记录
-- biz_task_scope_state 保存解析载荷 OSS 指针、scope 状态和心跳辅助状态
-- biz_task_chunk 保存 Python 回传分片和 Java/Coze 处理后的分片结果
-- 兼容老库：如果通用任务表缺少 user_id，则补齐用户归属字段。
SET @biz_file_task_user_id_exists := (
SELECT COUNT(1)
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'biz_file_task'
AND COLUMN_NAME = 'user_id'
);
SET @sql_add_biz_file_task_user_id := IF(
@biz_file_task_user_id_exists = 0,
'ALTER TABLE biz_file_task ADD COLUMN user_id BIGINT NULL COMMENT ''用户ID'' AFTER created_by',
'SELECT 1'
);
PREPARE stmt_add_biz_file_task_user_id FROM @sql_add_biz_file_task_user_id;
EXECUTE stmt_add_biz_file_task_user_id;
DEALLOCATE PREPARE stmt_add_biz_file_task_user_id;
SET @biz_file_result_user_id_exists := (
SELECT COUNT(1)
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'biz_file_result'
AND COLUMN_NAME = 'user_id'
);
SET @sql_add_biz_file_result_user_id := IF(
@biz_file_result_user_id_exists = 0,
'ALTER TABLE biz_file_result ADD COLUMN user_id BIGINT NULL COMMENT ''用户ID'' AFTER error_message',
'SELECT 1'
);
PREPARE stmt_add_biz_file_result_user_id FROM @sql_add_biz_file_result_user_id;
EXECUTE stmt_add_biz_file_result_user_id;
DEALLOCATE PREPARE stmt_add_biz_file_result_user_id;
-- 外观专利检测常用查询索引：按用户查任务、按用户查历史。
SET @idx_file_task_module_user_created_exists := (
SELECT COUNT(1)
FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'biz_file_task'
AND INDEX_NAME = 'idx_biz_file_task_module_user_created'
);
SET @sql_add_idx_file_task_module_user_created := IF(
@idx_file_task_module_user_created_exists = 0,
'ALTER TABLE biz_file_task ADD INDEX idx_biz_file_task_module_user_created (module_type, user_id, created_at)',
'SELECT 1'
);
PREPARE stmt_add_idx_file_task_module_user_created FROM @sql_add_idx_file_task_module_user_created;
EXECUTE stmt_add_idx_file_task_module_user_created;
DEALLOCATE PREPARE stmt_add_idx_file_task_module_user_created;
SET @idx_file_result_module_user_created_exists := (
SELECT COUNT(1)
FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'biz_file_result'
AND INDEX_NAME = 'idx_biz_file_result_module_user_created'
);
SET @sql_add_idx_file_result_module_user_created := IF(
@idx_file_result_module_user_created_exists = 0,
'ALTER TABLE biz_file_result ADD INDEX idx_biz_file_result_module_user_created (module_type, user_id, created_at)',
'SELECT 1'
);
PREPARE stmt_add_idx_file_result_module_user_created FROM @sql_add_idx_file_result_module_user_created;
EXECUTE stmt_add_idx_file_result_module_user_created;
DEALLOCATE PREPARE stmt_add_idx_file_result_module_user_created;
-- 前端栏目权限兜底：外观专利检测属于前端工具下的页面。
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`)
SELECT '外观专利检测', 'appearance_patent', 'app', 'appearance-patent', 111
WHERE NOT EXISTS (
SELECT 1 FROM `columns` WHERE `column_key` = 'appearance_patent'
);
UPDATE `columns`
SET `name` = '外观专利检测',
`menu_type` = 'app',
`route_path` = 'appearance-patent',
`sort_order` = 111
WHERE `column_key` = 'appearance_patent';

74
backend-java/src/main/resources/db/V35__task_result_payload_and_file_job.sql Normal file
View File

@@ -0,0 +1,74 @@
CREATE TABLE IF NOT EXISTS biz_task_result_payload (
id BIGINT PRIMARY KEY AUTO_INCREMENT COMMENT 'primary key',
task_id BIGINT NOT NULL COMMENT 'task id',
module_type VARCHAR(32) NOT NULL COMMENT 'module type',
scope_key VARCHAR(1000) NOT NULL COMMENT 'scope identifier, such as shop name or file key',
scope_hash CHAR(64) NOT NULL COMMENT 'hash of scope key',
submission_id VARCHAR(128) NULL COMMENT 'idempotency key from submitter when available',
payload_json JSON NOT NULL COMMENT 'raw or normalized callback payload',
payload_hash CHAR(64) NOT NULL COMMENT 'sha256 of payload json',
version BIGINT NOT NULL DEFAULT 1 COMMENT 'scope payload version',
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'created time',
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'updated time',
UNIQUE KEY uk_task_scope_submission (task_id, module_type, scope_hash, submission_id),
KEY idx_task_module_scope (task_id, module_type, scope_hash),
KEY idx_task_module_updated (task_id, module_type, updated_at)
) COMMENT='task result callback payload detail';
CREATE TABLE IF NOT EXISTS biz_task_file_job (
id BIGINT PRIMARY KEY AUTO_INCREMENT COMMENT 'primary key',
task_id BIGINT NOT NULL COMMENT 'task id',
module_type VARCHAR(32) NOT NULL COMMENT 'module type',
result_id BIGINT NULL COMMENT 'biz_file_result id when job is scoped to one result row',
scope_key VARCHAR(1000) NULL COMMENT 'scope identifier',
job_type VARCHAR(32) NOT NULL DEFAULT 'ASSEMBLE_RESULT' COMMENT 'job type',
status VARCHAR(32) NOT NULL DEFAULT 'PENDING' COMMENT 'PENDING/RUNNING/SUCCESS/FAILED',
retry_count INT NOT NULL DEFAULT 0 COMMENT 'retry count',
error_message VARCHAR(1000) NULL COMMENT 'last error',
result_file_url VARCHAR(1000) NULL COMMENT 'generated file object key or url',
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'created time',
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'updated time',
finished_at DATETIME NULL COMMENT 'finished time',
UNIQUE KEY uk_file_job_scope (task_id, module_type, result_id, job_type),
KEY idx_file_job_status (status, updated_at),
KEY idx_file_job_task (task_id, module_type)
) COMMENT='async result file assembly job';
CREATE TABLE IF NOT EXISTS biz_task_result_item (
id BIGINT PRIMARY KEY AUTO_INCREMENT COMMENT 'primary key',
task_id BIGINT NOT NULL COMMENT 'task id',
module_type VARCHAR(32) NOT NULL COMMENT 'module type',
result_id BIGINT NULL COMMENT 'biz_file_result id when available',
scope_key VARCHAR(1000) NOT NULL COMMENT 'scope identifier, such as shop name or file key',
scope_hash CHAR(64) NOT NULL COMMENT 'hash of scope key',
item_key VARCHAR(512) NOT NULL COMMENT 'dedupe key, such as country + asin',
country_code VARCHAR(32) NULL COMMENT 'country code',
asin VARCHAR(128) NULL COMMENT 'asin or item id',
status VARCHAR(64) NULL COMMENT 'business status',
payload_json JSON NOT NULL COMMENT 'normalized row payload',
payload_hash CHAR(64) NOT NULL COMMENT 'sha256 of payload json',
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'created time',
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'updated time',
UNIQUE KEY uk_task_scope_item (task_id, module_type, scope_hash, item_key),
KEY idx_result_item_result (result_id),
KEY idx_result_item_task (task_id, module_type),
KEY idx_result_item_country_asin (task_id, module_type, country_code, asin)
) COMMENT='normalized task result item detail';
CREATE TABLE IF NOT EXISTS biz_task_progress_snapshot (
id BIGINT PRIMARY KEY AUTO_INCREMENT COMMENT 'primary key',
task_id BIGINT NOT NULL COMMENT 'task id',
module_type VARCHAR(32) NOT NULL COMMENT 'module type',
status VARCHAR(32) NOT NULL COMMENT 'task status',
total_count INT NOT NULL DEFAULT 0 COMMENT 'total item count',
success_count INT NOT NULL DEFAULT 0 COMMENT 'success count',
failed_count INT NOT NULL DEFAULT 0 COMMENT 'failed count',
pending_count INT NOT NULL DEFAULT 0 COMMENT 'pending count',
current_scope_key VARCHAR(1000) NULL COMMENT 'current scope identifier',
message VARCHAR(1000) NULL COMMENT 'progress message',
snapshot_json JSON NULL COMMENT 'small progress snapshot',
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'created time',
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'updated time',
UNIQUE KEY uk_task_progress_snapshot (task_id, module_type),
KEY idx_progress_module_status (module_type, status, updated_at)
) COMMENT='lightweight task progress snapshot';

12
frontend-vue/appearance-patent.html Normal file
View File

@@ -0,0 +1,12 @@
<!doctype html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>外观专利检测</title>
</head>
<body>
<div id="app"></div>
<script type="module" src="/src/appearance-patent-main.ts"></script>
</body>
</html>

7
frontend-vue/src/appearance-patent-main.ts Normal file
View File

@@ -0,0 +1,7 @@
import { createApp } from 'vue'
import ElementPlus from 'element-plus'
import 'element-plus/dist/index.css'
import '@/styles/main.css'
import BrandAppearancePatentTab from '@/pages/brand/components/BrandAppearancePatentTab.vue'
createApp(BrandAppearancePatentTab).use(ElementPlus).mount('#app')

487
frontend-vue/src/pages/brand/components/BrandAppearancePatentTab.vue Normal file
View File

@@ -0,0 +1,487 @@
<template>
<div class="page-shell module-page">
<BrandTopBar active="appearance-patent" />
<div class="main-content">
<aside class="left-panel">
<div class="section-title">上传文件</div>
<div class="upload-zone">
<div class="hint">选择 Excel 后点击解析，后端会提取 id、ASIN、国家，并保留整数 id 与子数据的第一条。</div>
<div class="btns">
<button type="button" class="opt-btn" @click="selectFiles">选择 Excel</button>
</div>
<div class="selected-files">
<span v-if="!selectedFileNames.length">暂未选择文件</span>
<span v-for="name in selectedFileNames" v-else :key="name">{{ name }}</span>
</div>
</div>
<div class="prompt-card">
<div class="section-title">AI 提示词</div>
<textarea v-model="aiPrompt" class="prompt-input" rows="8" placeholder="可选，留空时使用下方默认提示词" />
<div class="prompt-default-label">留空时默认使用以下提示词</div>
<div class="prompt-preview">{{ defaultAiPrompt }}</div>
</div>
<div class="run-row">
<button type="button" class="btn-run" :disabled="parsing || !uploadedFiles.length" @click="parseFiles">
{{ parsing ? '解析中...' : '解析并创建任务' }}
</button>
<button type="button" class="btn-run btn-queue" :disabled="pushing || !parsedRows.length" @click="pushToPythonQueue">
{{ pushing ? '推送中...' : '推送到 Python 队列' }}
</button>
</div>
<p class="loading-msg">Python 回传字段按 asin、国家、url、标题提交；后端接收分片后按 10 条一批调用 Coze。</p>
<div v-if="parseResult" class="parse-card">
<div>任务 ID：{{ parseResult.taskId }}</div>
<div>总行数：{{ parseResult.totalRows }}，保留：{{ parseResult.acceptedRows }}，过滤：{{ parseResult.droppedRows }}</div>
</div>
<pre v-if="queuePayloadText" class="queue-payload">{{ queuePayloadText }}</pre>
</aside>
<section class="right-panel">
<div class="panel-header">外观专利检测</div>
<div class="task-list-wrap">
<div class="clean-result-summary">
<div class="summary-card">
<span class="summary-label">运行中任务</span>
<strong>{{ dashboard.pendingTaskCount }}</strong>
</div>
<div class="summary-card">
<span class="summary-label">已结束任务</span>
<strong>{{ dashboard.processedTaskCount }}</strong>
</div>
<div class="summary-card">
<span class="summary-label">成功任务</span>
<strong>{{ dashboard.successTaskCount }}</strong>
</div>
<div class="summary-card">
<span class="summary-label">失败任务</span>
<strong>{{ dashboard.failedTaskCount }}</strong>
</div>
</div>
<div class="subsection-title">匹配任务</div>
<div class="result-list-wrap preview-wrap">
<div class="result-list-header">
<span>解析结果</span>
<span v-if="parsedRows.length" class="muted">显示 1 条样例 / 共 {{ parsedRows.length }} 条</span>
</div>
<div v-if="!parsedRows.length" class="empty-tasks">暂无解析结果</div>
<el-table v-else :data="previewRows" height="120" class="result-table">
<el-table-column prop="displayId" label="ID" width="90" />
<el-table-column prop="asin" label="ASIN" width="130" />
<el-table-column prop="country" label="国家" width="100" />
<el-table-column prop="title" label="标题" min-width="160" show-overflow-tooltip />
<el-table-column prop="url" label="URL" min-width="160" show-overflow-tooltip />
</el-table>
</div>
<div class="result-list-wrap">
<div class="result-list-header">
<span>当前任务</span>
</div>
<div v-if="!currentItems.length" class="empty-tasks">暂无当前任务</div>
<ul v-else class="task-list clean-result-list">
<li v-for="item in currentItems" :key="`cur-${item.taskId}`" class="task-item">
<div class="left">
<span class="id">{{ item.sourceFilename || '外观专利检测' }}</span>
<div class="files">任务 ID：{{ item.taskId }}</div>
<div class="files">行数：{{ item.rowCount ?? '-' }}</div>
</div>
<div class="task-right">
<span class="status running">{{ statusText(item) }}</span>
<button type="button" class="btn-delete" @click="deleteTaskRecord(item)">删除</button>
</div>
</li>
</ul>
</div>
<div class="result-list-wrap">
<div class="result-list-header">
<span>历史记录</span>
</div>
<div v-if="!historyOnlyItems.length" class="empty-tasks">暂无历史记录</div>
<ul v-else class="task-list clean-result-list">
<li v-for="item in historyOnlyItems" :key="`his-${item.resultId}-${item.taskId}`" class="task-item">
<div class="left">
<span class="id">{{ item.sourceFilename || '外观专利检测' }}</span>
<div class="files">任务 ID：{{ item.taskId ?? '-' }}</div>
<div v-if="item.resultFilename" class="files">
{{ item.resultFilename || '下载结果' }}
</div>
<div v-if="item.error" class="files">错误：{{ item.error }}</div>
</div>
<div class="task-right">
<span class="status" :class="item.success ? 'success' : 'failed'">{{ statusText(item) }}</span>
<button v-if="canDownload(item)" type="button" class="download" @click="downloadResult(item)">下载</button>
<button v-if="item.resultId" type="button" class="btn-delete" @click="deleteTaskRecord(item)">删除</button>
</div>
</li>
</ul>
</div>
</div>
</section>
</div>
</div>
</template>
<script setup lang="ts">
import { computed, onMounted, onUnmounted, ref } from 'vue'
import { ElMessage } from 'element-plus'
import BrandTopBar from '@/pages/brand/components/BrandTopBar.vue'
import {
activateAppearancePatentTask,
deleteAppearancePatentHistory,
deleteAppearancePatentTask,
getAppearancePatentDashboard,
getAppearancePatentHistory,
getAppearancePatentResultDownloadUrl,
getAppearancePatentTaskProgressBatch,
parseAppearancePatent,
type AppearancePatentDashboardVo,
type AppearancePatentHistoryItem,
type AppearancePatentParsedRow,
type AppearancePatentParseVo,
type UploadedFileRef,
type UploadFileVo,
} from '@/shared/api/java-modules'
import { getPywebviewApi } from '@/shared/bridges/pywebview'
import { getTaskPollIntervalMs } from '@/shared/task-progress-config'
const selectedFileNames = ref<string[]>([])
const uploadedFiles = ref<UploadFileVo[]>([])
const parseResult = ref<AppearancePatentParseVo | null>(null)
const parsedRows = ref<AppearancePatentParsedRow[]>([])
const defaultAiPrompt = `请帮我排查以下亚马逊商品在英国及欧洲地区是否存在知识产权侵权风险。
请直接给出明确结论（有侵权风险 或 未发现明显侵权风险），并严格按照以下三个维度提供精简的排查理由：
外观维度： 评估产品外形、图案设计是否与欧洲/英国常见外观专利雷同。
专利维度： 评估产品的核心技术或物理结构是否存在侵权可能。
标题维度： 排查标题中是否包含大牌商标、敏感词或版权保护词汇。
结论：XX`
const aiPrompt = ref('')
const parsing = ref(false)
const pushing = ref(false)
const queuePayloadText = ref('')
const pollingTaskIds = ref<number[]>([])
const pollTimer = ref<number | null>(null)
const pollingInFlight = ref(false)
const dashboard = ref<AppearancePatentDashboardVo>({
pendingTaskCount: 0,
processedTaskCount: 0,
successTaskCount: 0,
failedTaskCount: 0,
})
const historyItems = ref<AppearancePatentHistoryItem[]>([])
const previewRows = computed(() => parsedRows.value.slice(0, 1))
const currentItems = computed(() => historyItems.value.filter((i) => i.taskId != null && pollingTaskIds.value.includes(i.taskId)))
const historyOnlyItems = computed(() => historyItems.value.filter((i) => (i.taskStatus || '') !== 'RUNNING'))
function effectiveAiPrompt() {
return aiPrompt.value.trim() || defaultAiPrompt
}
function uidForStorage() {
return typeof window !== 'undefined' ? window.localStorage.getItem('uid') || '0' : '0'
}
function pollingKey() {
return `appearance-patent:tasks:${uidForStorage()}`
}
function savePollingIds() {
if (typeof window === 'undefined') return
if (!pollingTaskIds.value.length) window.localStorage.removeItem(pollingKey())
else window.localStorage.setItem(pollingKey(), JSON.stringify(pollingTaskIds.value))
}
function clearPollingIds() {
pollingTaskIds.value = []
savePollingIds()
stopPolling()
}
async function selectFiles() {
const api = getPywebviewApi()
if (!api?.select_brand_xlsx_files || !api.upload_file_to_java) {
ElMessage.warning('当前环境不支持文件选择或上传')
return
}
const paths = await api.select_brand_xlsx_files()
if (!paths?.length) return
const files: UploadFileVo[] = []
for (const path of paths) {
const uploaded = await api.upload_file_to_java(path)
if (!uploaded?.success || !uploaded.data) {
throw new Error(uploaded?.error || `上传失败：${path}`)
}
files.push(uploaded.data)
}
uploadedFiles.value = files
selectedFileNames.value = files.map((f) => f.originalFilename || f.fileKey)
parseResult.value = null
parsedRows.value = []
queuePayloadText.value = ''
}
async function parseFiles() {
if (!uploadedFiles.value.length) {
ElMessage.warning('请先选择 Excel')
return
}
parsing.value = true
try {
const files: UploadedFileRef[] = uploadedFiles.value.map((f) => ({
fileKey: f.fileKey,
originalFilename: f.originalFilename,
relativePath: f.relativePath,
}))
const res = await parseAppearancePatent(files, effectiveAiPrompt())
parseResult.value = res
parsedRows.value = res.items || []
queuePayloadText.value = ''
await loadDashboard()
await loadHistory()
ElMessage.success(`解析完成，保留 ${res.acceptedRows} 条`)
} catch (e) {
ElMessage.error(e instanceof Error ? e.message : '解析失败')
} finally {
parsing.value = false
}
}
async function pushToPythonQueue() {
const api = getPywebviewApi()
const taskId = parseResult.value?.taskId
if (!taskId || !parsedRows.value.length) {
ElMessage.warning('请先解析文件')
return
}
if (!api?.enqueue_json) {
ElMessage.error('当前环境未启用 pywebview enqueue_json')
return
}
pushing.value = true
try {
const payload = {
type: 'appearance-patent-run',
ts: Date.now(),
data: {
taskId,
prompt: parseResult.value?.aiPrompt || effectiveAiPrompt(),
rows: parsedRows.value.map((row) => ({
id: row.displayId,
asin: row.asin,
country: row.country,
url: row.url || '',
title: row.title || '',
})),
},
}
queuePayloadText.value = JSON.stringify(payload, null, 2)
const result = await api.enqueue_json(payload)
if (!result?.success) {
ElMessage.error(result?.error || '推送失败')
return
}
await activateAppearancePatentTask(taskId)
addPollingTask(taskId)
await loadDashboard()
await loadHistory()
ElMessage.success('已推送到 Python 队列')
} finally {
pushing.value = false
}
}
function addPollingTask(taskId: number) {
if (!pollingTaskIds.value.includes(taskId)) {
pollingTaskIds.value = [...pollingTaskIds.value, taskId]
savePollingIds()
}
ensurePolling()
}
function removePollingTask(taskId: number) {
pollingTaskIds.value = pollingTaskIds.value.filter((id) => id !== taskId)
savePollingIds()
}
function ensurePolling() {
if (pollTimer.value != null) return
scheduleNextPoll(true)
}
function stopPolling() {
if (pollTimer.value != null) {
window.clearTimeout(pollTimer.value)
pollTimer.value = null
}
}
function scheduleNextPoll(immediate = false) {
if (pollTimer.value != null) {
if (!immediate) return
window.clearTimeout(pollTimer.value)
pollTimer.value = null
}
const run = async () => {
pollTimer.value = null
if (!pollingTaskIds.value.length) return
await refreshTaskProgress()
if (pollingTaskIds.value.length) {
pollTimer.value = window.setTimeout(run, getTaskPollIntervalMs())
}
}
if (immediate) void run()
else pollTimer.value = window.setTimeout(run, getTaskPollIntervalMs())
}
async function refreshTaskProgress() {
if (pollingInFlight.value || !pollingTaskIds.value.length) {
if (!pollingTaskIds.value.length) stopPolling()
return
}
pollingInFlight.value = true
try {
const batch = await getAppearancePatentTaskProgressBatch(pollingTaskIds.value)
let hasTerminalTask = false
for (const detail of batch.items || []) {
const task = detail.task
if (!task?.id) continue
if (task.status === 'SUCCESS' || task.status === 'FAILED') {
removePollingTask(task.id)
hasTerminalTask = true
}
}
if (batch.missingTaskIds?.length) {
batch.missingTaskIds.forEach((taskId) => removePollingTask(taskId))
}
if (hasTerminalTask) {
await loadDashboard()
await loadHistory()
}
} finally {
pollingInFlight.value = false
}
}
async function loadDashboard() {
dashboard.value = await getAppearancePatentDashboard()
}
async function loadHistory() {
const res = await getAppearancePatentHistory()
historyItems.value = res.items || []
}
function statusText(item: AppearancePatentHistoryItem) {
const status = item.taskStatus || ''
if (status === 'RUNNING') return '执行中'
if (status === 'SUCCESS' || item.success) return '已完成'
if (status === 'FAILED') return '失败'
return '等待中'
}
function canDownload(item: AppearancePatentHistoryItem) {
return Boolean(item.resultId && (item.fileReady || item.downloadUrl))
}
async function downloadResult(item: AppearancePatentHistoryItem) {
if (!item.resultId) return
const api = getPywebviewApi()
if (!api?.save_file_from_url_new) {
ElMessage.error('当前客户端未提供下载能力')
return
}
const url = item.downloadUrl || getAppearancePatentResultDownloadUrl(item.resultId)
const filename = item.resultFilename || `${item.sourceFilename || 'appearance-patent'}.xlsx`
const result = await api.save_file_from_url_new(url, filename)
if (result.success) {
ElMessage.success(`已保存: ${result.path || filename}`)
} else if (result.error && result.error !== '用户取消') {
ElMessage.error(result.error)
}
}
async function deleteTaskRecord(item: AppearancePatentHistoryItem) {
try {
if (item.taskStatus === 'RUNNING' && item.taskId) {
await deleteAppearancePatentTask(item.taskId)
removePollingTask(item.taskId)
} else if (item.resultId) {
await deleteAppearancePatentHistory(item.resultId)
}
await loadDashboard()
await loadHistory()
ElMessage.success('已删除')
} catch (e) {
ElMessage.error(e instanceof Error ? e.message : '删除失败')
}
}
onMounted(async () => {
clearPollingIds()
await Promise.all([
loadDashboard().catch(() => undefined),
loadHistory().catch(() => undefined),
])
})
onUnmounted(() => stopPolling())
</script>
<style scoped>
.module-page { min-height: 100vh; background: #1a1a1a; }
.main-content { display: flex; height: calc(100vh - 56px); min-height: calc(100vh - 56px); }
.left-panel { width: 400px; background: #1e1e1e; padding: 20px; overflow-y: auto; border-right: 1px solid #2a2a2a; }
.right-panel { flex: 1; min-width: 0; background: #1a1a1a; display: flex; flex-direction: column; }
.section-title, .subsection-title { font-size: 13px; color: #bbb; margin-bottom: 10px; }
.upload-zone { border: 1px dashed #3a3a3a; border-radius: 10px; padding: 18px; background: #252525; margin-bottom: 18px; }
.hint, .loading-msg, .files, .muted { color: #888; font-size: 12px; line-height: 1.5; }
.link { color: #6ea8fe; text-decoration: none; }
.link:hover { color: #9fc5ff; }
.btns, .run-row { display: flex; gap: 10px; flex-wrap: wrap; }
.opt-btn, .btn-run, .btn-delete { border: none; cursor: pointer; border-radius: 7px; }
.opt-btn { padding: 8px 14px; color: #ccc; background: #2a2a2a; border: 1px solid #3a3a3a; }
.btn-run { padding: 10px 18px; color: #fff; background: #3498db; font-weight: 600; }
.btn-queue { background: #27ae60; }
.btn-run:disabled { opacity: .55; cursor: not-allowed; }
.selected-files { margin-top: 14px; color: #999; font-size: 12px; word-break: break-all; }
.selected-files span { display: block; margin: 4px 0; }
.prompt-card { margin-bottom: 18px; }
.prompt-input { width: 100%; box-sizing: border-box; resize: vertical; min-height: 180px; padding: 10px 12px; border: 1px solid #333; border-radius: 8px; background: #202020; color: #d8d8d8; font-size: 12px; line-height: 1.6; outline: none; }
.prompt-input:focus { border-color: #3498db; }
.prompt-default-label { margin-top: 10px; color: #8d8d8d; font-size: 12px; }
.prompt-preview { margin-top: 10px; padding: 12px; border: 1px solid #2a2a2a; border-radius: 8px; background: #202020; color: #9ea7b3; font-size: 12px; line-height: 1.6; white-space: pre-wrap; }
.parse-card, .queue-payload { margin-top: 14px; padding: 12px; border: 1px solid #2a2a2a; border-radius: 8px; background: #202020; color: #b8c1cc; font-size: 12px; }
.queue-payload { max-height: 220px; overflow: auto; color: #8fd3ff; white-space: pre-wrap; }
.panel-header { padding: 16px 20px; border-bottom: 1px solid #2a2a2a; font-size: 15px; font-weight: 600; color: #ddd; }
.task-list-wrap { flex: 1; padding: 16px 20px; overflow: auto; }
.clean-result-summary { display: grid; grid-template-columns: repeat(4, minmax(0, 1fr)); gap: 12px; margin-bottom: 16px; }
.summary-card { padding: 14px 16px; border: 1px solid #2a2a2a; border-radius: 8px; background: #1e1e1e; }
.summary-card strong { display: block; margin-top: 8px; color: #eaf4ff; font-size: 22px; }
.summary-label { color: #8d8d8d; font-size: 12px; }
.result-list-wrap { border: 1px solid #2a2a2a; border-radius: 8px; background: #1e1e1e; min-height: 180px; margin: 0 0 16px; }
.result-list-header { display: flex; justify-content: space-between; padding: 12px 16px; border-bottom: 1px solid #2a2a2a; color: #ddd; font-size: 14px; }
.empty-tasks { color: #666; font-size: 13px; padding: 18px; text-align: center; }
.result-table { --el-table-bg-color: #222; --el-table-tr-bg-color: #222; --el-table-header-bg-color: #2a2a2a; --el-table-text-color: #ccc; --el-table-border-color: #333; }
.task-list { list-style: none; margin: 0; padding: 12px; }
.task-item { display: flex; align-items: flex-start; justify-content: space-between; gap: 12px; padding: 12px 14px; border: 1px solid #2a2a2a; border-radius: 8px; margin-bottom: 8px; background: #222; }
.left { flex: 1; min-width: 0; }
.id { color: #e0e0e0; font-size: 13px; font-weight: 600; }
.task-right { display: flex; gap: 8px; align-items: center; flex-wrap: wrap; }
.status { padding: 4px 10px; border-radius: 6px; font-size: 12px; }
.status.success { background: rgba(46, 204, 113, .18); color: #2ecc71; }
.status.failed { background: rgba(231, 76, 60, .18); color: #ff6b6b; }
.status.running { background: rgba(52, 152, 219, .18); color: #3498db; }
.btn-delete { padding: 6px 10px; color: #ff8f8f; background: rgba(231, 76, 60, .12); }
@media (max-width: 1100px) {
.main-content { flex-direction: column; height: auto; }
.left-panel { width: 100%; border-right: none; border-bottom: 1px solid #2a2a2a; }
.clean-result-summary { grid-template-columns: repeat(2, minmax(0, 1fr)); }
}
</style>

8
frontend-vue/src/pages/brand/components/BrandConvertTab.vue
View File

@@ -151,7 +151,7 @@
<span class="status" :class="item.success ? 'success' : 'failed'">
{{ item.success ? '已完成' : '失败' }}
</span>
<button v-if="item.success && item.downloadUrl" type="button" class="download"
<button v-if="item.success && (item.downloadUrl || item.resultId)" type="button" class="download"
@click="downloadConvertResult(item)">
下载压缩包
</button>
@@ -179,6 +179,7 @@ import {
deleteConvertHistory,
deleteConvertTemplate,
getConvertHistory,
getConvertResultDownloadUrl,
getConvertTemplates,
importConvertTemplate,
runConvert,
@@ -472,14 +473,15 @@ async function deleteConvertHistoryRecord(resultId: number) {
async function downloadConvertResult(item: ConvertResultItem) {
const api = getPywebviewApi()
if (!item.downloadUrl) {
const url = item.downloadUrl || (item.resultId ? getConvertResultDownloadUrl(item.resultId) : '')
if (!url) {
ElMessage.warning('当前结果没有下载地址')
return
}
const filename = item.outputFilename || `${new Date().toISOString().slice(0, 10).replace(/-/g, '')}.txt`
if (api?.save_file_from_url_new) {
const result = await api.save_file_from_url_new(item.downloadUrl, filename)
const result = await api.save_file_from_url_new(url, filename)
if (result.success) {
ElMessage.success(`已保存：${result.path || filename}`)
} else if (result.error && result.error !== '用户取消') {

Some files were not shown because too many files have changed in this diff Show More