super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

提交优化更新

Browse Source
This commit is contained in:
super
2026-05-29 16:49:18 +08:00
parent 2ed1250604
commit 225d13fb6e
36 changed files with 370 additions and 220 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
backend-java/src/main/java/com/nanri/aiimage/modules/collectdata/model/dto/CollectDataFiltersDto.java
View File

@@ -1,6 +1,7 @@
package com.nanri.aiimage.modules.collectdata.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import com.fasterxml.jackson.annotation.JsonAlias;
import lombok.Data;
import java.math.BigDecimal;
@@ -14,8 +15,16 @@ public class CollectDataFiltersDto {
@Schema(description = "金额（手动输入），可为空", example = "59.99")
private BigDecimal amount;
@Schema(description = "排名（手动输入），可为空", example = "100000")
private Integer rank;
@JsonAlias({"min_amount"})
@Schema(description = "最小金额，可为空", example = "10.00")
private BigDecimal minAmount;
@JsonAlias({"max_amount"})
@Schema(description = "最大金额，可为空", example = "59.99")
private BigDecimal maxAmount;
@Schema(description = "是否启用排名条件")
private Boolean rank;
@Schema(description = "是否包含 FBA 商品")
private Boolean fba;
@@ -25,4 +34,8 @@ public class CollectDataFiltersDto {
@Schema(description = "国家代码列表，按用户拖拽顺序保存", example = "[\"DE\",\"UK\"]")
private List<String> countryCodes = new ArrayList<>();
@JsonAlias({"country_code"})
@Schema(description = "单选国家代码，采集数据页面当前只选择一个国家", example = "DE")
private String countryCode;
}

47
backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/client/SimilarAsinCozeClient.java
View File

@@ -42,6 +42,10 @@ public class SimilarAsinCozeClient {
private volatile RestClient sharedRestClient;
public List<SimilarAsinResultRowDto> inspect(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey) {
return inspect(rows, prompt, apiKey, false);
}
public List<SimilarAsinResultRowDto> inspect(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch) {
if (rows == null || rows.isEmpty()) {
return List.of();
}
@@ -50,7 +54,7 @@ public class SimilarAsinCozeClient {
return rows.stream().map(this::copy).toList();
}
try {
return inspectWithFallback(rows, prompt, apiKey);
return inspectWithFallback(rows, prompt, apiKey, imgSwitch);
} catch (Exception ex) {
String failureMessage = failureMessage(ex);
log.warn("[similar-asin] coze batch failed size={} err={}", rows.size(), failureMessage);
@@ -59,15 +63,20 @@ public class SimilarAsinCozeClient {
}
public CozeSubmitResponse submitWorkflow(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey) throws Exception {
return submitWorkflow(rows, prompt, apiKey, nextCredential());
return submitWorkflow(rows, prompt, apiKey, false, nextCredential());
}
public CozeSubmitResponse submitWorkflow(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch) throws Exception {
return submitWorkflow(rows, prompt, apiKey, imgSwitch, nextCredential());
}
public CozeSubmitResponse submitWorkflow(List<SimilarAsinResultRowDto> rows,
String prompt,
String apiKey,
boolean imgSwitch,
CozeCredentialRef credential) throws Exception {
CozeCredentialRef resolvedCredential = resolveCredential(credential);
JsonNode submitRoot = objectMapper.readTree(postWorkflow(rows, prompt, apiKey, resolvedCredential));
JsonNode submitRoot = objectMapper.readTree(postWorkflow(rows, prompt, apiKey, imgSwitch, resolvedCredential));
ensureSuccess(submitRoot);
return new CozeSubmitResponse(
extractExecuteId(submitRoot),
@@ -109,12 +118,12 @@ public class SimilarAsinCozeClient {
return rows.stream().map(this::copy).map(row -> markFailed(row, failureMessage)).toList();
}
private List<SimilarAsinResultRowDto> inspectWithFallback(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey) {
private List<SimilarAsinResultRowDto> inspectWithFallback(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch) {
try {
if (rows.size() == 1) {
return inspectSingleRowWithRetry(rows, prompt, apiKey);
return inspectSingleRowWithRetry(rows, prompt, apiKey, imgSwitch);
}
InspectAttempt attempt = inspectOnce(rows, prompt, apiKey);
InspectAttempt attempt = inspectOnce(rows, prompt, apiKey, imgSwitch);
if (attempt.resolvedCount() < rows.size()) {
throw new PartialCozeResultException(attempt.resolvedCount(), rows.size(), attempt.rawResultCount());
}
@@ -125,17 +134,17 @@ public class SimilarAsinCozeClient {
log.warn("[similar-asin] coze batch fallback split size={} left={} right={} err={}",
rows.size(), middle, rows.size() - middle, failureMessage(ex));
List<SimilarAsinResultRowDto> merged = new ArrayList<>(rows.size());
merged.addAll(inspectPartitionWithFailureFallback(rows.subList(0, middle), prompt, apiKey));
merged.addAll(inspectPartitionWithFailureFallback(rows.subList(middle, rows.size()), prompt, apiKey));
merged.addAll(inspectPartitionWithFailureFallback(rows.subList(0, middle), prompt, apiKey, imgSwitch));
merged.addAll(inspectPartitionWithFailureFallback(rows.subList(middle, rows.size()), prompt, apiKey, imgSwitch));
return merged;
}
throw propagate(ex);
}
}
private List<SimilarAsinResultRowDto> inspectPartitionWithFailureFallback(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey) {
private List<SimilarAsinResultRowDto> inspectPartitionWithFailureFallback(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch) {
try {
return inspectWithFallback(rows, prompt, apiKey);
return inspectWithFallback(rows, prompt, apiKey, imgSwitch);
} catch (Exception ex) {
String failureMessage = failureMessage(ex);
log.warn("[similar-asin] coze partition failed size={} err={}", rows.size(), failureMessage);
@@ -143,12 +152,12 @@ public class SimilarAsinCozeClient {
}
}
private List<SimilarAsinResultRowDto> inspectSingleRowWithRetry(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey) throws Exception {
private List<SimilarAsinResultRowDto> inspectSingleRowWithRetry(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch) throws Exception {
SimilarAsinResultRowDto row = rows.getFirst();
PartialCozeResultException lastFailure = null;
for (int attemptIndex = 1; attemptIndex <= 3; attemptIndex++) {
try {
InspectAttempt attempt = inspectOnce(rows, prompt, apiKey);
InspectAttempt attempt = inspectOnce(rows, prompt, apiKey, imgSwitch);
if (attempt.resolvedCount() == rows.size()) {
return attempt.mergedRows();
}
@@ -179,16 +188,16 @@ public class SimilarAsinCozeClient {
throw lastFailure == null ? new PartialCozeResultException(0, rows.size(), 0) : lastFailure;
}
private InspectAttempt inspectOnce(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey) throws Exception {
String raw = runWorkflowAsyncAndWait(rows, prompt, apiKey);
private InspectAttempt inspectOnce(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch) throws Exception {
String raw = runWorkflowAsyncAndWait(rows, prompt, apiKey, imgSwitch);
List<CozeResult> results = parseResults(raw);
List<SimilarAsinResultRowDto> merged = mergeRows(rows, results);
return new InspectAttempt(raw, merged, resolvedCount(merged), results.size());
}
private String runWorkflowAsyncAndWait(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey) throws Exception {
private String runWorkflowAsyncAndWait(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch) throws Exception {
CozeCredentialRef credential = nextCredential();
JsonNode submitRoot = objectMapper.readTree(postWorkflow(rows, prompt, apiKey, credential));
JsonNode submitRoot = objectMapper.readTree(postWorkflow(rows, prompt, apiKey, imgSwitch, credential));
ensureSuccess(submitRoot);
String immediateData = extractResultDataText(submitRoot);
@@ -232,8 +241,9 @@ public class SimilarAsinCozeClient {
private String postWorkflow(List<SimilarAsinResultRowDto> rows,
String prompt,
String apiKey,
boolean imgSwitch,
CozeCredentialRef credential) {
Map<String, Object> parameters = buildParameters(rows, prompt, apiKey);
Map<String, Object> parameters = buildParameters(rows, prompt, apiKey, imgSwitch);
Map<String, Object> body = new LinkedHashMap<>();
body.put("workflow_id", credential.workflowId());
body.put("parameters", parameters);
@@ -354,7 +364,7 @@ public class SimilarAsinCozeClient {
return credentials;
}
private Map<String, Object> buildParameters(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey) {
private Map<String, Object> buildParameters(List<SimilarAsinResultRowDto> rows, String prompt, String apiKey, boolean imgSwitch) {
List<String> asins = rows.stream().map(row -> safeText(row.getAsin())).toList();
List<String> titles = rows.stream().map(row -> safeText(firstNonBlank(row.getTitle(), row.getAsin()))).toList();
List<String> skus = rows.stream().map(row -> safeText(row.getSku())).toList();
@@ -375,6 +385,7 @@ public class SimilarAsinCozeClient {
if (includeLegacyApiKey && apiKey != null && !apiKey.isBlank()) {
parameters.put("api_key", apiKey.trim());
}
parameters.put("img_switch", imgSwitch);
return parameters;
}

5
backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/dto/SimilarAsinParseRequest.java
View File

@@ -32,4 +32,9 @@ public class SimilarAsinParseRequest {
@Schema(description = "传递给 Coze workflow parameters.api_key 的任务级密钥。")
@NotBlank(message = "密钥不能为空")
private String apiKey;
@JsonProperty("img_switch")
@JsonAlias({"imgSwitch"})
@Schema(description = "传递给 Coze workflow parameters.img_switch 的图片检测开关，true 为开启，false 为关闭。")
private Boolean imgSwitch = Boolean.FALSE;
}

3
backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/dto/SimilarAsinParsedPayloadDto.java
View File

@@ -17,6 +17,9 @@ public class SimilarAsinParsedPayloadDto {
@Schema(description = "传递给 Coze workflow parameters.api_key 的任务级密钥")
private String apiKey;
@Schema(description = "传递给 Coze workflow parameters.img_switch 的图片检测开关")
private Boolean imgSwitch = Boolean.FALSE;
@Schema(description = "本次解析的源文件列表")
private List<SimilarAsinSourceFileDto> sourceFiles = new ArrayList<>();

3
backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/model/vo/SimilarAsinParseVo.java
View File

@@ -33,6 +33,9 @@ public class SimilarAsinParseVo {
@Schema(description = "本任务最终使用的 AI 提示词")
private String aiPrompt;
@Schema(description = "图片检测开关，true 为开启，false 为关闭")
private Boolean imgSwitch = Boolean.FALSE;
@Schema(description = "兼容旧前端的平铺有效行列表，现为全部有效行")
private List<SimilarAsinParsedRowVo> items = new ArrayList<>();

211
backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java
View File

@@ -401,11 +401,11 @@ public class SimilarAsinTaskService {
String aggregateScopeKey = buildAggregateScopeKey(sourceFiles);
String sourceScopeHash = DigestUtil.sha256Hex(aggregateScopeKey);
String parsedPayload = buildParsedPayloadJson(request.getAiPrompt(), request.getApiKey(), sourceFiles, mergedHeaders, groups, allRows);
String parsedPayload = buildParsedPayloadJson(request.getAiPrompt(), request.getApiKey(), request.getImgSwitch(), sourceFiles, mergedHeaders, groups, allRows);
long payloadBuiltAt = System.nanoTime();
String parsedPayloadPointer = storeParsedPayload(task.getId(), sourceScopeHash, parsedPayload);
long payloadStoredAt = System.nanoTime();
task.setResultJson(buildTaskResultJson(request.getAiPrompt(), request.getApiKey(), sourceFiles, parsedPayloadPointer));
task.setResultJson(buildTaskResultJson(request.getAiPrompt(), request.getApiKey(), request.getImgSwitch(), sourceFiles, parsedPayloadPointer));
task.setUpdatedAt(LocalDateTime.now());
fileTaskMapper.updateById(task);
@@ -444,6 +444,7 @@ public class SimilarAsinTaskService {
vo.setDroppedRows(droppedRows);
vo.setGroupCount(groups.size());
vo.setAiPrompt(normalize(request.getAiPrompt()));
vo.setImgSwitch(Boolean.TRUE.equals(request.getImgSwitch()));
vo.setItems(new ArrayList<>(allRows));
vo.setGroups(groups);
long finishedAt = System.nanoTime();
@@ -1229,10 +1230,11 @@ public class SimilarAsinTaskService {
}
String prompt = readAiPrompt(task);
String apiKey = readApiKey(task);
boolean imgSwitch = readImgSwitch(task);
int batchSize = Math.max(1, properties.getCozeBatchSize());
List<SimilarAsinResultRowDto> result = new ArrayList<>();
for (int i = 0; i < items.size(); i += batchSize) {
result.addAll(cozeClient.inspect(items.subList(i, Math.min(i + batchSize, items.size())), prompt, apiKey));
result.addAll(cozeClient.inspect(items.subList(i, Math.min(i + batchSize, items.size())), prompt, apiKey, imgSwitch));
if (progressHook != null) {
progressHook.run();
}
@@ -1543,6 +1545,14 @@ public class SimilarAsinTaskService {
}
}
private boolean readImgSwitch(FileTaskEntity task) {
try {
return Boolean.TRUE.equals(readParsedPayload(task).getImgSwitch());
} catch (Exception ignored) {
return false;
}
}
private void finalizeTask(FileTaskEntity task, String error, int rowCount, boolean assembleWorkbook) {
String finalError = error;
FileResultEntity result = null;
@@ -1910,6 +1920,7 @@ public class SimilarAsinTaskService {
}
String prompt = readAiPrompt(task);
String apiKey = readApiKey(task);
boolean imgSwitch = readImgSwitch(task);
int batchSize = Math.max(1, properties.getCozeBatchSize());
// P1-1：检测到 720712008 风暴时强制把 batch 降到 1，隔离毒行；
// 持续 5+ 次提交命中率 ≥ 40% 才会触发，正常波动不影响吞吐。
@@ -1974,7 +1985,7 @@ public class SimilarAsinTaskService {
int batchIndex = 1;
for (int i = 0; i < submitLimit; i += batchSize) {
List<CozeCandidate> batchCandidates = readyCandidates.subList(i, Math.min(i + batchSize, submitLimit));
pending |= submitCozeBatch(task, result, job, batchCandidates, batchIndex, batchTotal, prompt, apiKey, allRowsByBaseId);
pending |= submitCozeBatch(task, result, job, batchCandidates, batchIndex, batchTotal, prompt, apiKey, imgSwitch, allRowsByBaseId);
batchIndex++;
}
return pending || countPendingCozeStates(task.getId()) > 0;
@@ -2089,6 +2100,7 @@ public class SimilarAsinTaskService {
int batchTotal,
String prompt,
String apiKey,
boolean imgSwitch,
Map<String, List<SimilarAsinParsedRowVo>> allRowsByBaseId) {
if (batchCandidates == null || batchCandidates.isEmpty()) {
return false;
@@ -2115,7 +2127,7 @@ public class SimilarAsinTaskService {
SimilarAsinCozeClient.CozeCredentialRef credential = cozeClient.nextCredential();
try {
SimilarAsinCozeClient.CozeSubmitResponse submit = submitCozeWorkflowThrottled(
batchRows, prompt, apiKey, credential, true);
batchRows, prompt, apiKey, imgSwitch, credential, true);
if (submit.immediateData() != null && !submit.immediateData().isBlank()) {
List<SimilarAsinResultRowDto> cozeRows = cozeClient.mergeRowsFromDataText(batchRows, submit.immediateData());
String emptyResultMessage = emptyCozeResultMessage(cozeRows, batchRows.size());
@@ -2426,7 +2438,7 @@ public class SimilarAsinTaskService {
}
try {
SimilarAsinCozeClient.CozeSubmitResponse submit =
submitCozeWorkflowThrottled(batchRows, readAiPrompt(task), readApiKey(task),
submitCozeWorkflowThrottled(batchRows, readAiPrompt(task), readApiKey(task), readImgSwitch(task),
cozeClient.credentialByName(context.credentialName()), false);
Map<String, List<SimilarAsinParsedRowVo>> allRowsByBaseId = allRowsByBaseIdForPoll(task);
if (submit.immediateData() != null && !submit.immediateData().isBlank()) {
@@ -2488,6 +2500,7 @@ public class SimilarAsinTaskService {
List<SimilarAsinResultRowDto> rows,
String prompt,
String apiKey,
boolean imgSwitch,
SimilarAsinCozeClient.CozeCredentialRef credential,
boolean allowCredentialFallback) throws Exception {
int attempts = allowCredentialFallback ? Math.max(1, cozeClient.configuredCredentialCount()) : 1;
@@ -2521,7 +2534,7 @@ public class SimilarAsinTaskService {
}
try (lockHandle; borrowedCredential) {
SimilarAsinCozeClient.CozeSubmitResponse response =
cozeClient.submitWorkflow(rows, prompt, apiKey, currentCredential);
cozeClient.submitWorkflow(rows, prompt, apiKey, imgSwitch, currentCredential);
// 提交完成立即记录时间戳，用于下次进入循环时计算节流等待。
lastCozeSubmitAtByCredential.put(credentialKey, System.currentTimeMillis());
return response;
@@ -2713,7 +2726,7 @@ public class SimilarAsinTaskService {
int partIndex = 1;
for (List<SimilarAsinResultRowDto> partRows : partitions) {
SimilarAsinCozeClient.CozeSubmitResponse submit =
submitCozeWorkflowThrottled(partRows, readAiPrompt(task), readApiKey(task),
submitCozeWorkflowThrottled(partRows, readAiPrompt(task), readApiKey(task), readImgSwitch(task),
cozeClient.credentialByName(context.credentialName()), false);
if (submit.immediateData() != null && !submit.immediateData().isBlank()) {
List<SimilarAsinResultRowDto> cozeRows =
@@ -2847,7 +2860,7 @@ public class SimilarAsinTaskService {
try {
taskFileJobService.touchRunning(context.jobId());
SimilarAsinCozeClient.CozeSubmitResponse submit =
submitCozeWorkflowThrottled(batchRows, readAiPrompt(task), readApiKey(task),
submitCozeWorkflowThrottled(batchRows, readAiPrompt(task), readApiKey(task), readImgSwitch(task),
cozeClient.credentialByName(context.credentialName()), false);
Map<String, List<SimilarAsinParsedRowVo>> allRowsByBaseId = allRowsByBaseIdForPoll(task);
if (submit.immediateData() != null && !submit.immediateData().isBlank()) {
@@ -3691,13 +3704,26 @@ public class SimilarAsinTaskService {
if (taskId == null || taskId <= 0) {
return false;
}
Long count = taskScopeStateMapper.selectCount(new LambdaQueryWrapper<TaskScopeStateEntity>()
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.isNull(TaskScopeStateEntity::getCozeStatus)
.isNotNull(TaskScopeStateEntity::getLastChunkAt)
.eq(TaskScopeStateEntity::getCompleted, 1));
return count != null && count > 0;
.isNotNull(TaskScopeStateEntity::getLastChunkAt));
if (states == null || states.isEmpty()) {
return false;
}
boolean hasCompletedScope = false;
for (TaskScopeStateEntity state : states) {
if (state == null) {
continue;
}
if (Integer.valueOf(1).equals(state.getCompleted())) {
hasCompletedScope = true;
continue;
}
return false;
}
return hasCompletedScope;
}
private boolean isJavaSideProcessing(Long taskId) {
@@ -3737,6 +3763,9 @@ public class SimilarAsinTaskService {
SimilarAsinResultRowDto row) {
}
private record PythonUploadProgress(int current, int total, String unit) {
}
private String buildTaskOwnerScopeKey(FileTaskEntity task) {
Long taskId = task == null ? null : task.getId();
return "task:" + taskId + ":owner:" + firstNonBlank(ownerFromTask(task), currentInstanceId());
@@ -4812,8 +4841,8 @@ public class SimilarAsinTaskService {
vo.setSourceFilename(row.getSourceFilename());
vo.setResultFilename(row.getResultFilename());
vo.setDownloadUrl(buildFreshDownloadUrl(row));
attachFileJobState(vo, row, job);
vo.setTaskStatus(task == null ? null : task.getStatus());
attachFileJobState(vo, row, job, task);
vo.setSuccess(row.getSuccess() != null && row.getSuccess() == 1);
vo.setError(row.getErrorMessage());
vo.setRowCount(row.getRowCount());
@@ -4838,20 +4867,20 @@ public class SimilarAsinTaskService {
}
}
private void attachFileJobState(SimilarAsinHistoryItemVo vo, FileResultEntity row, TaskFileJobEntity job) {
private void attachFileJobState(SimilarAsinHistoryItemVo vo, FileResultEntity row, TaskFileJobEntity job, FileTaskEntity task) {
vo.setFileReady(row.getResultFileUrl() != null && !row.getResultFileUrl().isBlank());
if (job == null) {
vo.setFileStatus(Boolean.TRUE.equals(vo.getFileReady()) ? "SUCCESS" : null);
attachFileProgress(vo, row, job);
attachFileProgress(vo, row, job, task);
return;
}
vo.setFileJobId(job.getId());
vo.setFileStatus(job.getStatus());
vo.setFileError(job.getErrorMessage());
attachFileProgress(vo, row, job);
attachFileProgress(vo, row, job, task);
}
private void attachFileProgress(SimilarAsinHistoryItemVo vo, FileResultEntity row, TaskFileJobEntity job) {
private void attachFileProgress(SimilarAsinHistoryItemVo vo, FileResultEntity row, TaskFileJobEntity job, FileTaskEntity task) {
if (row == null || row.getTaskId() == null) {
return;
}
@@ -4866,22 +4895,22 @@ public class SimilarAsinTaskService {
int cozeCompleted = countCompletedCozeStates(taskId);
int cozePending = countPendingCozeStates(taskId);
boolean uploadComplete = isResultSubmissionComplete(taskId);
if (!uploadComplete) {
if (task != null && !STATUS_RUNNING.equals(task.getStatus())) {
return;
}
attachPythonUploadProgress(vo, taskId);
return;
}
TaskProgressSnapshotEntity snapshot = taskProgressSnapshotService.find(taskId, MODULE_TYPE);
if (cozeCompleted + cozePending > 0) {
int total = calculateCozeDisplayTotal(snapshot, cozeCompleted + cozePending);
if (!uploadComplete) {
total = Math.max(total, cozeCompleted + cozePending + 1);
}
int current = Math.max(0, Math.min(cozeCompleted, total));
vo.setFileProgressCurrent(current);
vo.setFileProgressTotal(total);
int percent = calculateDisplayProgressPercent(current, total, job, snapshot == null ? null : snapshot.getUpdatedAt());
percent = Math.max(percent, calculateSnapshotDisplayPercent(snapshot, job));
vo.setFileProgressPercent(Math.min(uploadComplete ? 99 : 98, percent));
if (!uploadComplete) {
vo.setFileProgressMessage(buildUploadingCozeProgressMessage(current, total, cozePending));
return;
}
vo.setFileProgressPercent(Math.min(99, percent));
vo.setFileProgressMessage(cozePending > 0
? buildCozeProgressMessage(current, total, cozePending)
: "Coze 已回流 " + current + "/" + total + " 批次，正在生成结果文件");
@@ -4905,6 +4934,121 @@ public class SimilarAsinTaskService {
vo.setFileProgressMessage(snapshot.getMessage());
}
private void attachPythonUploadProgress(SimilarAsinHistoryItemVo vo, Long taskId) {
PythonUploadProgress progress = resolvePythonUploadProgress(taskId);
int total = progress.total() > 0 ? progress.total() : Math.max(1, progress.current() + 1);
int current = Math.max(0, Math.min(progress.current(), total));
int percent = current <= 0 ? 1 : Math.min(95, (int) Math.floor(current * 100.0 / total));
vo.setFileProgressCurrent(current);
vo.setFileProgressTotal(total);
vo.setFileProgressPercent(percent);
vo.setFileProgressMessage(buildPythonUploadProgressMessage(current, total, progress.unit()));
}
private PythonUploadProgress resolvePythonUploadProgress(Long taskId) {
PythonUploadProgress chunkProgress = resolvePythonChunkProgress(taskId);
if (chunkProgress.total() > 0) {
return chunkProgress;
}
if (chunkProgress.current() > 0) {
return new PythonUploadProgress(chunkProgress.current(), chunkProgress.current() + 1, chunkProgress.unit());
}
int totalRows = 0;
try {
FileTaskEntity task = fileTaskMapper.selectById(taskId);
totalRows = allRowCount(task);
} catch (Exception ignored) {
totalRows = 0;
}
int uploadedRows = countSubmittedRows(taskId);
if (totalRows <= 0 && uploadedRows > 0) {
totalRows = uploadedRows + 1;
}
return new PythonUploadProgress(uploadedRows, totalRows, "行");
}
private PythonUploadProgress resolvePythonChunkProgress(Long taskId) {
if (taskId == null || taskId <= 0) {
return new PythonUploadProgress(0, 0, "分片");
}
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.isNull(TaskScopeStateEntity::getCozeStatus)
.orderByDesc(TaskScopeStateEntity::getUpdatedAt));
if (states == null || states.isEmpty()) {
return resolvePythonChunkProgressFromChunks(taskId);
}
int current = 0;
int total = 0;
for (TaskScopeStateEntity state : states) {
if (state == null) {
continue;
}
int stateTotal = state.getChunkTotal() == null ? 0 : state.getChunkTotal();
int stateCurrent = state.getReceivedChunkCount() == null ? 0 : state.getReceivedChunkCount();
if (stateTotal > total || total <= 0 && stateCurrent > current) {
total = stateTotal;
current = stateCurrent;
}
}
return new PythonUploadProgress(current, total, "分片");
}
private PythonUploadProgress resolvePythonChunkProgressFromChunks(Long taskId) {
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
if (chunks == null || chunks.isEmpty()) {
return new PythonUploadProgress(0, 0, "分片");
}
Map<String, Integer> receivedByScope = new LinkedHashMap<>();
Map<String, Integer> totalByScope = new LinkedHashMap<>();
for (TaskChunkEntity chunk : chunks) {
if (chunk == null) {
continue;
}
String scopeHash = firstNonBlank(chunk.getScopeHash(), "");
receivedByScope.merge(scopeHash, 1, Integer::sum);
if (chunk.getChunkTotal() != null && chunk.getChunkTotal() > 0) {
totalByScope.merge(scopeHash, chunk.getChunkTotal(), Math::max);
}
}
int current = 0;
int total = 0;
for (Map.Entry<String, Integer> entry : receivedByScope.entrySet()) {
int scopeCurrent = entry.getValue() == null ? 0 : entry.getValue();
int scopeTotal = totalByScope.getOrDefault(entry.getKey(), 0);
if (scopeTotal > total || total <= 0 && scopeCurrent > current) {
current = scopeCurrent;
total = scopeTotal;
}
}
return new PythonUploadProgress(current, total, "分片");
}
private int countSubmittedRows(Long taskId) {
if (taskId == null || taskId <= 0) {
return 0;
}
int total = 0;
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
if (chunks == null || chunks.isEmpty()) {
return 0;
}
for (TaskChunkEntity chunk : chunks) {
total += readChunkRows(chunk).size();
}
return total;
}
private String buildPythonUploadProgressMessage(int current, int total, String unit) {
String safeUnit = unit == null || unit.isBlank() ? "分片" : unit;
return "等待 Python 回传，已回传 " + Math.max(0, current) + "/" + Math.max(1, total) + " " + safeUnit;
}
private int calculateCozeDisplayTotal(TaskProgressSnapshotEntity snapshot, int observedCozeStates) {
int observed = Math.max(0, observedCozeStates);
// 小任务 totalCount <= 3 时直接使用 observed，避免 totalCount - 3 → 0 引起进度条 0/0。
@@ -4924,15 +5068,6 @@ public class SimilarAsinTaskService {
}
return "Coze 已回流 " + safeCompleted + "/" + safeTotal + " 批次，等待结果文件";
}
private String buildUploadingCozeProgressMessage(int completed, int total, int pending) {
int safeTotal = Math.max(1, total);
int safeCompleted = Math.max(0, Math.min(completed, safeTotal));
int submitted = Math.max(safeCompleted, Math.min(safeTotal, safeCompleted + Math.max(0, pending)));
if (pending > 0) {
return "Python 仍在回传，Coze 已提交 " + submitted + "/" + safeTotal + " 批次";
}
return "Python 仍在回传，Coze 已回流 " + safeCompleted + "/" + safeTotal + " 批次";
}
private String fmt(LocalDateTime t) {
return t == null ? null : t.toString();
@@ -4968,10 +5103,11 @@ public class SimilarAsinTaskService {
return val == null ? "" : val.replace(String.valueOf((char) 0xFEFF), "").replace((char) 0x3000, ' ').trim().replaceAll("\\s+", " ");
}
private String buildParsedPayloadJson(String aiPrompt, String apiKey, List<SimilarAsinSourceFileDto> sourceFiles, List<String> headers, List<SimilarAsinParsedGroupVo> groups, List<SimilarAsinParsedRowVo> allRows) {
private String buildParsedPayloadJson(String aiPrompt, String apiKey, Boolean imgSwitch, List<SimilarAsinSourceFileDto> sourceFiles, List<String> headers, List<SimilarAsinParsedGroupVo> groups, List<SimilarAsinParsedRowVo> allRows) {
SimilarAsinParsedPayloadDto payload = new SimilarAsinParsedPayloadDto();
payload.setAiPrompt(normalize(aiPrompt));
payload.setApiKey(normalize(apiKey));
payload.setImgSwitch(Boolean.TRUE.equals(imgSwitch));
payload.setSourceFiles(sourceFiles == null ? List.of() : sourceFiles);
payload.setHeaders(headers == null ? List.of() : headers);
payload.setItems(allRows == null ? List.of() : new ArrayList<>(allRows));
@@ -4980,10 +5116,11 @@ public class SimilarAsinTaskService {
return writeJson(payload, "保存解析结果失败");
}
private String buildTaskResultJson(String aiPrompt, String apiKey, List<SimilarAsinSourceFileDto> sourceFiles, String parsedPayloadPointer) {
private String buildTaskResultJson(String aiPrompt, String apiKey, Boolean imgSwitch, List<SimilarAsinSourceFileDto> sourceFiles, String parsedPayloadPointer) {
Map<String, Object> payload = new LinkedHashMap<>();
payload.put("aiPrompt", normalize(aiPrompt));
payload.put("apiKey", normalize(apiKey));
payload.put("imgSwitch", Boolean.TRUE.equals(imgSwitch));
payload.put("sourceFileKeys", sourceFiles == null ? List.of() : sourceFiles.stream()
.map(SimilarAsinSourceFileDto::getFileKey)
.filter(Objects::nonNull)

2
backend-java/src/main/resources/db/V51__coze_credential_pool.sql
View File

@@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS biz_coze_credential (
INSERT INTO biz_coze_credential (module_type, credential_name, workflow_id, token, enabled, max_concurrent, sort_order)
VALUES
('APPEARANCE_PATENT', 'appearance-1', '7639685157562089513', 'Bearer sat_CztofPRhIKFKeaPZBIRxocfckqhsdCZZ45NPhkf7WOZjbX36vnVSfVV30T6u2rSX', 1, 0, 10),
('APPEARANCE_PATENT', 'appearance-2', '7639688221823778850', 'Bearer sat_2VsTpfWXYG3EkCo8OQfLwJEWqhduCL4NEw7sSCsABppkW61kkSRPIac9coNlEnHE', 1, 0, 20),
('APPEARANCE_PATENT', 'appearance-2', '7644952624353329202', 'Bearer sat_2VsTpfWXYG3EkCo8OQfLwJEWqhduCL4NEw7sSCsABppkW61kkSRPIac9coNlEnHE', 1, 0, 20),
('SIMILAR_ASIN', 'similar-1', '7639708860686024756', 'Bearer sat_vqmb97BiXbMW3XVyG9htWQKjZ2F55CaW2FRrT9bDdPha7a3hxmXnMNCE1XNHdCVU', 1, 0, 10),
('SIMILAR_ASIN', 'similar-2', '7635328462404583478', 'Bearer sat_gsInckdsN0qqhnZSK7K4Zevw6neJCHmYHyIbm85oGgJIGEmSdkH5OFctSKLsAHvT', 1, 0, 20)
ON DUPLICATE KEY UPDATE

4
backend-java/src/main/resources/db/V57__update_appearance_patent_workflow.sql Normal file
View File

@@ -0,0 +1,4 @@
UPDATE biz_coze_credential
SET workflow_id = '7644952624353329202'
WHERE module_type = 'APPEARANCE_PATENT'
AND credential_name = 'appearance-2';