Compare commits
2 Commits
5f4fcad2ef
...
52c59755ab
| Author | SHA1 | Date | |
|---|---|---|---|
 huangzd1997
|
52c59755ab | ||
|
huangzd1997
|
478233ba7a |
+12
-5
@@ -97,6 +97,13 @@ public class CollectDataService {
|
||||
|
||||
public static final String MODULE_TYPE = "COLLECT_DATA";
|
||||
|
||||
/**
|
||||
* task-50 上线初期写入端（CollectDataResultItemBatchWriter）曾误用小写
|
||||
* "collectdata"，遗留 5386 行存量数据；读取端查询统一 in(MODULE_TYPE, LEGACY_MODULE_TYPE)
|
||||
* 兼容，否则这些任务的明细行在生成结果文件时查不出来。
|
||||
*/
|
||||
private static final String LEGACY_MODULE_TYPE = "collectdata";
|
||||
|
||||
public TaskProgressLightBatchVo progressLight(List<Long> taskIds) {
|
||||
return taskProgressLightAssembler.assemble(MODULE_TYPE, null, taskIds);
|
||||
}
|
||||
@@ -937,7 +944,7 @@ public class CollectDataService {
|
||||
private List<CollectDataResultRowVo> loadFinalRows(Long taskId) {
|
||||
List<TaskResultItemEntity> rows = taskResultItemMapper.selectList(new LambdaQueryWrapper<TaskResultItemEntity>()
|
||||
.eq(TaskResultItemEntity::getTaskId, taskId)
|
||||
.eq(TaskResultItemEntity::getModuleType, MODULE_TYPE)
|
||||
.in(TaskResultItemEntity::getModuleType, MODULE_TYPE, LEGACY_MODULE_TYPE)
|
||||
.orderByAsc(TaskResultItemEntity::getId));
|
||||
if (rows == null || rows.isEmpty()) {
|
||||
return new ArrayList<>();
|
||||
@@ -1258,7 +1265,7 @@ public class CollectDataService {
|
||||
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
|
||||
taskResultItemMapper.delete(new LambdaQueryWrapper<TaskResultItemEntity>()
|
||||
.eq(TaskResultItemEntity::getTaskId, task.getId())
|
||||
.eq(TaskResultItemEntity::getModuleType, MODULE_TYPE));
|
||||
.in(TaskResultItemEntity::getModuleType, MODULE_TYPE, LEGACY_MODULE_TYPE));
|
||||
deleteTransientTaskPayloads(
|
||||
taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.select(TaskChunkEntity::getPayloadJson)
|
||||
@@ -1267,7 +1274,7 @@ public class CollectDataService {
|
||||
taskResultItemMapper.selectList(new LambdaQueryWrapper<TaskResultItemEntity>()
|
||||
.select(TaskResultItemEntity::getPayloadJson)
|
||||
.eq(TaskResultItemEntity::getTaskId, task.getId())
|
||||
.eq(TaskResultItemEntity::getModuleType, MODULE_TYPE)));
|
||||
.in(TaskResultItemEntity::getModuleType, MODULE_TYPE, LEGACY_MODULE_TYPE)));
|
||||
taskFileJobService.deleteTaskJobs(task.getId(), MODULE_TYPE);
|
||||
fileTaskMapper.deleteById(task.getId());
|
||||
}
|
||||
@@ -1280,11 +1287,11 @@ public class CollectDataService {
|
||||
List<TaskResultItemEntity> resultItems = taskResultItemMapper.selectList(new LambdaQueryWrapper<TaskResultItemEntity>()
|
||||
.select(TaskResultItemEntity::getPayloadJson)
|
||||
.eq(TaskResultItemEntity::getTaskId, row.getTaskId())
|
||||
.eq(TaskResultItemEntity::getModuleType, MODULE_TYPE)
|
||||
.in(TaskResultItemEntity::getModuleType, MODULE_TYPE, LEGACY_MODULE_TYPE)
|
||||
.eq(TaskResultItemEntity::getResultId, row.getId()));
|
||||
taskResultItemMapper.delete(new LambdaQueryWrapper<TaskResultItemEntity>()
|
||||
.eq(TaskResultItemEntity::getTaskId, row.getTaskId())
|
||||
.eq(TaskResultItemEntity::getModuleType, MODULE_TYPE)
|
||||
.in(TaskResultItemEntity::getModuleType, MODULE_TYPE, LEGACY_MODULE_TYPE)
|
||||
.eq(TaskResultItemEntity::getResultId, row.getId()));
|
||||
// 与 deleteTask 一致：先删 DB 行再物理删对象，保证行删除与对象删除一致。
|
||||
deleteResultItemPayloads(resultItems);
|
||||
|
||||
+9
-2
@@ -27,7 +27,12 @@ import java.util.Objects;
|
||||
@Component
|
||||
public class CollectDataResultItemBatchWriter {
|
||||
|
||||
private static final String MODULE_TYPE = "collectdata";
|
||||
/** 必须与 CollectDataService.MODULE_TYPE 一致（大写 COLLECT_DATA）。
|
||||
* 曾因写端误用小写 "collectdata"，导致 task-50 之后所有采集任务的明细行
|
||||
* 在生成结果文件时查不出来（biz_task_result_item.module_type 大小写分叉）。 */
|
||||
private static final String MODULE_TYPE = "COLLECT_DATA";
|
||||
/** 历史存量小写值，查询现有行时兼容，避免旧任务重试重复插入（与 CollectDataService.LEGACY_MODULE_TYPE 对应）。 */
|
||||
private static final String LEGACY_MODULE_TYPE = "collectdata";
|
||||
private static final int DEFAULT_BATCH_SIZE = 100;
|
||||
|
||||
private final TaskResultItemMapper taskResultItemMapper;
|
||||
@@ -63,10 +68,12 @@ public class CollectDataResultItemBatchWriter {
|
||||
}
|
||||
String scopeHash = sha256(scopeKey);
|
||||
// 一次性取回本 scope 现有行，构建 item_key → 现有行 映射（hash 相等即跳过）。
|
||||
// 现有行查询兼容历史小写 "collectdata"（见 MODULE_TYPE 注释），避免旧任务重试
|
||||
// 在同一唯一键 (task_id, module_type, scope_hash, item_key) 下重复插入。
|
||||
List<TaskResultItemEntity> existingList = taskResultItemMapper.selectList(
|
||||
new LambdaQueryWrapper<TaskResultItemEntity>()
|
||||
.eq(TaskResultItemEntity::getTaskId, taskId)
|
||||
.eq(TaskResultItemEntity::getModuleType, MODULE_TYPE)
|
||||
.in(TaskResultItemEntity::getModuleType, MODULE_TYPE, LEGACY_MODULE_TYPE)
|
||||
.eq(TaskResultItemEntity::getScopeHash, scopeHash));
|
||||
Map<String, TaskResultItemEntity> existingByKey = new HashMap<>();
|
||||
for (TaskResultItemEntity existing : existingList) {
|
||||
|
||||
+29
-1
@@ -4623,7 +4623,8 @@ public class SimilarAsinTaskService {
|
||||
String isConform = resultRow == null ? "" : userFacingLlmCellValue(resultRow, resultRow.getIsConform());
|
||||
String reason = resultRow == null ? "" : userFacingLlmCellValue(resultRow, resultRow.getReason());
|
||||
String category = resultRow == null ? "" : userFacingLlmCellValue(resultRow, resultRow.getCategory());
|
||||
String status = resultRow == null || isSilentOutputFailure(resultRow)
|
||||
String status = resultRow == null
|
||||
|| (isSilentOutputFailure(resultRow) && !isAsinNotFound(resultRow.getError()))
|
||||
? ""
|
||||
: resolveResultStatus(resultRow, isStock, similarity, isConform, reason, category);
|
||||
row.createCell(col++).setCellValue(isStock);
|
||||
@@ -4858,6 +4859,12 @@ public class SimilarAsinTaskService {
|
||||
if (hasText(row.getMainUrl()) || hasText(row.getPuzzleImg1()) || hasText(row.getPuzzleImg2())) {
|
||||
return "\u6210\u529f";
|
||||
}
|
||||
// Python \u7aef\u641c\u4e0d\u5230\u5546\u54c1\uff08404\uff09\u7684\u884c\uff1a\u72b6\u6001\u5217\u660e\u786e\u6807\u8bb0\uff0c\u4eba\u5de5\u590d\u6838\u65f6\u4e00\u773c\u53ef\u8fa8\u3002
|
||||
// \u653e\u5728\u56fe\u7247/\u72b6\u6001\u5224\u65ad\u4e4b\u524d\uff1a404 \u884c\u5fc5\u7136\u65e0\u56fe\u4e14 status \u5df2\u88ab markRowFailed \u7f6e\u4e3a FAILED\uff0c
|
||||
// \u4e0d\u80fd\u843d\u56de"\u5931\u8d25"\uff08\u90a3\u662f LLM/\u751f\u6210\u94fe\u8def\u5931\u8d25\u8bed\u4e49\uff0c\u4e0e"ASIN \u641c\u4e0d\u5230"\u662f\u4e24\u56de\u4e8b\uff09\u3002
|
||||
if (isAsinNotFound(row.getError())) {
|
||||
return "ASIN\u672a\u627e\u5230";
|
||||
}
|
||||
String rawStatus = row.getStatus() == null ? "" : row.getStatus().trim();
|
||||
if (!rawStatus.isBlank()) {
|
||||
return isFailedStatus(rawStatus) ? "\u5931\u8d25" : "\u6210\u529f";
|
||||
@@ -4872,6 +4879,22 @@ public class SimilarAsinTaskService {
|
||||
return hasText(row.getConclusion()) ? "\u6210\u529f" : "\u5931\u8d25";
|
||||
}
|
||||
|
||||
/**
|
||||
* \u5224\u5b9a\u662f\u5426"ASIN \u672a\u627e\u5230"\uff1aPython \u7aef\u6293\u53d6 404\uff08error="404 Not Found"\uff09\u6216\u7b49\u4ef7\u6587\u6848\u3002
|
||||
* \u547d\u4e2d\u8be5\u5224\u5b9a\u7684\u884c\u4e0d\u4f1a\u88ab\u5f53\u6210\u751f\u6210\u5931\u8d25\u8ba1\u91cd\u8bd5\uff0c\u4e5f\u4e0d\u4f1a\u88ab\u5bfc\u51fa\u8fc7\u6ee4\uff0c\u72b6\u6001\u5217\u663e\u793a"ASIN\u672a\u627e\u5230"\u3002
|
||||
*/
|
||||
static boolean isAsinNotFound(String error) {
|
||||
String normalized = error == null ? "" : error.toLowerCase(Locale.ROOT).trim();
|
||||
if (normalized.isBlank()) {
|
||||
return false;
|
||||
}
|
||||
return normalized.contains("404")
|
||||
|| normalized.contains("not found")
|
||||
|| normalized.contains("asin \u672a\u627e\u5230")
|
||||
|| normalized.contains("asin\u4e0d\u5b58\u5728")
|
||||
|| normalized.contains("\u5546\u54c1\u672a\u627e\u5230");
|
||||
}
|
||||
|
||||
private static boolean isFailedStatus(String status) {
|
||||
String normalized = status == null ? "" : status.trim().toLowerCase(Locale.ROOT);
|
||||
return normalized.contains("fail")
|
||||
@@ -4895,6 +4918,11 @@ public class SimilarAsinTaskService {
|
||||
if (error.isBlank()) {
|
||||
return false;
|
||||
}
|
||||
// Python 端搜不到商品的 404 行：虽然 title/url/图全空，也必须导出，
|
||||
// 并在状态列标记"ASIN未找到"，供人工复核。在"其它错误需带 title/url/图"之前判断。
|
||||
if (isAsinNotFound(error)) {
|
||||
return true;
|
||||
}
|
||||
return !normalize(row.getTitle()).isBlank()
|
||||
|| !normalize(row.getUrl()).isBlank()
|
||||
|| row.hasImageUrl();
|
||||
|
||||
+39
@@ -72,6 +72,45 @@ class CollectDataResultItemBatchWriterTest {
|
||||
assertEquals(1L, entities.get(0).getTaskId(), "task_id 保留");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_module_type_matches_reader_constant() {
|
||||
// 回归：写入端 module_type 必须与 CollectDataService.MODULE_TYPE（读取端）
|
||||
// 写同一大小写值，否则生成结果文件时 loadFinalRows 查不出来（曾用小写
|
||||
// "collectdata" 导致 task-50 之后所有采集任务明细行读不到）。
|
||||
when(taskResultItemMapper.selectList(any())).thenReturn(List.of());
|
||||
when(taskResultItemMapper.upsertBatch(anyList())).thenReturn(1);
|
||||
|
||||
writer.upsertAccepted(1L, 2L, "task:1", 0,
|
||||
List.of(row("B000000001", "Nike")), "rustfs:detail/abc");
|
||||
|
||||
ArgumentCaptor<List> captor = ArgumentCaptor.forClass(List.class);
|
||||
verify(taskResultItemMapper).upsertBatch(captor.capture());
|
||||
TaskResultItemEntity entity = (TaskResultItemEntity) captor.getValue().get(0);
|
||||
assertEquals("COLLECT_DATA", entity.getModuleType(), "写入 module_type 须为读取端大写常量");
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_legacy_lowercase_rows_are_found_on_retry() {
|
||||
// 回归：历史小写 "collectdata" 存量行在旧任务重提时也要被现有行查询
|
||||
// 命中（按大小写兼容匹配），避免同一唯一键重复插入。
|
||||
String refJson = codec.encodeRef(0, 0, "rustfs:detail/x");
|
||||
TaskResultItemEntity legacy = new TaskResultItemEntity();
|
||||
legacy.setId(100L);
|
||||
legacy.setModuleType("collectdata");
|
||||
legacy.setItemKey("asin:B000000001");
|
||||
legacy.setPayloadJson(refJson);
|
||||
legacy.setPayloadHash(sha256(refJson));
|
||||
when(taskResultItemMapper.selectList(any())).thenReturn(List.of(legacy));
|
||||
|
||||
CollectDataResultItemBatchWriter.UpsertCounts counts =
|
||||
writer.upsertAccepted(1L, 2L, "task:1", 0,
|
||||
List.of(row("B000000001", "Nike")), "rustfs:detail/x");
|
||||
|
||||
assertEquals(0, counts.insertedOrUpdated(), "小写存量行命中 → 全部跳过");
|
||||
assertEquals(1, counts.skipped(), "1 行跳过");
|
||||
verify(taskResultItemMapper, never()).upsertBatch(anyList());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_050_task_normal_multiple_items() {
|
||||
// 批量场景：75 行（8 批次），顺序稳定不丢失，每批数量正确。
|
||||
|
||||
+35
@@ -54,6 +54,41 @@ class SimilarAsinTaskServiceTest {
|
||||
assertEquals("\u6210\u529f", SimilarAsinTaskService.resolveResultStatus(withVisibleResultData, "", "80%", "", "", ""));
|
||||
}
|
||||
|
||||
@Test
|
||||
void resultStatusMarksAsinNotFoundRows() {
|
||||
// Python \u7aef 404 \u884c\uff1aerror="404 Not Found"\uff0cstatus \u88ab markRowFailed \u7f6e\u4e3a FAILED\uff0c
|
||||
// \u4f46\u72b6\u6001\u5217\u5fc5\u987b\u663e\u793a"ASIN\u672a\u627e\u5230"\uff0c\u4e0d\u80fd\u843d\u56de\u666e\u901a"\u5931\u8d25"\u3002
|
||||
SimilarAsinResultRowDto notFound = new SimilarAsinResultRowDto();
|
||||
notFound.setError("404 Not Found");
|
||||
notFound.setStatus("FAILED");
|
||||
notFound.setReason("required field missing (asin/title/url), skip LLM");
|
||||
assertEquals("ASIN\u672a\u627e\u5230", SimilarAsinTaskService.resolveResultStatus(notFound));
|
||||
|
||||
// \u5176\u5b83\u7b49\u4ef7\u6587\u6848\u540c\u6837\u547d\u4e2d
|
||||
SimilarAsinResultRowDto chinese = new SimilarAsinResultRowDto();
|
||||
chinese.setError("\u5546\u54c1\u672a\u627e\u5230");
|
||||
assertEquals("ASIN\u672a\u627e\u5230", SimilarAsinTaskService.resolveResultStatus(chinese));
|
||||
|
||||
// \u666e\u901a\u6280\u672f\u6027\u5931\u8d25\u4e0d\u8bef\u5224\uff1aerror \u4e0d\u542b 404/not found
|
||||
SimilarAsinResultRowDto llmFailure = new SimilarAsinResultRowDto();
|
||||
llmFailure.setError("LLM \u8c03\u7528\u8d85\u65f6");
|
||||
llmFailure.setStatus("FAILED");
|
||||
assertEquals("\u5931\u8d25", SimilarAsinTaskService.resolveResultStatus(llmFailure));
|
||||
}
|
||||
|
||||
@Test
|
||||
void asinNotFoundDetectionMatchesPythonErrors() {
|
||||
assertTrue(SimilarAsinTaskService.isAsinNotFound("404 Not Found"));
|
||||
assertTrue(SimilarAsinTaskService.isAsinNotFound("404 not found"));
|
||||
assertTrue(SimilarAsinTaskService.isAsinNotFound("404"));
|
||||
assertTrue(SimilarAsinTaskService.isAsinNotFound("\u5546\u54c1\u672a\u627e\u5230"));
|
||||
assertTrue(SimilarAsinTaskService.isAsinNotFound("ASIN \u672a\u627e\u5230"));
|
||||
assertFalse(SimilarAsinTaskService.isAsinNotFound(null));
|
||||
assertFalse(SimilarAsinTaskService.isAsinNotFound(""));
|
||||
assertFalse(SimilarAsinTaskService.isAsinNotFound("LLM \u8c03\u7528\u8d85\u65f6"));
|
||||
assertFalse(SimilarAsinTaskService.isAsinNotFound("\u4e0a\u4f20\u5931\u8d25"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void fileBuildProgressIsTerminalOnlyAfterTaskAndStageComplete() {
|
||||
assertTrue(SimilarAsinTaskService.isTerminalFileBuildProgress("SUCCESS", 3, 3));
|
||||
|
||||
Reference in New Issue
Block a user