From b11bc599cdc5a1b15da2a79aec52b12abe60dd10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E8=87=AA=E8=BE=BE?= <980324341@qq.com> Date: Mon, 19 Oct 2026 02:31:42 +0800 Subject: [PATCH] =?UTF-8?q?test(phase-3):=203B=20=E8=BF=81=E7=A7=BB?= =?UTF-8?q?=E5=89=8D=E6=8A=8A=20shopdatacrawl=20=E7=9A=84=E5=BB=BA?= =?UTF-8?q?=E5=8D=95=E4=B8=8E=E5=9B=9E=E4=BC=A0=E4=B8=A4=E6=AE=B5=E9=92=89?= =?UTF-8?q?=E6=88=90=20golden?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 蓝图 §4 Phase 3 首迁实录写的开工前置第①条——「那五支的 submitResult 同样没有 golden, 各自补完才能动」。本笔只补件、不动生产码一支。 回传段 20 例、建单段 14 例,渲染的是副作用**先后**流水而不是分组清单,三格专门钉住 Phase 2/3 已登记的「不许顺手统一」: - ownershipCheckedBeforeTerminal:本支先查归属再判终态(底座 admitSubmission 默认序相反), 套错就 40903 → 「任务已结束」; - terminalTaskRejectedWithoutCode:终态重复提交本支不带 40901,线协议与孪生两支不同; - completedRowFlushesWorkbookColumns:收尾刷结果列走 LambdaUpdateWrapper 批量条件更新 (只刷 success=1 的行、不写 success/errorMessage、objectKey 传 null),publishResultFile 不适用。 建单段另钉一格快照本来看不见的性质:建单时刻一枚复用给头行与每一行结果行,故只断 createdAtEqualsTaskCreatedAt=true 而不打印时钟——底座五参 newTaskHeader 是「逐行各取一次」那一派, 照搬就把这格改成 false。 --- .../ShopDataCrawlCreateTaskSnapshotTest.java | 487 +++++++++++ ...ShopDataCrawlSubmitResultSnapshotTest.java | 808 ++++++++++++++++++ .../golden/create-task-snapshot.txt | 63 ++ .../golden/submit-result-snapshot.txt | 138 +++ 4 files changed, 1496 insertions(+) create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/support/ShopDataCrawlCreateTaskSnapshotTest.java create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/support/ShopDataCrawlSubmitResultSnapshotTest.java create mode 100644 backend-java/src/test/resources/shopdatacrawl/golden/create-task-snapshot.txt create mode 100644 backend-java/src/test/resources/shopdatacrawl/golden/submit-result-snapshot.txt diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/support/ShopDataCrawlCreateTaskSnapshotTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/support/ShopDataCrawlCreateTaskSnapshotTest.java new file mode 100644 index 000000000..8a86f4d80 --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/support/ShopDataCrawlCreateTaskSnapshotTest.java @@ -0,0 +1,487 @@ +package com.nanri.aiimage.modules.shopdatacrawl.service.support; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.json.JsonMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.config.InstanceMetadata; +import com.nanri.aiimage.config.TaskPressureProperties; +import com.nanri.aiimage.modules.file.service.oss.OssStorageService; +import com.nanri.aiimage.modules.shopdatacrawl.mapper.ShopDataCrawlCountryPrefMapper; +import com.nanri.aiimage.modules.shopdatacrawl.mapper.ShopDataCrawlShopCandidateMapper; +import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCreateTaskRequest; +import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlTaskItemDto; +import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlCreateTaskVo; +import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo; +import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlDailyFileService; +import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlExcelAssemblyService; +import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlItemStoreService; +import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlResolveService; +import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskCacheService; +import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskService; +import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort; +import com.nanri.aiimage.modules.shopkey.service.ShopManageService; +import com.nanri.aiimage.modules.task.mapper.FileResultMapper; +import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; +import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper; +import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper; +import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.service.TaskDistributedLockService; +import com.nanri.aiimage.modules.task.service.TaskFileJobService; +import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService; +import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler; +import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService; +import com.nanri.aiimage.modules.task.service.TaskResultItemService; +import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService; +import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoShopMatchResultVo; +import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * 店铺数据抓取建单(段 1 落库形状)的现网行为快照——Phase 3 迁移 3B 的另一半护身符。 + * + *

为什么非但要有回传快照、还要有建单快照。3B 要动的不只是 {@code submitResult}: + * 建单那截的 task 头行与结果行是底座 {@code newTaskHeader}/{@code newScopeRow} 的直接替换点。 + * 而本支有一个快照本来量不到的性质——建单时刻 {@code LocalDateTime.now()} 被一枚复用给 + * 头行和每一行结果行({@code ShopDataCrawlTaskService:574 → :586-587 / :605})。时钟不能进快照 + * (进了就逐字节对撞不了),所以这里只断言相等关系:每行渲染一枚 + * {@code createdAtEqualsTaskCreatedAt=}。底座五参版 {@code newTaskHeader} 取的是「默认时区、逐行各取一次」 + * 那一派,照搬就会把这格从 {@code true} 改成 {@code false}——这条只有这一处量得到。 + * + *

另外三格被钉住的本支现网形状: + *

+ * + *

国家代码校验走的是 {@link ShopDataCrawlResolveService#validateCountryCodes} 真实实现 + * (那段是 static、不碰库),店铺匹配则按现网出口逐例打桩——快照要锁的是本支怎么用这些件, + * 不是替它们重写一遍判据。 + * + * golden:src/test/resources/shopdatacrawl/golden/create-task-snapshot.txt + * 重新生成:{@code SHUFU_SNAPSHOT_WRITE=true mvn -o test -Dtest=ShopDataCrawlCreateTaskSnapshotTest} + */ +class ShopDataCrawlCreateTaskSnapshotTest { + + private static final File GOLDEN_CREATE = + new File("src/test/resources/shopdatacrawl/golden/create-task-snapshot.txt"); + private static final String MODULE_TYPE = "SHOP_DATA_CRAWL"; + + /** Spring 注入的那只 ObjectMapper ≠ {@code new ObjectMapper()}:Boot 会挂上 jsr310(Phase 0 实测教训)。 */ + private static ObjectMapper bootLikeObjectMapper() { + return JsonMapper.builder() + .addModule(new JavaTimeModule()) + .disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS) + .build(); + } + + // ------------------------------------------------------------------ + // 夹具 + // ------------------------------------------------------------------ + + private static final class Fixture { + private final ObjectMapper objectMapper = bootLikeObjectMapper(); + private final List events = new ArrayList<>(); + private final List taskInserts = new ArrayList<>(); + private final List resultInserts = new ArrayList<>(); + private ShopDataCrawlTaskService service; + private int heartbeats; + /** {@code >=0} 时,第 N+1 次归一化起返回空串——用来够到现网建单循环里那道二次归一化空白闸。 */ + private int blankNormalizeAfter = -1; + private int normalizeCalls; + /** 打桩 {@code requireMatchedShop} 的出口:null 表示匹配成功,否则按该文案抛错。 */ + private String matchFailure; + + private ShopDataCrawlTaskService service() { + if (service != null) { + return service; + } + FileTaskMapper fileTaskMapper = mock(FileTaskMapper.class); + FileResultMapper fileResultMapper = mock(FileResultMapper.class); + ShopDataCrawlTaskCacheService cacheService = mock(ShopDataCrawlTaskCacheService.class); + ZiniaoShopSwitchService ziniao = mock(ZiniaoShopSwitchService.class); + TaskResultItemService resultItemService = mock(TaskResultItemService.class); + TaskProgressSnapshotService progressSnapshotService = mock(TaskProgressSnapshotService.class); + InstanceMetadata instanceMetadata = mock(InstanceMetadata.class); + + // 国家代码判据走真实实现(static、不碰库),不为它另造一套形状 + ShopDataCrawlResolveService realResolve = new ShopDataCrawlResolveService( + mock(ShopDataCrawlShopCandidateMapper.class), mock(ShopDataCrawlCountryPrefMapper.class), + bootLikeObjectMapper(), ziniao, mock(ShopManageService.class)); + ShopDataCrawlResolveService resolveService = mock(ShopDataCrawlResolveService.class); + when(resolveService.validateCountryCodes(any())) + .thenAnswer(invocation -> realResolve.validateCountryCodes(invocation.getArgument(0))); + when(resolveService.requireMatchedShop(anyString())).thenAnswer(invocation -> { + if (matchFailure != null) { + throw new BusinessException(matchFailure); + } + String shopName = invocation.getArgument(0); + return new ZiniaoShopMatchResultVo(true, "shop-" + shopName.trim(), shopName, + "公司-" + shopName.trim(), "Amazon", 9L, "https://example.invalid", + "MATCHED", "店铺索引匹配成功"); + }); + + doAnswer(invocation -> { + FileTaskEntity entity = invocation.getArgument(0); + entity.setId(1001L + taskInserts.size()); + taskInserts.add(entity); + events.add("task.insert: " + renderTaskInsert(entity)); + return 1; + }).when(fileTaskMapper).insert(any(FileTaskEntity.class)); + doAnswer(invocation -> { + events.add("task.update: " + renderTaskUpdate(invocation.getArgument(0))); + return 1; + }).when(fileTaskMapper).updateById(any(FileTaskEntity.class)); + doAnswer(invocation -> { + FileResultEntity entity = invocation.getArgument(0); + entity.setId(2001L + resultInserts.size()); + resultInserts.add(entity); + events.add("result.insert: " + renderResultInsert(entity, + taskInserts.isEmpty() ? null : taskInserts.get(0))); + return 1; + }).when(fileResultMapper).insert(any(FileResultEntity.class)); + + doAnswer(invocation -> { + heartbeats++; + return null; + }).when(cacheService).touchTaskHeartbeat(any()); + doAnswer(invocation -> { + events.add("taskCacheSave: id=" + ((FileTaskEntity) invocation.getArgument(0)).getId()); + return null; + }).when(cacheService).saveTaskCache(any(FileTaskEntity.class)); + doAnswer(invocation -> { + events.add("snapshotTable.replace: taskId=" + invocation.getArgument(0) + + " items=" + ((List) invocation.getArgument(2)).size()); + return null; + }).when(resultItemService).replaceTaskSnapshots(anyLong(), anyString(), any(), any()); + doAnswer(invocation -> { + events.add("progressSave: taskId=" + invocation.getArgument(0) + + " status=" + dash(invocation.getArgument(2)) + + " total=" + invocation.getArgument(3) + + " ok=" + invocation.getArgument(4) + + " fail=" + invocation.getArgument(5)); + return null; + }).when(progressSnapshotService).save(anyLong(), anyString(), anyString(), + org.mockito.ArgumentMatchers.anyInt(), org.mockito.ArgumentMatchers.anyInt(), + org.mockito.ArgumentMatchers.anyInt(), any(), any(), any()); + when(instanceMetadata.getInstanceId()).thenReturn("instance-a"); + when(ziniao.normalizeShopName(any())).thenAnswer(invocation -> { + String shopName = invocation.getArgument(0); + normalizeCalls++; + if (blankNormalizeAfter >= 0 && normalizeCalls > blankNormalizeAfter) { + return ""; + } + return shopName == null ? "" : shopName.trim(); + }); + + service = new ShopDataCrawlTaskService( + fileTaskMapper, + fileResultMapper, + resolveService, + mock(ShopDataCrawlExcelAssemblyService.class), + cacheService, + mock(OssStorageService.class), + ziniao, + objectMapper, + mock(TaskPressureProperties.class), + mock(TaskFileJobService.class), + resultItemService, + progressSnapshotService, + mock(TaskDistributedLockService.class), + mock(TaskChunkMapper.class), + mock(TaskScopeStateMapper.class), + mock(TransientPayloadStorageService.class), + instanceMetadata, + mock(ShopDataCrawlDailyFileService.class), + mock(ShopDataCrawlItemStoreService.class), + mock(DuplicateCheckRefreshPort.class), + null, + mock(TaskProgressLightAssembler.class), + mock(TaskHeartbeatPositionService.class)); + return service; + } + } + + // ------------------------------------------------------------------ + // 渲染(时钟/雪花 id 不进快照,只断相等关系) + // ------------------------------------------------------------------ + + private static String renderTaskInsert(FileTaskEntity task) { + return "id=" + task.getId() + + " taskNoPrefix=" + prefix(task.getTaskNo()) + + " moduleType=" + dash(task.getModuleType()) + + " taskMode=" + dash(task.getTaskMode()) + + " status=" + dash(task.getStatus()) + + " sourceFileCount=" + task.getSourceFileCount() + + " ok=" + task.getSuccessFileCount() + + " fail=" + task.getFailedFileCount() + + " userId=" + dash(task.getUserId()) + + " createdBy=" + dash(task.getCreatedBy()) + + " deviceId=" + dash(task.getDeviceId()) + + " ownerInstanceId=" + dash(task.getOwnerInstanceId()) + + " requestJsonPresent=" + (task.getRequestJson() != null) + + " resultJsonPresent=" + (task.getResultJson() != null); + } + + private static String renderTaskUpdate(FileTaskEntity task) { + return "id=" + task.getId() + + " status=" + dash(task.getStatus()) + + " ok=" + task.getSuccessFileCount() + + " fail=" + task.getFailedFileCount() + + " requestJsonHasOwner=" + (task.getRequestJson() != null + && task.getRequestJson().contains("\"ownerInstanceId\":\"instance-a\"")) + + " requestJsonItems=" + (task.getRequestJson() == null ? -1 + : count(task.getRequestJson(), "\"shopName\"")) + + " resultJsonItems=" + (task.getResultJson() == null ? -1 + : count(task.getResultJson(), "\"resultId\"")) + + " createdAtEqualsUpdatedAt=" + (task.getCreatedAt() != null + && task.getCreatedAt().equals(task.getUpdatedAt())); + } + + private static String renderResultInsert(FileResultEntity row, FileTaskEntity task) { + return "id=" + row.getId() + + " taskId=" + dash(row.getTaskId()) + + " moduleType=" + dash(row.getModuleType()) + + " sourceFilename=" + dash(row.getSourceFilename()) + + " sourceFileUrl=" + dash(row.getSourceFileUrl()) + + " userId=" + dash(row.getUserId()) + + " success=" + dash(row.getSuccess()) + + " rowCount=" + dash(row.getRowCount()) + + " createdAtEqualsTaskCreatedAt=" + (task != null && row.getCreatedAt() != null + && row.getCreatedAt().equals(task.getCreatedAt())); + } + + private static String prefix(String taskNo) { + if (taskNo == null) { + return "-"; + } + int index = taskNo.lastIndexOf('-'); + return index < 0 ? taskNo : taskNo.substring(0, index); + } + + private static String dash(Object value) { + String text = value == null ? "" : String.valueOf(value); + return text.isBlank() ? "-" : text; + } + + private static int count(String text, String needle) { + int occurrences = 0; + int index = text.indexOf(needle); + while (index >= 0) { + occurrences++; + index = text.indexOf(needle, index + needle.length()); + } + return occurrences; + } + + // ------------------------------------------------------------------ + // 请求构造 + // ------------------------------------------------------------------ + + private static ShopDataCrawlTaskItemDto shop(String shopName) { + ShopDataCrawlTaskItemDto item = new ShopDataCrawlTaskItemDto(); + item.setShopName(shopName); + item.setShopId("请求里带的旧 id"); + item.setPlatform("请求里带的旧平台"); + item.setCompanyName("请求里带的旧公司"); + item.setMatchStatus("请求里带的旧状态"); + item.setMatchMessage("请求里带的旧说明"); + return item; + } + + private static ShopDataCrawlCreateTaskRequest request(Long userId, + List items, + List countryCodes) { + ShopDataCrawlCreateTaskRequest request = new ShopDataCrawlCreateTaskRequest(); + request.setUserId(userId); + request.setItems(items); + request.setCountryCodes(countryCodes); + return request; + } + + private static ShopDataCrawlCreateTaskRequest oneShop(Long userId, String shopName, List countryCodes) { + return request(userId, new ArrayList<>(List.of(shop(shopName))), countryCodes); + } + + // ------------------------------------------------------------------ + // 用例 + // ------------------------------------------------------------------ + + private static String runCase(String caseName, Fixture fixture, ShopDataCrawlCreateTaskRequest request) { + StringBuilder sb = new StringBuilder(); + sb.append("== case=").append(caseName).append('\n'); + try { + ShopDataCrawlCreateTaskVo vo = fixture.service().createTask(request, "dev-1"); + sb.append("vo: taskId=").append(dash(vo.getTaskId())) + .append(" items=").append(vo.getItems().size()).append('\n'); + for (ShopDataCrawlResultItemVo item : vo.getItems()) { + sb.append(" item: resultId=").append(dash(item.getResultId())) + .append(" taskId=").append(dash(item.getTaskId())) + .append(" shopName=").append(dash(item.getShopName())) + .append(" shopId=").append(dash(item.getShopId())) + .append(" platform=").append(dash(item.getPlatform())) + .append(" companyName=").append(dash(item.getCompanyName())) + .append(" matched=").append(item.isMatched()) + .append(" matchStatus=").append(dash(item.getMatchStatus())) + .append(" taskStatus=").append(dash(item.getTaskStatus())) + .append(" success=").append(dash(item.getSuccess())) + .append(" error=").append(dash(item.getError())) + .append(" countryCodes=").append(item.getCountryCodes()) + .append(" countryResults=").append(item.getCountryResults() == null ? -1 + : item.getCountryResults().size()) + .append(" downloadUrl=").append(dash(item.getDownloadUrl())) + .append('\n'); + } + } catch (Exception ex) { + sb.append("thrown: ").append(ex.getClass().getSimpleName()) + .append(" code=").append(ex instanceof BusinessException businessException + ? dash(businessException.getCode()) : "-") + .append(" message=").append(dash(ex.getMessage())).append('\n'); + } + sb.append("events=").append(fixture.events.size()) + .append(" taskInserts=").append(fixture.taskInserts.size()) + .append(" resultInserts=").append(fixture.resultInserts.size()) + .append(" heartbeatTouches=").append(fixture.heartbeats).append('\n'); + fixture.events.forEach(line -> sb.append(" evt: ").append(line).append('\n')); + return sb.toString(); + } + + public static String runAll() { + StringBuilder sb = new StringBuilder(); + + // 1. 单店两国家:头行先落(拿到自增主键)→ 缓存+心跳 → 结果行 → requestJson/resultJson 一起刷回 + Fixture happy = new Fixture(); + sb.append(runCase("happySingleShopTwoCountries", happy, + oneShop(7L, " Demo Shop ", List.of("UK", "DE")))); + + // 2. 请求里带的 shopId/platform/company 一律以紫鸟重新匹配到的为准(这几列会被覆盖) + Fixture overwrite = new Fixture(); + sb.append(runCase("requestFieldsOverwrittenByMatch", overwrite, + oneShop(7L, "Demo Shop", List.of("UK")))); + + // 3. items 两枚:本支的合同是「必须且只能一个店铺」,且不碰库 + Fixture twoItems = new Fixture(); + sb.append(runCase("twoItemsRejected", twoItems, + request(7L, new ArrayList<>(List.of(shop("店铺A"), shop("店铺B"))), List.of("UK")))); + + // 4. items 空列表 + Fixture emptyItems = new Fixture(); + sb.append(runCase("emptyItemsRejected", emptyItems, + request(7L, new ArrayList<>(), List.of("UK")))); + + // 5. items 为 null + Fixture nullItems = new Fixture(); + sb.append(runCase("nullItemsRejected", nullItems, + request(7L, null, List.of("UK")))); + + // 6. 两枚同店名:第一道闸数的是原始 items(:552),dedupe(:557)根本没机会救它 + // ⇒「必须且只能包含一个有效店铺」那句只在 dedupe 把唯一一枚丢光时才到得了(下一例) + Fixture duplicatedShops = new Fixture(); + sb.append(runCase("rawTwoItemsSameShopRejectedFirstGate", duplicatedShops, + request(7L, new ArrayList<>(List.of(shop("店铺A"), shop(" 店铺A "))), List.of("UK")))); + + // 7. 店名归一化后为空:去重阶段就被丢掉 + Fixture blankShop = new Fixture(); + sb.append(runCase("blankShopNameDropped", blankShop, + oneShop(7L, " ", List.of("UK")))); + + // 8. 建单循环里二次归一化变空(夹具把第 3 次调用打空):头行已落库、结果行一行都不写 + Fixture renormalizedBlank = new Fixture(); + renormalizedBlank.blankNormalizeAfter = 2; + sb.append(runCase("reNormalizedToBlankRejects", renormalizedBlank, + oneShop(7L, "Demo Shop", List.of("UK")))); + + // 9. 紫鸟匹配不上:requireMatchedShop 直接抛,建单不留下任何行 + Fixture unmatched = new Fixture(); + unmatched.matchFailure = "店铺未匹配"; + sb.append(runCase("unmatchedShopRejected", unmatched, + oneShop(7L, "Demo Shop", List.of("UK")))); + + // 10~12. 国家代码三格判据(真实实现):非法、重复、空项/至少一个 + Fixture usCountry = new Fixture(); + sb.append(runCase("usIsNotSupportedCountry", usCountry, + oneShop(7L, "Demo Shop", List.of("US")))); + Fixture duplicateCountry = new Fixture(); + sb.append(runCase("duplicateCountryCodeRejected", duplicateCountry, + oneShop(7L, "Demo Shop", List.of("uk", "UK")))); + Fixture emptyCountries = new Fixture(); + sb.append(runCase("emptyCountryCodesRejected", emptyCountries, + oneShop(7L, "Demo Shop", List.of()))); + Fixture blankCountry = new Fixture(); + sb.append(runCase("blankCountryCodeRejected", blankCountry, + oneShop(7L, "Demo Shop", Arrays.asList("UK", " ")))); + + // 13~14. userId 两格(null 与 0)都走同一句「user_id 不合法」 + Fixture nullUser = new Fixture(); + sb.append(runCase("nullUserIdRejected", nullUser, + oneShop(null, "Demo Shop", List.of("UK")))); + Fixture zeroUser = new Fixture(); + sb.append(runCase("zeroUserIdRejected", zeroUser, + oneShop(0L, "Demo Shop", List.of("UK")))); + + return sb.toString(); + } + + private static String read(File file) throws Exception { + return Files.readString(file.toPath(), StandardCharsets.UTF_8).replace("\r\n", "\n"); + } + + @Test + void create_task_snapshot_matches_golden() throws Exception { + assertEquals(read(GOLDEN_CREATE), runAll(), "建单落库流水与 golden 一致"); + } + + @Test + void create_task_snapshot_is_reproducible() { + assertEquals(runAll(), runAll(), "同一夹具跑两次流水一致(时钟与雪花 id 不得进快照)"); + } + + @Test + void create_task_snapshot_golden_is_committed() { + assertTrue(GOLDEN_CREATE.isFile(), "golden 文件必须存在并入库: " + GOLDEN_CREATE.getAbsolutePath()); + assertTrue(GOLDEN_CREATE.length() > 0, "golden 文件非空"); + } + + @Test + void create_task_snapshot_tamper_is_detected() throws Exception { + String original = read(GOLDEN_CREATE); + assertTrue(original.contains("createdAtEqualsTaskCreatedAt=true"), + "golden 必须钉住「建单时刻一枚复用给头行与每一行」这条底座量不到的形状"); + try { + Files.writeString(GOLDEN_CREATE.toPath(), original + "\n# tampered", StandardCharsets.UTF_8); + assertTrue(!read(GOLDEN_CREATE).equals(runAll()), "篡改 golden 后断言应失败"); + } finally { + Files.writeString(GOLDEN_CREATE.toPath(), original, StandardCharsets.UTF_8); + } + assertEquals(original, read(GOLDEN_CREATE), "恢复原始 golden"); + } + + /** 只在显式点名时写 golden:默认跑测试永远不许改是非源。 */ + @Test + @EnabledIfEnvironmentVariable(named = "SHUFU_SNAPSHOT_WRITE", matches = "true") + void write_golden_on_demand() throws Exception { + Files.createDirectories(GOLDEN_CREATE.getParentFile().toPath()); + Files.writeString(GOLDEN_CREATE.toPath(), runAll(), StandardCharsets.UTF_8); + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/support/ShopDataCrawlSubmitResultSnapshotTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/support/ShopDataCrawlSubmitResultSnapshotTest.java new file mode 100644 index 000000000..6b0aad102 --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/support/ShopDataCrawlSubmitResultSnapshotTest.java @@ -0,0 +1,808 @@ +package com.nanri.aiimage.modules.shopdatacrawl.service.support; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; +import com.baomidou.mybatisplus.core.metadata.TableInfoHelper; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.json.JsonMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.nanri.aiimage.config.InstanceMetadata; +import com.nanri.aiimage.config.TaskPressureProperties; +import com.nanri.aiimage.modules.file.service.oss.OssStorageService; +import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryResultDto; +import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto; +import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto; +import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResultRequest; +import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlDailyFileService; +import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlExcelAssemblyService; +import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlItemStoreService; +import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlResolveService; +import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskCacheService; +import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskService; +import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort; +import com.nanri.aiimage.modules.task.mapper.FileResultMapper; +import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; +import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper; +import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper; +import com.nanri.aiimage.modules.task.model.entity.FileResultEntity; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity; +import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity; +import com.nanri.aiimage.modules.task.service.TaskDistributedLockService; +import com.nanri.aiimage.modules.task.service.TaskFileJobService; +import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService; +import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler; +import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService; +import com.nanri.aiimage.modules.task.service.TaskResultItemService; +import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService; +import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService; +import org.apache.ibatis.builder.MapperBuilderAssistant; +import org.apache.ibatis.session.Configuration; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; +import org.springframework.dao.DuplicateKeyException; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.nullable; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * 店铺数据抓取回传落库(段 3)的现网行为快照——Phase 3 迁移 3B 的护身符。 + * + *

为什么补这一支。蓝图 §4 Phase 3 首迁实录把话说死了:「那五支的 {@code submitResult} + * 同样没有 golden ⇒ 各自补完才能动」。{@code ShopDataCrawlSnapshotTest} 锁的是 Excel 分组与行布局, + * 一行库都没碰;建单与回传两段此前只有断言式用例({@code ShopDataCrawlTaskServiceChunkTest}), + * 那些用例断的是结论({@code result.getSuccess()==1}),不锁副作用先后。而 3B 要改的 + * 恰恰是先后:准入步序、合并与标记的顺序、终态那格的收尾链。所以这一支必须落在迁移之前。 + * + *

这一支同时钉住 3B 的三处「不许顺手统一」(Phase 2/3 逐条登记过的差异,代码注释里也写着): + *

    + *
  1. {@code caseOwnershipCheckedBeforeTerminal}:本支现网先查归属再判终态,与底座 + * {@code admitSubmission} 的默认序相反。若迁移时改成「直接套底座那一派」, + * 同一枚既被别人持有又已终态的任务会从 {@code TaskOwnerMismatchException}(40903) + * 变成「任务已结束,拒绝重复提交」——本例当场判红。
  2. + *
  3. {@code caseTerminalTaskRejectedWithoutCode}:终态重复提交本支不带 + * {@code TASK_ALREADY_FINISHED}(40901),线协议与 ShopMatch/PriceTrack 那两支不同 + * (那两支是 success=true、Python 停止重试;本支是 success=false)。 + * golden 里那行 {@code code=-} 就是这件事的唯一读数。
  4. + *
  5. {@code caseCompletedRowFlushesWorkbookColumns}:终态收尾的结果列写走 + * {@code LambdaUpdateWrapper} 批量条件更新(只刷 {@code success=1} 的行、不写 + * {@code success}/{@code errorMessage}、{@code objectKey} 传 null), + * 底座 {@code publishResultFile} 对它不适用。本例把「一条 batchUpdate、零条 + * {@code result.update}」这个形状钉住,套错件就红。
  6. + *
+ * + *

快照吃的是事件流水,不是分组清单。顺序换了就红。时钟、雪花 id、OSS 直链不进快照; + * 分片内容哈希取前 12 位(既灵敏又不把 golden 变成一堵十六进制墙)。 + * + * golden:src/test/resources/shopdatacrawl/golden/submit-result-snapshot.txt + * 重新生成:{@code SHUFU_SNAPSHOT_WRITE=true mvn -o test -Dtest=ShopDataCrawlSubmitResultSnapshotTest} + * (用环境变量而不是 -D:pom 里 {@code } 是显式配置,命令行覆盖不动它。 + * 这条 golden 只许在改动本支之前生成一次,此后它就是是非源) + */ +class ShopDataCrawlSubmitResultSnapshotTest { + + private static final File GOLDEN_SUBMIT = + new File("src/test/resources/shopdatacrawl/golden/submit-result-snapshot.txt"); + private static final String MODULE_TYPE = "SHOP_DATA_CRAWL"; + private static final String SHOP_NAME = "Demo Shop"; + private static final Long TASK_ID = 1001L; + + @BeforeAll + static void initializeMybatisMetadata() { + MapperBuilderAssistant assistant = new MapperBuilderAssistant(new Configuration(), ""); + TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class); + TableInfoHelper.initTableInfo(assistant, FileResultEntity.class); + TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class); + TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class); + } + + /** Spring 注入的那只 ObjectMapper ≠ {@code new ObjectMapper()}:Boot 会挂上 jsr310(Phase 0 实测教训)。 */ + private static ObjectMapper bootLikeObjectMapper() { + return JsonMapper.builder() + .addModule(new JavaTimeModule()) + .disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS) + .build(); + } + + // ------------------------------------------------------------------ + // 夹具:一份内存世界(结果行/合并载荷/分片表/scope/RustFS),每次用例全新建一套 + // ------------------------------------------------------------------ + + private static final class Fixture { + private final ObjectMapper objectMapper = bootLikeObjectMapper(); + private final List events = new ArrayList<>(); + private final Map mergedPayloads = new LinkedHashMap<>(); + private final List rows = new ArrayList<>(); + private final List chunks = new ArrayList<>(); + private final List scopes = new ArrayList<>(); + private final Map rustfsPayloads = new LinkedHashMap<>(); + private final FileTaskEntity task = new FileTaskEntity(); + private ShopDataCrawlTaskService service; + private int heartbeats; + private int lockCloses; + private int nextPayloadId; + private boolean sharedWriteEnabled = true; + + private Fixture withRows(String... shopNames) { + long id = 2001L; + for (String shopName : shopNames) { + FileResultEntity row = new FileResultEntity(); + row.setId(id++); + row.setTaskId(TASK_ID); + row.setModuleType(MODULE_TYPE); + row.setSourceFilename(shopName); + row.setSourceFileUrl("shop-1"); + row.setUserId(7L); + row.setSuccess(-1); + row.setCreatedAt(LocalDateTime.now()); + rows.add(row); + } + return this; + } + + /** {@code owner} 传 null 表示 requestJson 里不带 ownerInstanceId(走「未绑定实例」分支)。 */ + private Fixture withTask(String status, String moduleType, String owner) { + task.setId(TASK_ID); + task.setUserId(7L); + task.setModuleType(moduleType); + task.setStatus(status); + task.setRequestJson(owner == null ? "{}" : "{\"ownerInstanceId\":\"" + owner + "\"}"); + task.setResultJson("[]"); + task.setCreatedAt(LocalDateTime.now()); + task.setUpdatedAt(LocalDateTime.now()); + return this; + } + + private ShopDataCrawlTaskService service() { + if (service != null) { + return service; + } + FileTaskMapper fileTaskMapper = mock(FileTaskMapper.class); + FileResultMapper fileResultMapper = mock(FileResultMapper.class); + ShopDataCrawlResolveService resolveService = mock(ShopDataCrawlResolveService.class); + ShopDataCrawlExcelAssemblyService assembly = mock(ShopDataCrawlExcelAssemblyService.class); + ShopDataCrawlTaskCacheService cacheService = mock(ShopDataCrawlTaskCacheService.class); + ZiniaoShopSwitchService ziniao = mock(ZiniaoShopSwitchService.class); + TaskFileJobService fileJobService = mock(TaskFileJobService.class); + TaskResultItemService resultItemService = mock(TaskResultItemService.class); + TaskProgressSnapshotService progressSnapshotService = mock(TaskProgressSnapshotService.class); + TaskDistributedLockService lockService = mock(TaskDistributedLockService.class); + TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class); + TaskChunkMapper chunkMapper = mock(TaskChunkMapper.class); + TaskScopeStateMapper scopeMapper = mock(TaskScopeStateMapper.class); + TransientPayloadStorageService payloadStorage = mock(TransientPayloadStorageService.class); + InstanceMetadata instanceMetadata = mock(InstanceMetadata.class); + + doAnswer(invocation -> { + events.add("result.update: " + renderRow(invocation.getArgument(0))); + return 1; + }).when(fileResultMapper).updateById(any(FileResultEntity.class)); + doAnswer(invocation -> { + events.add("task.update: " + renderTask(invocation.getArgument(0))); + return 1; + }).when(fileTaskMapper).updateById(any(FileTaskEntity.class)); + when(fileTaskMapper.selectById(anyLong())).thenReturn(task); + when(fileResultMapper.selectList(any())).thenAnswer(invocation -> List.copyOf(rows)); + when(fileTaskMapper.selectList(any())).thenAnswer(invocation -> List.of(task)); + + // 终态收尾那条「只刷成功行、不动状态位」的批量条件更新(底座 publishResultFile 不适用的那格) + doAnswer(invocation -> { + LambdaUpdateWrapper wrapper = invocation.getArgument(1); + events.add("result.batchUpdate: " + renderUpdateWrapper(wrapper)); + return 1; + }).when(fileResultMapper).update(any(), any()); + + when(lockService.acquire(eq(MODULE_TYPE), anyLong())).thenReturn(lock); + when(lockService.acquire(eq(MODULE_TYPE), anyLong(), anyLong())).thenReturn(lock); + doAnswer(invocation -> { + lockCloses++; + return null; + }).when(lock).close(); + + when(cacheService.getTaskCacheBatch(any())).thenReturn(Map.of()); + doAnswer(invocation -> { + heartbeats++; + return null; + }).when(cacheService).touchTaskHeartbeat(any()); + when(cacheService.getShopMergedPayload(anyLong(), anyString())) + .thenAnswer(invocation -> mergedPayloads.get(invocation.getArgument(1))); + when(cacheService.getAllShopMergedPayload(anyLong())).thenAnswer(invocation -> + new LinkedHashMap<>(mergedPayloads)); + doAnswer(invocation -> { + String shopKey = invocation.getArgument(1); + ShopDataCrawlShopPayloadDto payload = invocation.getArgument(2); + mergedPayloads.put(shopKey, payload); + events.add("mergedSave: taskId=" + invocation.getArgument(0) + + " shopKey=" + shopKey + " " + renderPayload(payload)); + return null; + }).when(cacheService).saveShopMergedPayload(anyLong(), anyString(), any()); + doAnswer(invocation -> { + events.add("mergedRemove: taskId=" + invocation.getArgument(0) + + " shopKey=" + invocation.getArgument(1)); + mergedPayloads.remove(invocation.getArgument(1)); + return null; + }).when(cacheService).removeShopMergedPayload(anyLong(), anyString()); + doAnswer(invocation -> { + events.add("taskCacheSave: id=" + ((FileTaskEntity) invocation.getArgument(0)).getId()); + return null; + }).when(cacheService).saveTaskCache(any(FileTaskEntity.class)); + doAnswer(invocation -> { + events.add("taskCacheDelete: taskId=" + invocation.getArgument(0)); + return null; + }).when(cacheService).deleteTaskCache(any()); + + doAnswer(invocation -> { + events.add("snapshotTable.replace: taskId=" + invocation.getArgument(0) + + " items=" + ((List) invocation.getArgument(2)).size()); + return null; + }).when(resultItemService).replaceTaskSnapshots(anyLong(), anyString(), any(), any()); + when(resultItemService.listResultSnapshots(anyLong(), anyString(), any())).thenReturn(List.of()); + doAnswer(invocation -> { + events.add("progressSave: taskId=" + invocation.getArgument(0) + + " status=" + dash(invocation.getArgument(2)) + + " total=" + invocation.getArgument(3) + + " ok=" + invocation.getArgument(4) + + " fail=" + invocation.getArgument(5)); + return null; + }).when(progressSnapshotService).save(anyLong(), anyString(), anyString(), + anyInt(), anyInt(), anyInt(), any(), any(), any()); + + when(fileJobService.findAssembleJob(anyLong(), anyString(), anyLong())).thenReturn(null); + when(fileJobService.findAssembleJobsByResultIds(anyString(), any())).thenReturn(Map.of()); + when(fileJobService.countUnfinishedAssembleJobs(anyLong(), anyString())).thenReturn(0L); + doAnswer(invocation -> { + events.add("enqueue: taskId=" + invocation.getArgument(0) + + " resultId=" + invocation.getArgument(2) + + " scopeKey=" + invocation.getArgument(3)); + return null; + }).when(fileJobService).enqueueAssembleResult(anyLong(), anyString(), anyLong(), anyString()); + + when(ziniao.normalizeShopName(nullable(String.class))).thenAnswer(invocation -> { + String shopName = invocation.getArgument(0); + return shopName == null ? "" : shopName.trim(); + }); + when(assembly.countRows(any())).thenAnswer(invocation -> + ((List) invocation.getArgument(0)).size()); + + configurePayloadStorage(payloadStorage); + configureChunkMapper(chunkMapper); + configureScopeMapper(scopeMapper); + + lenient().when(instanceMetadata.getInstanceId()).thenReturn("instance-a"); + + service = new ShopDataCrawlTaskService( + fileTaskMapper, + fileResultMapper, + resolveService, + assembly, + cacheService, + mock(OssStorageService.class), + ziniao, + objectMapper, + mock(TaskPressureProperties.class), + fileJobService, + resultItemService, + progressSnapshotService, + lockService, + chunkMapper, + scopeMapper, + payloadStorage, + instanceMetadata, + mock(ShopDataCrawlDailyFileService.class), + mock(ShopDataCrawlItemStoreService.class), + mock(DuplicateCheckRefreshPort.class), + null, + mock(TaskProgressLightAssembler.class), + mock(TaskHeartbeatPositionService.class)); + return service; + } + + private void configurePayloadStorage(TransientPayloadStorageService storage) { + lenient().when(storage.isSharedWriteEnabled()).thenReturn(sharedWriteEnabled); + lenient().when(storage.storeChunkPayloadVersioned(anyString(), anyLong(), anyString(), any(), anyString())) + .thenAnswer(invocation -> { + String pointer = "rustfs:payload-" + (++nextPayloadId); + rustfsPayloads.put(pointer, invocation.getArgument(4)); + events.add("payloadStore: pointer=" + pointer); + return pointer; + }); + lenient().when(storage.extractPointer(nullable(String.class))).thenAnswer(invocation -> { + String value = invocation.getArgument(0); + return value != null && value.startsWith("rustfs:") ? value : null; + }); + lenient().when(storage.resolvePayload(any(), any())).thenAnswer(invocation -> { + String pointer = invocation.getArgument(0); + events.add("payloadResolve: pointer=" + pointer); + String payload = rustfsPayloads.get(pointer); + if (payload == null) { + throw new IllegalStateException("missing test RustFS payload: " + pointer); + } + return payload; + }); + lenient().doAnswer(invocation -> { + String pointer = invocation.getArgument(0); + if (rustfsPayloads.containsKey(pointer)) { + events.add("payloadDelete: pointer=" + pointer); + rustfsPayloads.remove(pointer); + } + return null; + }).when(storage).deletePayloadIfPresent(any()); + } + + private void configureChunkMapper(TaskChunkMapper chunkMapper) { + lenient().when(chunkMapper.insert(any(TaskChunkEntity.class))).thenAnswer(invocation -> { + TaskChunkEntity chunk = invocation.getArgument(0); + boolean duplicate = chunks.stream().anyMatch(existing -> + Objects.equals(existing.getTaskId(), chunk.getTaskId()) + && Objects.equals(existing.getScopeHash(), chunk.getScopeHash()) + && Objects.equals(existing.getChunkIndex(), chunk.getChunkIndex())); + if (duplicate) { + throw new DuplicateKeyException("duplicate chunk key: " + + chunk.getScopeHash() + "/" + chunk.getChunkIndex()); + } + chunk.setId((long) chunks.size() + 1L); + chunks.add(chunk); + events.add("chunkInsert: idx=" + chunk.getChunkIndex() + "/" + chunk.getChunkTotal() + + " hash=" + shortHash(chunk.getPayloadHash()) + " scopeKey=" + chunk.getScopeKey()); + return 1; + }); + lenient().when(chunkMapper.selectOne(any())).thenAnswer(invocation -> { + LambdaQueryWrapper query = invocation.getArgument(0); + String scopeHash = queryHash(query); + Integer chunkIndex = queryInt(query); + return chunks.stream() + .filter(chunk -> Objects.equals(scopeHash, chunk.getScopeHash())) + .filter(chunk -> Objects.equals(chunkIndex, chunk.getChunkIndex())) + .findFirst().orElse(null); + }); + lenient().when(chunkMapper.selectList(any())).thenAnswer(invocation -> chunks.stream() + .sorted(Comparator.comparing(TaskChunkEntity::getChunkIndex)).toList()); + lenient().when(chunkMapper.selectCount(any())).thenAnswer(invocation -> (long) chunks.size()); + lenient().when(chunkMapper.delete(any())).thenAnswer(invocation -> { + int removed = chunks.size(); + if (removed > 0) { + events.add("chunkDelete: count=" + removed); + } + chunks.clear(); + return removed; + }); + } + + private void configureScopeMapper(TaskScopeStateMapper scopeMapper) { + lenient().when(scopeMapper.insert(any(TaskScopeStateEntity.class))).thenAnswer(invocation -> { + TaskScopeStateEntity scope = invocation.getArgument(0); + scope.setId((long) scopes.size() + 1L); + scopes.add(scope); + events.add("scopeInsert: " + renderScope(scope)); + return 1; + }); + lenient().when(scopeMapper.selectOne(any())).thenAnswer(invocation -> scopes.stream() + .filter(scope -> Objects.equals(queryHash(invocation.getArgument(0)), scope.getScopeHash())) + .findFirst().orElse(null)); + lenient().when(scopeMapper.updateById(any(TaskScopeStateEntity.class))).thenAnswer(invocation -> { + TaskScopeStateEntity scope = invocation.getArgument(0); + if (!scopes.contains(scope)) { + scopes.add(scope); + } + events.add("scopeUpdate: " + renderScope(scope)); + return 1; + }); + lenient().when(scopeMapper.delete(any())).thenAnswer(invocation -> { + int removed = scopes.size(); + if (removed > 0) { + events.add("scopeDelete: count=" + removed); + } + scopes.clear(); + return removed; + }); + } + } + + // ------------------------------------------------------------------ + // 渲染(时钟/雪花 id/OSS 直链不进快照) + // ------------------------------------------------------------------ + + private static String renderRow(FileResultEntity row) { + return "id=" + row.getId() + + " success=" + row.getSuccess() + + " error=" + dash(row.getErrorMessage()) + + " filename=" + dash(row.getResultFilename()) + + " urlPresent=" + (row.getResultFileUrl() != null && !row.getResultFileUrl().isBlank()) + + " size=" + row.getResultFileSize() + + " contentType=" + dash(row.getResultContentType()) + + " rowCount=" + row.getRowCount(); + } + + private static String renderTask(FileTaskEntity task) { + return "id=" + task.getId() + + " status=" + dash(task.getStatus()) + + " ok=" + task.getSuccessFileCount() + + " fail=" + task.getFailedFileCount() + + " sourceFileCount=" + task.getSourceFileCount() + + " error=" + dash(task.getErrorMessage()) + + " resultJsonItems=" + (task.getResultJson() == null ? -1 + : count(task.getResultJson(), "\"resultId\"")) + + " resultJsonHasRows=" + (task.getResultJson() != null + && task.getResultJson().contains("\"asin\"")) + + " finishedPresent=" + (task.getFinishedAt() != null); + } + + private static String renderScope(TaskScopeStateEntity scope) { + return "received=" + scope.getReceivedChunkCount() + + " total=" + scope.getChunkTotal() + + " completed=" + scope.getCompleted() + + " scopeKey=" + dash(scope.getScopeKey()); + } + + private static String renderPayload(ShopDataCrawlShopPayloadDto payload) { + if (payload == null) { + return "payload="; + } + int countries = payload.getCountryResults() == null ? 0 : payload.getCountryResults().size(); + int rows = 0; + if (payload.getCountryResults() != null) { + for (ShopDataCrawlCountryResultDto country : payload.getCountryResults()) { + rows += country == null || country.getItems() == null ? 0 : country.getItems().size(); + } + } + return "shopName=" + dash(payload.getShopName()) + + " error=" + dash(payload.getError()) + + " countries=" + countries + " rows=" + rows + + " done=" + payload.getShopDone() + + " chunk=" + payload.getChunkIndex() + "/" + payload.getChunkTotal(); + } + + /** + * 批量条件更新要认三件事:刷哪些列(顺序)、按哪些行(WHERE 原文)、带哪些值(按 {@code MPGENVAL} 序号)。 + * + *

WHERE 只能取原文:MyBatis-Plus 对单元素集合走 {@code formatSql("{0}", value)} 直接拼进 SQL、 + * 不进 {@code paramNameValuePairs},本支收尾只有首行那一枚 id,靠参数表读不到它。 + */ + private static String renderUpdateWrapper(LambdaUpdateWrapper wrapper) { + String sqlSet = wrapper.getSqlSet() == null ? "" : wrapper.getSqlSet(); + String columns = Arrays.stream(sqlSet.split(",")) + .map(part -> part.split("=")[0].trim()) + .collect(Collectors.joining(",")); + String params = wrapper.getParamNameValuePairs().entrySet().stream() + .sorted(Comparator.comparingInt(entry -> Integer.parseInt(entry.getKey().substring("MPGENVAL".length())))) + .map(entry -> entry.getKey() + "=" + (entry.getValue() instanceof String text + ? "\"" + text + "\"" : String.valueOf(entry.getValue()))) + .collect(Collectors.joining(",")); + return "columns=" + dash(columns) + " where=" + wrapper.getSqlSegment() + " params=" + params; + } + + private static String dash(Object value) { + String text = value == null ? "" : String.valueOf(value); + return text.isBlank() ? "-" : text; + } + + private static String shortHash(String value) { + return value == null || value.length() < 12 ? dash(value) : value.substring(0, 12); + } + + private static int count(String text, String needle) { + int occurrences = 0; + int index = text.indexOf(needle); + while (index >= 0) { + occurrences++; + index = text.indexOf(needle, index + needle.length()); + } + return occurrences; + } + + private static Integer queryInt(LambdaQueryWrapper query) { + query.getSqlSegment(); + return query.getParamNameValuePairs().values().stream() + .filter(Integer.class::isInstance) + .map(Integer.class::cast) + .findFirst().orElse(null); + } + + private static String queryHash(LambdaQueryWrapper query) { + query.getSqlSegment(); + return query.getParamNameValuePairs().values().stream() + .filter(String.class::isInstance) + .map(String.class::cast) + .filter(value -> value.length() == 64) + .findFirst().orElse(null); + } + + // ------------------------------------------------------------------ + // 请求构造 + // ------------------------------------------------------------------ + + private static ShopDataCrawlRowDto row(String date, String asin) { + ShopDataCrawlRowDto row = new ShopDataCrawlRowDto(); + row.setDate(date); + row.setAsin(asin); + row.setBrand("Example Brand"); + row.setInventorySales("10"); + row.setSalesRank("20"); + row.setPageViews("30"); + row.setUnitsSold("40"); + row.setPrice("50"); + row.setRecommendedOffer("60"); + return row; + } + + private static ShopDataCrawlCountryResultDto country(String code, ShopDataCrawlRowDto... rows) { + ShopDataCrawlCountryResultDto country = new ShopDataCrawlCountryResultDto(); + country.setCountry(code); + country.setItems(List.of(rows)); + return country; + } + + /** 旧版增量协议:不带 chunk_index/chunk_total,靠 shopDone 收尾。 */ + private static ShopDataCrawlShopPayloadDto legacy(boolean shopDone, String country, ShopDataCrawlRowDto... rows) { + ShopDataCrawlShopPayloadDto payload = new ShopDataCrawlShopPayloadDto(); + payload.setShopName(SHOP_NAME); + payload.setShopDone(shopDone); + payload.setCountryResults(country == null + ? new ArrayList<>() : new ArrayList<>(List.of(country(country, rows)))); + return payload; + } + + private static ShopDataCrawlShopPayloadDto chunk(int index, int total, String country, ShopDataCrawlRowDto... rows) { + ShopDataCrawlShopPayloadDto payload = legacy(false, country, rows); + payload.setChunkIndex(index); + payload.setChunkTotal(total); + return payload; + } + + private static ShopDataCrawlShopPayloadDto failedShop(String error) { + ShopDataCrawlShopPayloadDto payload = new ShopDataCrawlShopPayloadDto(); + payload.setShopName(SHOP_NAME); + payload.setError(error); + return payload; + } + + private static ShopDataCrawlSubmitResultRequest request(ShopDataCrawlShopPayloadDto... shops) { + ShopDataCrawlSubmitResultRequest request = new ShopDataCrawlSubmitResultRequest(); + request.setShops(List.of(shops)); + return request; + } + + // ------------------------------------------------------------------ + // 用例 + // ------------------------------------------------------------------ + + private interface Emit { + String render() throws Exception; + } + + private static void appendCase(StringBuilder sb, String caseName, Emit emit) { + sb.append("== case=").append(caseName).append('\n'); + try { + sb.append(emit.render()); + } catch (Exception ex) { + sb.append("thrown: ").append(ex.getClass().getSimpleName()) + .append(" code=").append(ex instanceof com.nanri.aiimage.common.exception.BusinessException businessException + ? dash(businessException.getCode()) : "-") + .append(" message=").append(dash(ex.getMessage())).append('\n'); + } + } + + private static String submit(Fixture fixture, ShopDataCrawlSubmitResultRequest request) { + fixture.service().submitResult(TASK_ID, request); + StringBuilder sb = new StringBuilder(); + fixture.events.forEach(line -> sb.append(line).append('\n')); + return sb.append(counters(fixture)).toString(); + } + + private static String counters(Fixture fixture) { + return "events=" + fixture.events.size() + + " heartbeatTouches=" + fixture.heartbeats + + " lockCloses=" + fixture.lockCloses + + " chunks=" + fixture.chunks.size() + + " scopes=" + fixture.scopes.size() + + " rustfs=" + fixture.rustfsPayloads.size() + '\n'; + } + + /** 默认世界:RUNNING + 单店一行 + 归属本实例。服务在首次用到时才建(夹具开关得先就位)。 */ + private static Fixture running() { + Fixture fixture = new Fixture(); + fixture.withRows(SHOP_NAME).withTask("RUNNING", MODULE_TYPE, "instance-a"); + return fixture; + } + + /** 只取本次调用的流水(分片跨调用场景:前一次提交的事件不算)。 */ + private static String submitFreshEvents(Fixture fixture, ShopDataCrawlSubmitResultRequest request) { + fixture.events.clear(); + return submit(fixture, request); + } + + public static String runAll() throws Exception { + StringBuilder sb = new StringBuilder(); + + // 1. 旧协议一片到底:合并→行标成功→清合并载荷→进度落库→tryFinalize 收尾链 + Fixture happy = running(); + appendCase(sb, "happyLegacyDone", () -> submit(happy, request(legacy(true, "UK", row("2026-07-25", "B001"))))); + + // 2. 店铺报错:先并快照再标失败,任务终态 FAILED,且终态无成功行要走「静默清理分片」那格 + Fixture failed = running(); + appendCase(sb, "failedShopFinalizesTaskFailed", () -> submit(failed, request(failedShop("店铺打不开")))); + + // 3. 没 done 也没分片元数据:行留在待处理(-1),任务表照常刷一次进度 + Fixture waiting = running(); + appendCase(sb, "legacyWithoutDoneWaits", () -> submit(waiting, request(legacy(false, "UK", row("2026-07-25", "B001"))))); + + // 4. 回传店名与任务不符:normalizePayloadByShop 直接拒,一行都不许被碰 + Fixture mismatched = running(); + appendCase(sb, "shopNameMismatchRejected", () -> { + ShopDataCrawlShopPayloadDto other = legacy(true, "UK", row("2026-07-25", "B001")); + other.setShopName("别的店"); + return submit(mismatched, request(other)); + }); + + // 5. 一次回传塞两个店铺:本支的合同是「必须且只能一个」,抢锁之前就拒 + Fixture twoShops = running(); + appendCase(sb, "twoShopsRejectedBeforeLock", () -> submit(twoShops, + request(legacy(true, "UK", row("2026-07-25", "B001")), legacy(true, "DE", row("2026-07-26", "B002"))))); + + // 6. 整个请求为 null + Fixture nullRequest = running(); + appendCase(sb, "nullRequestRejectedBeforeLock", () -> submit(nullRequest, null)); + + // 7. shops 里夹 null:进到建索引那一步才拒(与第 5 例是两条不同出口) + Fixture nullShop = running(); + ShopDataCrawlSubmitResultRequest nullish = new ShopDataCrawlSubmitResultRequest(); + nullish.setShops(Arrays.asList((ShopDataCrawlShopPayloadDto) null)); + appendCase(sb, "nullShopInListRejected", () -> submit(nullShop, nullish)); + + // 8. taskId 非法:锁一次都不该被碰 + Fixture badId = running(); + appendCase(sb, "invalidTaskIdRejectedBeforeLock", () -> { + StringBuilder inner = new StringBuilder(); + try { + badId.service().submitResult(0L, request(legacy(true, "UK", row("2026-07-25", "B001")))); + inner.append("thrown: none\n"); + } catch (Exception ex) { + inner.append("thrown: ").append(ex.getClass().getSimpleName()) + .append(" code=").append(ex instanceof com.nanri.aiimage.common.exception.BusinessException businessException + ? dash(businessException.getCode()) : "-") + .append(" message=").append(dash(ex.getMessage())).append('\n'); + } + return inner.append(counters(badId)).toString(); + }); + + // 9. 任务已终态:本支是不带 code 的 BusinessException(与孪生两支的 40901 是两种线协议) + Fixture terminal = new Fixture(); + terminal.withRows(SHOP_NAME).withTask("SUCCESS", MODULE_TYPE, "instance-a"); + appendCase(sb, "terminalTaskRejectedWithoutCode", () -> submit(terminal, + request(legacy(true, "UK", row("2026-07-25", "B001"))))); + + // 10. 任务属于别的模块:「任务不存在」 + Fixture wrongModule = new Fixture(); + wrongModule.withRows(SHOP_NAME).withTask("RUNNING", "APPEARANCE_PATENT", "instance-a"); + appendCase(sb, "wrongModuleTaskNotFound", () -> submit(wrongModule, + request(legacy(true, "UK", row("2026-07-25", "B001"))))); + + // 11. 归属别人且已终态:本支现网先查归属 ⇒ 40903,不是终态拒绝(步序钉死格) + Fixture foreign = new Fixture(); + foreign.withRows(SHOP_NAME).withTask("SUCCESS", MODULE_TYPE, "instance-b"); + appendCase(sb, "ownershipCheckedBeforeTerminal", () -> submit(foreign, + request(legacy(true, "UK", row("2026-07-25", "B001"))))); + + // 12. 任务里两行:本支的合同是「一支一店」,「任务店铺数据异常」 + Fixture twoRows = new Fixture(); + twoRows.withRows(SHOP_NAME, "店铺B").withTask("RUNNING", MODULE_TYPE, "instance-a"); + appendCase(sb, "twoResultRowsRejected", () -> submit(twoRows, + request(legacy(true, "UK", row("2026-07-25", "B001"))))); + + // 13. 显式分片、只到一片:落 RustFS+分片行+scope 计数,行状态不动 + Fixture partialChunk = running(); + appendCase(sb, "partialChunkStored", () -> submit(partialChunk, + request(chunk(1, 2, "DE", row("2026-07-25", "B001"))))); + + // 14. 第二片先到、第一片后到:完成判定按 chunk_index 顺序合并,不受到达顺序影响 + Fixture reordered = running(); + reordered.service().submitResult(TASK_ID, request(chunk(2, 2, "UK", row("2026-07-26", "B002")))); + appendCase(sb, "secondChunkArrivesFirst", () -> submitFreshEvents(reordered, + request(chunk(1, 2, "DE", row("2026-07-25", "B001"))))); + + // 15. 同一片重复回传(内容一致):唯一键冲突被吸收,计数器不重复加 + Fixture retried = running(); + ShopDataCrawlSubmitResultRequest first = request(chunk(1, 2, "DE", row("2026-07-25", "B001"))); + retried.service().submitResult(TASK_ID, first); + appendCase(sb, "identicalChunkRetryAbsorbed", () -> submitFreshEvents(retried, first)); + + // 16. 同一片换了内容:拒(分片幂等靠内容哈希,不是靠 index) + Fixture conflicting = running(); + conflicting.service().submitResult(TASK_ID, request(chunk(1, 2, "DE", row("2026-07-25", "B001")))); + appendCase(sb, "chunkRetryWithDifferentContentRejected", () -> submitFreshEvents(conflicting, + request(chunk(1, 2, "DE", row("2026-07-25", "B099"))))); + + // 17. chunk_index 与 chunk_total 只传一个:建索引后第一道元数据校验就拒 + Fixture halfMetadata = running(); + appendCase(sb, "chunkIndexWithoutTotalRejected", () -> { + ShopDataCrawlShopPayloadDto payload = legacy(true, "DE", row("2026-07-25", "B001")); + payload.setChunkIndex(1); + return submit(halfMetadata, request(payload)); + }); + + // 18. chunk_total 中途改口:同一店铺必须一致 + Fixture changedTotal = running(); + changedTotal.service().submitResult(TASK_ID, request(chunk(1, 2, "DE", row("2026-07-25", "B001")))); + appendCase(sb, "chunkTotalChangeRejected", () -> submitFreshEvents(changedTotal, + request(chunk(2, 3, "UK", row("2026-07-26", "B002"))))); + + // 19. RustFS 没开:分片一条都不许落库 + Fixture noRustfs = running(); + noRustfs.sharedWriteEnabled = false; + appendCase(sb, "chunkRejectedWhenRustfsDisabled", () -> submit(noRustfs, + request(chunk(1, 1, "DE", row("2026-07-25", "B001"))))); + + // 20. 单片即全片:走 loadCompleteResultPayload→行标成功→终态收尾刷结果列(批量条件更新那格) + Fixture singleChunk = running(); + appendCase(sb, "completedRowFlushesWorkbookColumns", () -> submit(singleChunk, + request(chunk(1, 1, "UK", row("2026-07-25", "B001"), row("2026-07-26", "B002"))))); + + return sb.toString(); + } + + private static String read(File file) throws Exception { + return Files.readString(file.toPath(), StandardCharsets.UTF_8).replace("\r\n", "\n"); + } + + @Test + void submit_result_snapshot_matches_golden() throws Exception { + assertEquals(read(GOLDEN_SUBMIT), runAll(), "回传段落库流水与 golden 一致"); + } + + @Test + void submit_result_snapshot_is_reproducible() throws Exception { + assertEquals(runAll(), runAll(), "同一夹具跑两次流水一致(时钟与雪花 id 不得进快照)"); + } + + @Test + void submit_result_snapshot_golden_is_committed() { + assertTrue(GOLDEN_SUBMIT.isFile(), "golden 文件必须存在并入库: " + GOLDEN_SUBMIT.getAbsolutePath()); + assertTrue(GOLDEN_SUBMIT.length() > 0, "golden 文件非空"); + } + + @Test + void submit_result_snapshot_tamper_is_detected() throws Exception { + String original = read(GOLDEN_SUBMIT); + assertTrue(original.contains("== case=happyLegacyDone"), "golden 内容合法"); + try { + Files.writeString(GOLDEN_SUBMIT.toPath(), original + "\n# tampered", StandardCharsets.UTF_8); + assertTrue(!read(GOLDEN_SUBMIT).equals(runAll()), "篡改 golden 后断言应失败"); + } finally { + Files.writeString(GOLDEN_SUBMIT.toPath(), original, StandardCharsets.UTF_8); + } + assertEquals(original, read(GOLDEN_SUBMIT), "恢复原始 golden"); + } + + /** 只在显式点名时写 golden:默认跑测试永远不许改是非源。 */ + @Test + @EnabledIfEnvironmentVariable(named = "SHUFU_SNAPSHOT_WRITE", matches = "true") + void write_golden_on_demand() throws Exception { + Files.createDirectories(GOLDEN_SUBMIT.getParentFile().toPath()); + Files.writeString(GOLDEN_SUBMIT.toPath(), runAll(), StandardCharsets.UTF_8); + } +} diff --git a/backend-java/src/test/resources/shopdatacrawl/golden/create-task-snapshot.txt b/backend-java/src/test/resources/shopdatacrawl/golden/create-task-snapshot.txt new file mode 100644 index 000000000..eb538d027 --- /dev/null +++ b/backend-java/src/test/resources/shopdatacrawl/golden/create-task-snapshot.txt @@ -0,0 +1,63 @@ +== case=happySingleShopTwoCountries +vo: taskId=1001 items=1 + item: resultId=2001 taskId=1001 shopName=Demo Shop shopId=shop-Demo Shop platform=Amazon companyName=公司-Demo Shop matched=true matchStatus=MATCHED taskStatus=RUNNING success=- error=- countryCodes=[UK, DE] countryResults=0 downloadUrl=- +events=7 taskInserts=1 resultInserts=1 heartbeatTouches=1 + evt: task.insert: id=1001 taskNoPrefix=SHOP_DATA_CRAWL moduleType=SHOP_DATA_CRAWL taskMode=PYTHON_QUEUE status=RUNNING sourceFileCount=1 ok=0 fail=0 userId=7 createdBy=user:7 deviceId=dev-1 ownerInstanceId=- requestJsonPresent=false resultJsonPresent=false + evt: taskCacheSave: id=1001 + evt: result.insert: id=2001 taskId=1001 moduleType=SHOP_DATA_CRAWL sourceFilename=Demo Shop sourceFileUrl=shop-Demo Shop userId=7 success=-1 rowCount=- createdAtEqualsTaskCreatedAt=true + evt: snapshotTable.replace: taskId=1001 items=1 + evt: progressSave: taskId=1001 status=RUNNING total=1 ok=0 fail=0 + evt: task.update: id=1001 status=RUNNING ok=0 fail=0 requestJsonHasOwner=true requestJsonItems=1 resultJsonItems=1 createdAtEqualsUpdatedAt=true + evt: taskCacheSave: id=1001 +== case=requestFieldsOverwrittenByMatch +vo: taskId=1001 items=1 + item: resultId=2001 taskId=1001 shopName=Demo Shop shopId=shop-Demo Shop platform=Amazon companyName=公司-Demo Shop matched=true matchStatus=MATCHED taskStatus=RUNNING success=- error=- countryCodes=[UK] countryResults=0 downloadUrl=- +events=7 taskInserts=1 resultInserts=1 heartbeatTouches=1 + evt: task.insert: id=1001 taskNoPrefix=SHOP_DATA_CRAWL moduleType=SHOP_DATA_CRAWL taskMode=PYTHON_QUEUE status=RUNNING sourceFileCount=1 ok=0 fail=0 userId=7 createdBy=user:7 deviceId=dev-1 ownerInstanceId=- requestJsonPresent=false resultJsonPresent=false + evt: taskCacheSave: id=1001 + evt: result.insert: id=2001 taskId=1001 moduleType=SHOP_DATA_CRAWL sourceFilename=Demo Shop sourceFileUrl=shop-Demo Shop userId=7 success=-1 rowCount=- createdAtEqualsTaskCreatedAt=true + evt: snapshotTable.replace: taskId=1001 items=1 + evt: progressSave: taskId=1001 status=RUNNING total=1 ok=0 fail=0 + evt: task.update: id=1001 status=RUNNING ok=0 fail=0 requestJsonHasOwner=true requestJsonItems=1 resultJsonItems=1 createdAtEqualsUpdatedAt=true + evt: taskCacheSave: id=1001 +== case=twoItemsRejected +thrown: BusinessException code=- message=items 必须且只能包含一个店铺 +events=0 taskInserts=0 resultInserts=0 heartbeatTouches=0 +== case=emptyItemsRejected +thrown: BusinessException code=- message=items 必须且只能包含一个店铺 +events=0 taskInserts=0 resultInserts=0 heartbeatTouches=0 +== case=nullItemsRejected +thrown: BusinessException code=- message=items 必须且只能包含一个店铺 +events=0 taskInserts=0 resultInserts=0 heartbeatTouches=0 +== case=rawTwoItemsSameShopRejectedFirstGate +thrown: BusinessException code=- message=items 必须且只能包含一个店铺 +events=0 taskInserts=0 resultInserts=0 heartbeatTouches=0 +== case=blankShopNameDropped +thrown: BusinessException code=- message=items 必须且只能包含一个有效店铺 +events=0 taskInserts=0 resultInserts=0 heartbeatTouches=0 +== case=reNormalizedToBlankRejects +thrown: BusinessException code=- message=任务数据已失效,请刷新后重试 +events=2 taskInserts=1 resultInserts=0 heartbeatTouches=1 + evt: task.insert: id=1001 taskNoPrefix=SHOP_DATA_CRAWL moduleType=SHOP_DATA_CRAWL taskMode=PYTHON_QUEUE status=RUNNING sourceFileCount=1 ok=0 fail=0 userId=7 createdBy=user:7 deviceId=dev-1 ownerInstanceId=- requestJsonPresent=false resultJsonPresent=false + evt: taskCacheSave: id=1001 +== case=unmatchedShopRejected +thrown: BusinessException code=- message=店铺未匹配 +events=0 taskInserts=0 resultInserts=0 heartbeatTouches=0 +== case=usIsNotSupportedCountry +thrown: BusinessException code=- message=非法国家代码: US +events=0 taskInserts=0 resultInserts=0 heartbeatTouches=0 +== case=duplicateCountryCodeRejected +thrown: BusinessException code=- message=country_codes 存在重复: UK +events=0 taskInserts=0 resultInserts=0 heartbeatTouches=0 +== case=emptyCountryCodesRejected +thrown: BusinessException code=- message=country_codes 至少选择 1 个国家 +events=0 taskInserts=0 resultInserts=0 heartbeatTouches=0 +== case=blankCountryCodeRejected +thrown: BusinessException code=- message=country_codes 含空项 +events=0 taskInserts=0 resultInserts=0 heartbeatTouches=0 +== case=nullUserIdRejected +thrown: BusinessException code=- message=user_id 不合法 +events=0 taskInserts=0 resultInserts=0 heartbeatTouches=0 +== case=zeroUserIdRejected +thrown: BusinessException code=- message=user_id 不合法 +events=0 taskInserts=0 resultInserts=0 heartbeatTouches=0 diff --git a/backend-java/src/test/resources/shopdatacrawl/golden/submit-result-snapshot.txt b/backend-java/src/test/resources/shopdatacrawl/golden/submit-result-snapshot.txt new file mode 100644 index 000000000..30ce8be9e --- /dev/null +++ b/backend-java/src/test/resources/shopdatacrawl/golden/submit-result-snapshot.txt @@ -0,0 +1,138 @@ +== case=happyLegacyDone +taskCacheSave: id=1001 +mergedSave: taskId=1001 shopKey=Demo Shop shopName=Demo Shop error=- countries=1 rows=1 done=true chunk=null/null +result.update: id=2001 success=1 error=- filename=- urlPresent=false size=null contentType=- rowCount=null +mergedRemove: taskId=1001 shopKey=Demo Shop +snapshotTable.replace: taskId=1001 items=1 +progressSave: taskId=1001 status=RUNNING total=1 ok=1 fail=0 +task.update: id=1001 status=RUNNING ok=null fail=null sourceFileCount=null error=- resultJsonItems=1 resultJsonHasRows=true finishedPresent=false +taskCacheSave: id=1001 +taskCacheSave: id=1001 +task.update: id=1001 status=SUCCESS ok=1 fail=0 sourceFileCount=null error=- resultJsonItems=1 resultJsonHasRows=true finishedPresent=true +result.batchUpdate: columns=resultFilename,resultFileUrl,resultFileSize,resultContentType,rowCount where=(id IN (#{ew.paramNameValuePairs.MPGENVAL6}) AND success = #{ew.paramNameValuePairs.MPGENVAL7}) params=MPGENVAL1="店铺数据抓取-Demo Shop-1001.xlsx",MPGENVAL2=null,MPGENVAL3=0,MPGENVAL4="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",MPGENVAL5=1 +enqueue: taskId=1001 resultId=2001 scopeKey=task:1001:owner:instance-a +snapshotTable.replace: taskId=1001 items=1 +progressSave: taskId=1001 status=SUCCESS total=1 ok=1 fail=0 +task.update: id=1001 status=SUCCESS ok=1 fail=0 sourceFileCount=null error=- resultJsonItems=1 resultJsonHasRows=true finishedPresent=true +taskCacheDelete: taskId=1001 +events=16 heartbeatTouches=1 lockCloses=2 chunks=0 scopes=0 rustfs=0 +== case=failedShopFinalizesTaskFailed +taskCacheSave: id=1001 +result.update: id=2001 success=0 error=店铺打不开 filename=- urlPresent=false size=0 contentType=- rowCount=0 +mergedRemove: taskId=1001 shopKey=Demo Shop +snapshotTable.replace: taskId=1001 items=1 +progressSave: taskId=1001 status=RUNNING total=1 ok=0 fail=1 +task.update: id=1001 status=RUNNING ok=null fail=null sourceFileCount=null error=- resultJsonItems=1 resultJsonHasRows=false finishedPresent=false +taskCacheSave: id=1001 +taskCacheSave: id=1001 +snapshotTable.replace: taskId=1001 items=1 +progressSave: taskId=1001 status=FAILED total=1 ok=0 fail=1 +task.update: id=1001 status=FAILED ok=0 fail=1 sourceFileCount=null error=Demo Shop: 店铺打不开 resultJsonItems=1 resultJsonHasRows=false finishedPresent=true +snapshotTable.replace: taskId=1001 items=1 +progressSave: taskId=1001 status=FAILED total=1 ok=0 fail=1 +task.update: id=1001 status=FAILED ok=0 fail=1 sourceFileCount=null error=Demo Shop: 店铺打不开 resultJsonItems=1 resultJsonHasRows=false finishedPresent=true +taskCacheDelete: taskId=1001 +events=15 heartbeatTouches=1 lockCloses=2 chunks=0 scopes=0 rustfs=0 +== case=legacyWithoutDoneWaits +taskCacheSave: id=1001 +mergedSave: taskId=1001 shopKey=Demo Shop shopName=Demo Shop error=- countries=1 rows=1 done=null chunk=null/null +snapshotTable.replace: taskId=1001 items=1 +progressSave: taskId=1001 status=RUNNING total=1 ok=0 fail=0 +task.update: id=1001 status=RUNNING ok=null fail=null sourceFileCount=null error=- resultJsonItems=1 resultJsonHasRows=true finishedPresent=false +taskCacheSave: id=1001 +taskCacheSave: id=1001 +task.update: id=1001 status=RUNNING ok=0 fail=0 sourceFileCount=null error=- resultJsonItems=1 resultJsonHasRows=true finishedPresent=false +taskCacheSave: id=1001 +events=9 heartbeatTouches=1 lockCloses=2 chunks=0 scopes=0 rustfs=0 +== case=shopNameMismatchRejected +thrown: BusinessException code=- message=回传店铺与任务不匹配 +== case=twoShopsRejectedBeforeLock +thrown: BusinessException code=- message=shops 必须且只能包含一个店铺结果 +== case=nullRequestRejectedBeforeLock +thrown: BusinessException code=- message=shops 必须且只能包含一个店铺结果 +== case=nullShopInListRejected +thrown: BusinessException code=- message=shops 包含空结果 +== case=invalidTaskIdRejectedBeforeLock +thrown: BusinessException code=- message=taskId 不合法 +events=0 heartbeatTouches=0 lockCloses=0 chunks=0 scopes=0 rustfs=0 +== case=terminalTaskRejectedWithoutCode +thrown: BusinessException code=- message=任务已结束,拒绝重复提交 +== case=wrongModuleTaskNotFound +thrown: BusinessException code=- message=任务不存在 +== case=ownershipCheckedBeforeTerminal +thrown: TaskOwnerMismatchException code=40903 message=该任务已绑定到另一台服务实例处理,请通过原实例继续处理 +== case=twoResultRowsRejected +thrown: BusinessException code=- message=任务店铺数据异常 +== case=partialChunkStored +taskCacheSave: id=1001 +payloadStore: pointer=rustfs:payload-1 +chunkInsert: idx=1/2 hash=eafc7b7e1451 scopeKey=result-chunks:Demo Shop +scopeInsert: received=1 total=2 completed=0 scopeKey=result-chunks:Demo Shop +task.update: id=1001 status=RUNNING ok=null fail=null sourceFileCount=null error=- resultJsonItems=0 resultJsonHasRows=false finishedPresent=false +taskCacheSave: id=1001 +taskCacheSave: id=1001 +task.update: id=1001 status=RUNNING ok=0 fail=0 sourceFileCount=null error=- resultJsonItems=0 resultJsonHasRows=false finishedPresent=false +taskCacheSave: id=1001 +events=9 heartbeatTouches=1 lockCloses=2 chunks=1 scopes=1 rustfs=1 +== case=secondChunkArrivesFirst +taskCacheSave: id=1001 +payloadStore: pointer=rustfs:payload-2 +chunkInsert: idx=1/2 hash=eafc7b7e1451 scopeKey=result-chunks:Demo Shop +scopeUpdate: received=2 total=2 completed=1 scopeKey=result-chunks:Demo Shop +payloadResolve: pointer=rustfs:payload-2 +payloadResolve: pointer=rustfs:payload-1 +result.update: id=2001 success=1 error=- filename=- urlPresent=false size=null contentType=- rowCount=null +mergedRemove: taskId=1001 shopKey=Demo Shop +snapshotTable.replace: taskId=1001 items=1 +progressSave: taskId=1001 status=RUNNING total=1 ok=1 fail=0 +task.update: id=1001 status=RUNNING ok=0 fail=0 sourceFileCount=null error=- resultJsonItems=1 resultJsonHasRows=true finishedPresent=false +taskCacheSave: id=1001 +taskCacheSave: id=1001 +task.update: id=1001 status=SUCCESS ok=1 fail=0 sourceFileCount=null error=- resultJsonItems=1 resultJsonHasRows=true finishedPresent=true +result.batchUpdate: columns=resultFilename,resultFileUrl,resultFileSize,resultContentType,rowCount where=(id IN (#{ew.paramNameValuePairs.MPGENVAL6}) AND success = #{ew.paramNameValuePairs.MPGENVAL7}) params=MPGENVAL1="店铺数据抓取-Demo Shop-1001.xlsx",MPGENVAL2=null,MPGENVAL3=0,MPGENVAL4="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",MPGENVAL5=1 +enqueue: taskId=1001 resultId=2001 scopeKey=task:1001:owner:instance-a +snapshotTable.replace: taskId=1001 items=1 +progressSave: taskId=1001 status=SUCCESS total=1 ok=1 fail=0 +task.update: id=1001 status=SUCCESS ok=1 fail=0 sourceFileCount=null error=- resultJsonItems=1 resultJsonHasRows=true finishedPresent=true +taskCacheDelete: taskId=1001 +events=20 heartbeatTouches=2 lockCloses=4 chunks=2 scopes=1 rustfs=2 +== case=identicalChunkRetryAbsorbed +taskCacheSave: id=1001 +payloadStore: pointer=rustfs:payload-2 +payloadDelete: pointer=rustfs:payload-2 +scopeUpdate: received=1 total=2 completed=0 scopeKey=result-chunks:Demo Shop +task.update: id=1001 status=RUNNING ok=0 fail=0 sourceFileCount=null error=- resultJsonItems=0 resultJsonHasRows=false finishedPresent=false +taskCacheSave: id=1001 +taskCacheSave: id=1001 +task.update: id=1001 status=RUNNING ok=0 fail=0 sourceFileCount=null error=- resultJsonItems=0 resultJsonHasRows=false finishedPresent=false +taskCacheSave: id=1001 +events=9 heartbeatTouches=2 lockCloses=4 chunks=1 scopes=1 rustfs=1 +== case=chunkRetryWithDifferentContentRejected +thrown: BusinessException code=- message=同一 chunk_index 已回传不同内容,拒绝覆盖 +== case=chunkIndexWithoutTotalRejected +thrown: BusinessException code=- message=chunk_index 与 chunk_total 必须同时传入 +== case=chunkTotalChangeRejected +thrown: BusinessException code=- message=同一店铺的 chunk_total 必须保持一致 +== case=chunkRejectedWhenRustfsDisabled +thrown: BusinessException code=- message=RustFS 未配置,店铺数据抓取结果分片暂不可接收 +== case=completedRowFlushesWorkbookColumns +taskCacheSave: id=1001 +payloadStore: pointer=rustfs:payload-1 +chunkInsert: idx=1/1 hash=b6c082cd9d58 scopeKey=result-chunks:Demo Shop +scopeInsert: received=1 total=1 completed=1 scopeKey=result-chunks:Demo Shop +payloadResolve: pointer=rustfs:payload-1 +result.update: id=2001 success=1 error=- filename=- urlPresent=false size=null contentType=- rowCount=null +mergedRemove: taskId=1001 shopKey=Demo Shop +snapshotTable.replace: taskId=1001 items=1 +progressSave: taskId=1001 status=RUNNING total=1 ok=1 fail=0 +task.update: id=1001 status=RUNNING ok=null fail=null sourceFileCount=null error=- resultJsonItems=1 resultJsonHasRows=true finishedPresent=false +taskCacheSave: id=1001 +taskCacheSave: id=1001 +task.update: id=1001 status=SUCCESS ok=1 fail=0 sourceFileCount=null error=- resultJsonItems=1 resultJsonHasRows=true finishedPresent=true +result.batchUpdate: columns=resultFilename,resultFileUrl,resultFileSize,resultContentType,rowCount where=(id IN (#{ew.paramNameValuePairs.MPGENVAL6}) AND success = #{ew.paramNameValuePairs.MPGENVAL7}) params=MPGENVAL1="店铺数据抓取-Demo Shop-1001.xlsx",MPGENVAL2=null,MPGENVAL3=0,MPGENVAL4="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",MPGENVAL5=1 +enqueue: taskId=1001 resultId=2001 scopeKey=task:1001:owner:instance-a +snapshotTable.replace: taskId=1001 items=1 +progressSave: taskId=1001 status=SUCCESS total=1 ok=1 fail=0 +task.update: id=1001 status=SUCCESS ok=1 fail=0 sourceFileCount=null error=- resultJsonItems=1 resultJsonHasRows=true finishedPresent=true +taskCacheDelete: taskId=1001 +events=19 heartbeatTouches=1 lockCloses=2 chunks=1 scopes=1 rustfs=1