test(phase-3): 3B 迁移前把 shopdatacrawl 的建单与回传两段钉成 golden

蓝图 §4 Phase 3 首迁实录写的开工前置第①条——「那五支的 submitResult 同样没有 golden,
各自补完才能动」。本笔只补件、不动生产码一支。

回传段 20 例、建单段 14 例,渲染的是副作用**先后**流水而不是分组清单,三格专门钉住
Phase 2/3 已登记的「不许顺手统一」:
- ownershipCheckedBeforeTerminal:本支先查归属再判终态(底座 admitSubmission 默认序相反),
  套错就 40903 → 「任务已结束」;
- terminalTaskRejectedWithoutCode:终态重复提交本支不带 40901,线协议与孪生两支不同;
- completedRowFlushesWorkbookColumns:收尾刷结果列走 LambdaUpdateWrapper 批量条件更新
  (只刷 success=1 的行、不写 success/errorMessage、objectKey 传 null),publishResultFile 不适用。

建单段另钉一格快照本来看不见的性质:建单时刻一枚复用给头行与每一行结果行,故只断
createdAtEqualsTaskCreatedAt=true 而不打印时钟——底座五参 newTaskHeader 是「逐行各取一次」那一派,
照搬就把这格改成 false。
This commit is contained in:
2026-10-19 02:31:42 +08:00
parent 278eaaec2c
commit b11bc599cd
4 changed files with 1496 additions and 0 deletions
@@ -0,0 +1,487 @@
package com.nanri.aiimage.modules.shopdatacrawl.service.support;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.databind.json.JsonMapper;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.InstanceMetadata;
import com.nanri.aiimage.config.TaskPressureProperties;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.shopdatacrawl.mapper.ShopDataCrawlCountryPrefMapper;
import com.nanri.aiimage.modules.shopdatacrawl.mapper.ShopDataCrawlShopCandidateMapper;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCreateTaskRequest;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlTaskItemDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlCreateTaskVo;
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlDailyFileService;
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlExcelAssemblyService;
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlItemStoreService;
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlResolveService;
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskCacheService;
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskService;
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
import com.nanri.aiimage.modules.shopkey.service.ShopManageService;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import com.nanri.aiimage.modules.task.service.TaskResultItemService;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoShopMatchResultVo;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
import java.io.File;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
/**
* 店铺数据抓取建单(段 1 落库形状)的现网行为快照——Phase 3 迁移 3B 的另一半护身符。
*
* <p><b>为什么非但要有回传快照、还要有建单快照。</b>3B 要动的不只是 {@code submitResult}:
* 建单那截的 task 头行与结果行是底座 {@code newTaskHeader}/{@code newScopeRow} 的直接替换点。
* 而本支有一个<b>快照本来量不到</b>的性质——建单时刻 {@code LocalDateTime.now()} 被<b>一枚复用</b>给
* 头行和每一行结果行({@code ShopDataCrawlTaskService:574 → :586-587 / :605})。时钟不能进快照
* (进了就逐字节对撞不了),所以这里只断言<b>相等关系</b>:每行渲染一枚
* {@code createdAtEqualsTaskCreatedAt=}。底座五参版 {@code newTaskHeader} 取的是「默认时区、逐行各取一次」
* 那一派,照搬就会把这格从 {@code true} 改成 {@code false}——这条只有这一处量得到。
*
* <p><b>另外三格被钉住的本支现网形状:</b>
* <ul>
* <li>{@code success=-1}(本支的 {@code RESULT_PENDING},不是其余六支的 {@code 0});
* {@code rowCount} 留 NULL(本支建单不写这一列)。</li>
* <li>{@code successFileCount}/{@code failedFileCount} 建单时写 0(底座刻意不初始化,见其注释)。</li>
* <li>{@code taskNo} 只渲染前缀:雪花 id 进快照就永远对撞不上。</li>
* </ul>
*
* <p>国家代码校验走的是 {@link ShopDataCrawlResolveService#validateCountryCodes} <b>真实实现</b>
* (那段是 static、不碰库),店铺匹配则按现网出口逐例打桩——快照要锁的是本支怎么用这些件,
* 不是替它们重写一遍判据。
*
* golden:src/test/resources/shopdatacrawl/golden/create-task-snapshot.txt
* 重新生成:{@code SHUFU_SNAPSHOT_WRITE=true mvn -o test -Dtest=ShopDataCrawlCreateTaskSnapshotTest}
*/
class ShopDataCrawlCreateTaskSnapshotTest {
private static final File GOLDEN_CREATE =
new File("src/test/resources/shopdatacrawl/golden/create-task-snapshot.txt");
private static final String MODULE_TYPE = "SHOP_DATA_CRAWL";
/** Spring 注入的那只 ObjectMapper ≠ {@code new ObjectMapper()}:Boot 会挂上 jsr310(Phase 0 实测教训)。 */
private static ObjectMapper bootLikeObjectMapper() {
return JsonMapper.builder()
.addModule(new JavaTimeModule())
.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS)
.build();
}
// ------------------------------------------------------------------
// 夹具
// ------------------------------------------------------------------
private static final class Fixture {
private final ObjectMapper objectMapper = bootLikeObjectMapper();
private final List<String> events = new ArrayList<>();
private final List<FileTaskEntity> taskInserts = new ArrayList<>();
private final List<FileResultEntity> resultInserts = new ArrayList<>();
private ShopDataCrawlTaskService service;
private int heartbeats;
/** {@code >=0} 时,第 N+1 次归一化起返回空串——用来够到现网建单循环里那道二次归一化空白闸。 */
private int blankNormalizeAfter = -1;
private int normalizeCalls;
/** 打桩 {@code requireMatchedShop} 的出口:null 表示匹配成功,否则按该文案抛错。 */
private String matchFailure;
private ShopDataCrawlTaskService service() {
if (service != null) {
return service;
}
FileTaskMapper fileTaskMapper = mock(FileTaskMapper.class);
FileResultMapper fileResultMapper = mock(FileResultMapper.class);
ShopDataCrawlTaskCacheService cacheService = mock(ShopDataCrawlTaskCacheService.class);
ZiniaoShopSwitchService ziniao = mock(ZiniaoShopSwitchService.class);
TaskResultItemService resultItemService = mock(TaskResultItemService.class);
TaskProgressSnapshotService progressSnapshotService = mock(TaskProgressSnapshotService.class);
InstanceMetadata instanceMetadata = mock(InstanceMetadata.class);
// 国家代码判据走真实实现(static、不碰库),不为它另造一套形状
ShopDataCrawlResolveService realResolve = new ShopDataCrawlResolveService(
mock(ShopDataCrawlShopCandidateMapper.class), mock(ShopDataCrawlCountryPrefMapper.class),
bootLikeObjectMapper(), ziniao, mock(ShopManageService.class));
ShopDataCrawlResolveService resolveService = mock(ShopDataCrawlResolveService.class);
when(resolveService.validateCountryCodes(any()))
.thenAnswer(invocation -> realResolve.validateCountryCodes(invocation.getArgument(0)));
when(resolveService.requireMatchedShop(anyString())).thenAnswer(invocation -> {
if (matchFailure != null) {
throw new BusinessException(matchFailure);
}
String shopName = invocation.getArgument(0);
return new ZiniaoShopMatchResultVo(true, "shop-" + shopName.trim(), shopName,
"公司-" + shopName.trim(), "Amazon", 9L, "https://example.invalid",
"MATCHED", "店铺索引匹配成功");
});
doAnswer(invocation -> {
FileTaskEntity entity = invocation.getArgument(0);
entity.setId(1001L + taskInserts.size());
taskInserts.add(entity);
events.add("task.insert: " + renderTaskInsert(entity));
return 1;
}).when(fileTaskMapper).insert(any(FileTaskEntity.class));
doAnswer(invocation -> {
events.add("task.update: " + renderTaskUpdate(invocation.<FileTaskEntity>getArgument(0)));
return 1;
}).when(fileTaskMapper).updateById(any(FileTaskEntity.class));
doAnswer(invocation -> {
FileResultEntity entity = invocation.getArgument(0);
entity.setId(2001L + resultInserts.size());
resultInserts.add(entity);
events.add("result.insert: " + renderResultInsert(entity,
taskInserts.isEmpty() ? null : taskInserts.get(0)));
return 1;
}).when(fileResultMapper).insert(any(FileResultEntity.class));
doAnswer(invocation -> {
heartbeats++;
return null;
}).when(cacheService).touchTaskHeartbeat(any());
doAnswer(invocation -> {
events.add("taskCacheSave: id=" + ((FileTaskEntity) invocation.getArgument(0)).getId());
return null;
}).when(cacheService).saveTaskCache(any(FileTaskEntity.class));
doAnswer(invocation -> {
events.add("snapshotTable.replace: taskId=" + invocation.getArgument(0)
+ " items=" + ((List<?>) invocation.getArgument(2)).size());
return null;
}).when(resultItemService).replaceTaskSnapshots(anyLong(), anyString(), any(), any());
doAnswer(invocation -> {
events.add("progressSave: taskId=" + invocation.getArgument(0)
+ " status=" + dash(invocation.getArgument(2))
+ " total=" + invocation.<Integer>getArgument(3)
+ " ok=" + invocation.<Integer>getArgument(4)
+ " fail=" + invocation.<Integer>getArgument(5));
return null;
}).when(progressSnapshotService).save(anyLong(), anyString(), anyString(),
org.mockito.ArgumentMatchers.anyInt(), org.mockito.ArgumentMatchers.anyInt(),
org.mockito.ArgumentMatchers.anyInt(), any(), any(), any());
when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
when(ziniao.normalizeShopName(any())).thenAnswer(invocation -> {
String shopName = invocation.<String>getArgument(0);
normalizeCalls++;
if (blankNormalizeAfter >= 0 && normalizeCalls > blankNormalizeAfter) {
return "";
}
return shopName == null ? "" : shopName.trim();
});
service = new ShopDataCrawlTaskService(
fileTaskMapper,
fileResultMapper,
resolveService,
mock(ShopDataCrawlExcelAssemblyService.class),
cacheService,
mock(OssStorageService.class),
ziniao,
objectMapper,
mock(TaskPressureProperties.class),
mock(TaskFileJobService.class),
resultItemService,
progressSnapshotService,
mock(TaskDistributedLockService.class),
mock(TaskChunkMapper.class),
mock(TaskScopeStateMapper.class),
mock(TransientPayloadStorageService.class),
instanceMetadata,
mock(ShopDataCrawlDailyFileService.class),
mock(ShopDataCrawlItemStoreService.class),
mock(DuplicateCheckRefreshPort.class),
null,
mock(TaskProgressLightAssembler.class),
mock(TaskHeartbeatPositionService.class));
return service;
}
}
// ------------------------------------------------------------------
// 渲染(时钟/雪花 id 不进快照,只断相等关系)
// ------------------------------------------------------------------
private static String renderTaskInsert(FileTaskEntity task) {
return "id=" + task.getId()
+ " taskNoPrefix=" + prefix(task.getTaskNo())
+ " moduleType=" + dash(task.getModuleType())
+ " taskMode=" + dash(task.getTaskMode())
+ " status=" + dash(task.getStatus())
+ " sourceFileCount=" + task.getSourceFileCount()
+ " ok=" + task.getSuccessFileCount()
+ " fail=" + task.getFailedFileCount()
+ " userId=" + dash(task.getUserId())
+ " createdBy=" + dash(task.getCreatedBy())
+ " deviceId=" + dash(task.getDeviceId())
+ " ownerInstanceId=" + dash(task.getOwnerInstanceId())
+ " requestJsonPresent=" + (task.getRequestJson() != null)
+ " resultJsonPresent=" + (task.getResultJson() != null);
}
private static String renderTaskUpdate(FileTaskEntity task) {
return "id=" + task.getId()
+ " status=" + dash(task.getStatus())
+ " ok=" + task.getSuccessFileCount()
+ " fail=" + task.getFailedFileCount()
+ " requestJsonHasOwner=" + (task.getRequestJson() != null
&& task.getRequestJson().contains("\"ownerInstanceId\":\"instance-a\""))
+ " requestJsonItems=" + (task.getRequestJson() == null ? -1
: count(task.getRequestJson(), "\"shopName\""))
+ " resultJsonItems=" + (task.getResultJson() == null ? -1
: count(task.getResultJson(), "\"resultId\""))
+ " createdAtEqualsUpdatedAt=" + (task.getCreatedAt() != null
&& task.getCreatedAt().equals(task.getUpdatedAt()));
}
private static String renderResultInsert(FileResultEntity row, FileTaskEntity task) {
return "id=" + row.getId()
+ " taskId=" + dash(row.getTaskId())
+ " moduleType=" + dash(row.getModuleType())
+ " sourceFilename=" + dash(row.getSourceFilename())
+ " sourceFileUrl=" + dash(row.getSourceFileUrl())
+ " userId=" + dash(row.getUserId())
+ " success=" + dash(row.getSuccess())
+ " rowCount=" + dash(row.getRowCount())
+ " createdAtEqualsTaskCreatedAt=" + (task != null && row.getCreatedAt() != null
&& row.getCreatedAt().equals(task.getCreatedAt()));
}
private static String prefix(String taskNo) {
if (taskNo == null) {
return "-";
}
int index = taskNo.lastIndexOf('-');
return index < 0 ? taskNo : taskNo.substring(0, index);
}
private static String dash(Object value) {
String text = value == null ? "" : String.valueOf(value);
return text.isBlank() ? "-" : text;
}
private static int count(String text, String needle) {
int occurrences = 0;
int index = text.indexOf(needle);
while (index >= 0) {
occurrences++;
index = text.indexOf(needle, index + needle.length());
}
return occurrences;
}
// ------------------------------------------------------------------
// 请求构造
// ------------------------------------------------------------------
private static ShopDataCrawlTaskItemDto shop(String shopName) {
ShopDataCrawlTaskItemDto item = new ShopDataCrawlTaskItemDto();
item.setShopName(shopName);
item.setShopId("请求里带的旧 id");
item.setPlatform("请求里带的旧平台");
item.setCompanyName("请求里带的旧公司");
item.setMatchStatus("请求里带的旧状态");
item.setMatchMessage("请求里带的旧说明");
return item;
}
private static ShopDataCrawlCreateTaskRequest request(Long userId,
List<ShopDataCrawlTaskItemDto> items,
List<String> countryCodes) {
ShopDataCrawlCreateTaskRequest request = new ShopDataCrawlCreateTaskRequest();
request.setUserId(userId);
request.setItems(items);
request.setCountryCodes(countryCodes);
return request;
}
private static ShopDataCrawlCreateTaskRequest oneShop(Long userId, String shopName, List<String> countryCodes) {
return request(userId, new ArrayList<>(List.of(shop(shopName))), countryCodes);
}
// ------------------------------------------------------------------
// 用例
// ------------------------------------------------------------------
private static String runCase(String caseName, Fixture fixture, ShopDataCrawlCreateTaskRequest request) {
StringBuilder sb = new StringBuilder();
sb.append("== case=").append(caseName).append('\n');
try {
ShopDataCrawlCreateTaskVo vo = fixture.service().createTask(request, "dev-1");
sb.append("vo: taskId=").append(dash(vo.getTaskId()))
.append(" items=").append(vo.getItems().size()).append('\n');
for (ShopDataCrawlResultItemVo item : vo.getItems()) {
sb.append(" item: resultId=").append(dash(item.getResultId()))
.append(" taskId=").append(dash(item.getTaskId()))
.append(" shopName=").append(dash(item.getShopName()))
.append(" shopId=").append(dash(item.getShopId()))
.append(" platform=").append(dash(item.getPlatform()))
.append(" companyName=").append(dash(item.getCompanyName()))
.append(" matched=").append(item.isMatched())
.append(" matchStatus=").append(dash(item.getMatchStatus()))
.append(" taskStatus=").append(dash(item.getTaskStatus()))
.append(" success=").append(dash(item.getSuccess()))
.append(" error=").append(dash(item.getError()))
.append(" countryCodes=").append(item.getCountryCodes())
.append(" countryResults=").append(item.getCountryResults() == null ? -1
: item.getCountryResults().size())
.append(" downloadUrl=").append(dash(item.getDownloadUrl()))
.append('\n');
}
} catch (Exception ex) {
sb.append("thrown: ").append(ex.getClass().getSimpleName())
.append(" code=").append(ex instanceof BusinessException businessException
? dash(businessException.getCode()) : "-")
.append(" message=").append(dash(ex.getMessage())).append('\n');
}
sb.append("events=").append(fixture.events.size())
.append(" taskInserts=").append(fixture.taskInserts.size())
.append(" resultInserts=").append(fixture.resultInserts.size())
.append(" heartbeatTouches=").append(fixture.heartbeats).append('\n');
fixture.events.forEach(line -> sb.append(" evt: ").append(line).append('\n'));
return sb.toString();
}
public static String runAll() {
StringBuilder sb = new StringBuilder();
// 1. 单店两国家:头行先落(拿到自增主键)→ 缓存+心跳 → 结果行 → requestJson/resultJson 一起刷回
Fixture happy = new Fixture();
sb.append(runCase("happySingleShopTwoCountries", happy,
oneShop(7L, " Demo Shop ", List.of("UK", "DE"))));
// 2. 请求里带的 shopId/platform/company 一律以紫鸟重新匹配到的为准(这几列会被覆盖)
Fixture overwrite = new Fixture();
sb.append(runCase("requestFieldsOverwrittenByMatch", overwrite,
oneShop(7L, "Demo Shop", List.of("UK"))));
// 3. items 两枚:本支的合同是「必须且只能一个店铺」,且不碰库
Fixture twoItems = new Fixture();
sb.append(runCase("twoItemsRejected", twoItems,
request(7L, new ArrayList<>(List.of(shop("店铺A"), shop("店铺B"))), List.of("UK"))));
// 4. items 空列表
Fixture emptyItems = new Fixture();
sb.append(runCase("emptyItemsRejected", emptyItems,
request(7L, new ArrayList<>(), List.of("UK"))));
// 5. items 为 null
Fixture nullItems = new Fixture();
sb.append(runCase("nullItemsRejected", nullItems,
request(7L, null, List.of("UK"))));
// 6. 两枚同店名:第一道闸数的是<b>原始</b> items(:552),dedupe(:557)根本没机会救它
// ⇒「必须且只能包含一个有效店铺」那句只在 dedupe 把唯一一枚丢光时才到得了(下一例)
Fixture duplicatedShops = new Fixture();
sb.append(runCase("rawTwoItemsSameShopRejectedFirstGate", duplicatedShops,
request(7L, new ArrayList<>(List.of(shop("店铺A"), shop(" 店铺A "))), List.of("UK"))));
// 7. 店名归一化后为空:去重阶段就被丢掉
Fixture blankShop = new Fixture();
sb.append(runCase("blankShopNameDropped", blankShop,
oneShop(7L, " ", List.of("UK"))));
// 8. 建单循环里二次归一化变空(夹具把第 3 次调用打空):头行已落库、结果行一行都不写
Fixture renormalizedBlank = new Fixture();
renormalizedBlank.blankNormalizeAfter = 2;
sb.append(runCase("reNormalizedToBlankRejects", renormalizedBlank,
oneShop(7L, "Demo Shop", List.of("UK"))));
// 9. 紫鸟匹配不上:requireMatchedShop 直接抛,建单不留下任何行
Fixture unmatched = new Fixture();
unmatched.matchFailure = "店铺未匹配";
sb.append(runCase("unmatchedShopRejected", unmatched,
oneShop(7L, "Demo Shop", List.of("UK"))));
// 10~12. 国家代码三格判据(真实实现):非法、重复、空项/至少一个
Fixture usCountry = new Fixture();
sb.append(runCase("usIsNotSupportedCountry", usCountry,
oneShop(7L, "Demo Shop", List.of("US"))));
Fixture duplicateCountry = new Fixture();
sb.append(runCase("duplicateCountryCodeRejected", duplicateCountry,
oneShop(7L, "Demo Shop", List.of("uk", "UK"))));
Fixture emptyCountries = new Fixture();
sb.append(runCase("emptyCountryCodesRejected", emptyCountries,
oneShop(7L, "Demo Shop", List.of())));
Fixture blankCountry = new Fixture();
sb.append(runCase("blankCountryCodeRejected", blankCountry,
oneShop(7L, "Demo Shop", Arrays.asList("UK", " "))));
// 13~14. userId 两格(null 与 0)都走同一句「user_id 不合法」
Fixture nullUser = new Fixture();
sb.append(runCase("nullUserIdRejected", nullUser,
oneShop(null, "Demo Shop", List.of("UK"))));
Fixture zeroUser = new Fixture();
sb.append(runCase("zeroUserIdRejected", zeroUser,
oneShop(0L, "Demo Shop", List.of("UK"))));
return sb.toString();
}
private static String read(File file) throws Exception {
return Files.readString(file.toPath(), StandardCharsets.UTF_8).replace("\r\n", "\n");
}
@Test
void create_task_snapshot_matches_golden() throws Exception {
assertEquals(read(GOLDEN_CREATE), runAll(), "建单落库流水与 golden 一致");
}
@Test
void create_task_snapshot_is_reproducible() {
assertEquals(runAll(), runAll(), "同一夹具跑两次流水一致(时钟与雪花 id 不得进快照)");
}
@Test
void create_task_snapshot_golden_is_committed() {
assertTrue(GOLDEN_CREATE.isFile(), "golden 文件必须存在并入库: " + GOLDEN_CREATE.getAbsolutePath());
assertTrue(GOLDEN_CREATE.length() > 0, "golden 文件非空");
}
@Test
void create_task_snapshot_tamper_is_detected() throws Exception {
String original = read(GOLDEN_CREATE);
assertTrue(original.contains("createdAtEqualsTaskCreatedAt=true"),
"golden 必须钉住「建单时刻一枚复用给头行与每一行」这条底座量不到的形状");
try {
Files.writeString(GOLDEN_CREATE.toPath(), original + "\n# tampered", StandardCharsets.UTF_8);
assertTrue(!read(GOLDEN_CREATE).equals(runAll()), "篡改 golden 后断言应失败");
} finally {
Files.writeString(GOLDEN_CREATE.toPath(), original, StandardCharsets.UTF_8);
}
assertEquals(original, read(GOLDEN_CREATE), "恢复原始 golden");
}
/** 只在显式点名时写 golden:默认跑测试永远不许改是非源。 */
@Test
@EnabledIfEnvironmentVariable(named = "SHUFU_SNAPSHOT_WRITE", matches = "true")
void write_golden_on_demand() throws Exception {
Files.createDirectories(GOLDEN_CREATE.getParentFile().toPath());
Files.writeString(GOLDEN_CREATE.toPath(), runAll(), StandardCharsets.UTF_8);
}
}
@@ -0,0 +1,808 @@
package com.nanri.aiimage.modules.shopdatacrawl.service.support;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.databind.json.JsonMapper;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import com.nanri.aiimage.config.InstanceMetadata;
import com.nanri.aiimage.config.TaskPressureProperties;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryResultDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto;
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResultRequest;
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlDailyFileService;
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlExcelAssemblyService;
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlItemStoreService;
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlResolveService;
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskCacheService;
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskService;
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import com.nanri.aiimage.modules.task.service.TaskResultItemService;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.apache.ibatis.session.Configuration;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
import org.springframework.dao.DuplicateKeyException;
import java.io.File;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.ArgumentMatchers.nullable;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
/**
* 店铺数据抓取回传落库(段 3)的现网行为快照——Phase 3 迁移 3B 的护身符。
*
* <p><b>为什么补这一支。</b>蓝图 §4 Phase 3 首迁实录把话说死了:「那五支的 {@code submitResult}
* 同样没有 golden ⇒ 各自补完才能动」。{@code ShopDataCrawlSnapshotTest} 锁的是 Excel 分组与行布局,
* 一行库都没碰;建单与回传两段此前只有断言式用例({@code ShopDataCrawlTaskServiceChunkTest}),
* 那些用例断的是<b>结论</b>({@code result.getSuccess()==1}),不锁<b>副作用先后</b>。而 3B 要改的
* 恰恰是先后:准入步序、合并与标记的顺序、终态那格的收尾链。所以这一支必须落在迁移<b>之前</b>。
*
* <p><b>这一支同时钉住 3B 的三处「不许顺手统一」</b>(Phase 2/3 逐条登记过的差异,代码注释里也写着):
* <ol>
* <li>{@code caseOwnershipCheckedBeforeTerminal}:本支现网先查归属再判终态,与底座
* {@code admitSubmission} 的默认序相反。若迁移时改成「直接套底座那一派」,
* 同一枚既被别人持有又已终态的任务会从 {@code TaskOwnerMismatchException}(40903)
* 变成「任务已结束,拒绝重复提交」——本例当场判红。</li>
* <li>{@code caseTerminalTaskRejectedWithoutCode}:终态重复提交本支<b>不带</b>
* {@code TASK_ALREADY_FINISHED}(40901),线协议与 ShopMatch/PriceTrack 那两支不同
* (那两支是 success=true、Python 停止重试;本支是 success=false)。
* golden 里那行 {@code code=-} 就是这件事的唯一读数。</li>
* <li>{@code caseCompletedRowFlushesWorkbookColumns}:终态收尾的结果列写走
* {@code LambdaUpdateWrapper} 批量条件更新(只刷 {@code success=1} 的行、不写
* {@code success}/{@code errorMessage}、{@code objectKey} 传 null),
* 底座 {@code publishResultFile} 对它<b>不适用</b>。本例把「一条 batchUpdate、零条
* {@code result.update}」这个形状钉住,套错件就红。</li>
* </ol>
*
* <p><b>快照吃的是事件流水,不是分组清单。</b>顺序换了就红。时钟、雪花 id、OSS 直链不进快照;
* 分片内容哈希取前 12 位(既灵敏又不把 golden 变成一堵十六进制墙)。
*
* golden:src/test/resources/shopdatacrawl/golden/submit-result-snapshot.txt
* 重新生成:{@code SHUFU_SNAPSHOT_WRITE=true mvn -o test -Dtest=ShopDataCrawlSubmitResultSnapshotTest}
* (用环境变量而不是 -D:pom 里 {@code <argLine>} 是显式配置,命令行覆盖不动它。
* 这条 golden 只许在改动本支<b>之前</b>生成一次,此后它就是是非源)
*/
class ShopDataCrawlSubmitResultSnapshotTest {
private static final File GOLDEN_SUBMIT =
new File("src/test/resources/shopdatacrawl/golden/submit-result-snapshot.txt");
private static final String MODULE_TYPE = "SHOP_DATA_CRAWL";
private static final String SHOP_NAME = "Demo Shop";
private static final Long TASK_ID = 1001L;
@BeforeAll
static void initializeMybatisMetadata() {
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new Configuration(), "");
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
TableInfoHelper.initTableInfo(assistant, FileResultEntity.class);
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class);
}
/** Spring 注入的那只 ObjectMapper ≠ {@code new ObjectMapper()}:Boot 会挂上 jsr310(Phase 0 实测教训)。 */
private static ObjectMapper bootLikeObjectMapper() {
return JsonMapper.builder()
.addModule(new JavaTimeModule())
.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS)
.build();
}
// ------------------------------------------------------------------
// 夹具:一份内存世界(结果行/合并载荷/分片表/scope/RustFS),每次用例全新建一套
// ------------------------------------------------------------------
private static final class Fixture {
private final ObjectMapper objectMapper = bootLikeObjectMapper();
private final List<String> events = new ArrayList<>();
private final Map<String, ShopDataCrawlShopPayloadDto> mergedPayloads = new LinkedHashMap<>();
private final List<FileResultEntity> rows = new ArrayList<>();
private final List<TaskChunkEntity> chunks = new ArrayList<>();
private final List<TaskScopeStateEntity> scopes = new ArrayList<>();
private final Map<String, String> rustfsPayloads = new LinkedHashMap<>();
private final FileTaskEntity task = new FileTaskEntity();
private ShopDataCrawlTaskService service;
private int heartbeats;
private int lockCloses;
private int nextPayloadId;
private boolean sharedWriteEnabled = true;
private Fixture withRows(String... shopNames) {
long id = 2001L;
for (String shopName : shopNames) {
FileResultEntity row = new FileResultEntity();
row.setId(id++);
row.setTaskId(TASK_ID);
row.setModuleType(MODULE_TYPE);
row.setSourceFilename(shopName);
row.setSourceFileUrl("shop-1");
row.setUserId(7L);
row.setSuccess(-1);
row.setCreatedAt(LocalDateTime.now());
rows.add(row);
}
return this;
}
/** {@code owner} 传 null 表示 requestJson 里不带 ownerInstanceId(走「未绑定实例」分支)。 */
private Fixture withTask(String status, String moduleType, String owner) {
task.setId(TASK_ID);
task.setUserId(7L);
task.setModuleType(moduleType);
task.setStatus(status);
task.setRequestJson(owner == null ? "{}" : "{\"ownerInstanceId\":\"" + owner + "\"}");
task.setResultJson("[]");
task.setCreatedAt(LocalDateTime.now());
task.setUpdatedAt(LocalDateTime.now());
return this;
}
private ShopDataCrawlTaskService service() {
if (service != null) {
return service;
}
FileTaskMapper fileTaskMapper = mock(FileTaskMapper.class);
FileResultMapper fileResultMapper = mock(FileResultMapper.class);
ShopDataCrawlResolveService resolveService = mock(ShopDataCrawlResolveService.class);
ShopDataCrawlExcelAssemblyService assembly = mock(ShopDataCrawlExcelAssemblyService.class);
ShopDataCrawlTaskCacheService cacheService = mock(ShopDataCrawlTaskCacheService.class);
ZiniaoShopSwitchService ziniao = mock(ZiniaoShopSwitchService.class);
TaskFileJobService fileJobService = mock(TaskFileJobService.class);
TaskResultItemService resultItemService = mock(TaskResultItemService.class);
TaskProgressSnapshotService progressSnapshotService = mock(TaskProgressSnapshotService.class);
TaskDistributedLockService lockService = mock(TaskDistributedLockService.class);
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
TaskChunkMapper chunkMapper = mock(TaskChunkMapper.class);
TaskScopeStateMapper scopeMapper = mock(TaskScopeStateMapper.class);
TransientPayloadStorageService payloadStorage = mock(TransientPayloadStorageService.class);
InstanceMetadata instanceMetadata = mock(InstanceMetadata.class);
doAnswer(invocation -> {
events.add("result.update: " + renderRow(invocation.getArgument(0)));
return 1;
}).when(fileResultMapper).updateById(any(FileResultEntity.class));
doAnswer(invocation -> {
events.add("task.update: " + renderTask(invocation.getArgument(0)));
return 1;
}).when(fileTaskMapper).updateById(any(FileTaskEntity.class));
when(fileTaskMapper.selectById(anyLong())).thenReturn(task);
when(fileResultMapper.selectList(any())).thenAnswer(invocation -> List.copyOf(rows));
when(fileTaskMapper.selectList(any())).thenAnswer(invocation -> List.of(task));
// 终态收尾那条「只刷成功行、不动状态位」的批量条件更新(底座 publishResultFile 不适用的那格)
doAnswer(invocation -> {
LambdaUpdateWrapper<FileResultEntity> wrapper = invocation.getArgument(1);
events.add("result.batchUpdate: " + renderUpdateWrapper(wrapper));
return 1;
}).when(fileResultMapper).update(any(), any());
when(lockService.acquire(eq(MODULE_TYPE), anyLong())).thenReturn(lock);
when(lockService.acquire(eq(MODULE_TYPE), anyLong(), anyLong())).thenReturn(lock);
doAnswer(invocation -> {
lockCloses++;
return null;
}).when(lock).close();
when(cacheService.getTaskCacheBatch(any())).thenReturn(Map.of());
doAnswer(invocation -> {
heartbeats++;
return null;
}).when(cacheService).touchTaskHeartbeat(any());
when(cacheService.getShopMergedPayload(anyLong(), anyString()))
.thenAnswer(invocation -> mergedPayloads.get(invocation.<String>getArgument(1)));
when(cacheService.getAllShopMergedPayload(anyLong())).thenAnswer(invocation ->
new LinkedHashMap<>(mergedPayloads));
doAnswer(invocation -> {
String shopKey = invocation.getArgument(1);
ShopDataCrawlShopPayloadDto payload = invocation.getArgument(2);
mergedPayloads.put(shopKey, payload);
events.add("mergedSave: taskId=" + invocation.<Long>getArgument(0)
+ " shopKey=" + shopKey + " " + renderPayload(payload));
return null;
}).when(cacheService).saveShopMergedPayload(anyLong(), anyString(), any());
doAnswer(invocation -> {
events.add("mergedRemove: taskId=" + invocation.getArgument(0)
+ " shopKey=" + invocation.<String>getArgument(1));
mergedPayloads.remove(invocation.<String>getArgument(1));
return null;
}).when(cacheService).removeShopMergedPayload(anyLong(), anyString());
doAnswer(invocation -> {
events.add("taskCacheSave: id=" + ((FileTaskEntity) invocation.getArgument(0)).getId());
return null;
}).when(cacheService).saveTaskCache(any(FileTaskEntity.class));
doAnswer(invocation -> {
events.add("taskCacheDelete: taskId=" + invocation.getArgument(0));
return null;
}).when(cacheService).deleteTaskCache(any());
doAnswer(invocation -> {
events.add("snapshotTable.replace: taskId=" + invocation.getArgument(0)
+ " items=" + ((List<?>) invocation.getArgument(2)).size());
return null;
}).when(resultItemService).replaceTaskSnapshots(anyLong(), anyString(), any(), any());
when(resultItemService.listResultSnapshots(anyLong(), anyString(), any())).thenReturn(List.of());
doAnswer(invocation -> {
events.add("progressSave: taskId=" + invocation.getArgument(0)
+ " status=" + dash(invocation.getArgument(2))
+ " total=" + invocation.<Integer>getArgument(3)
+ " ok=" + invocation.<Integer>getArgument(4)
+ " fail=" + invocation.<Integer>getArgument(5));
return null;
}).when(progressSnapshotService).save(anyLong(), anyString(), anyString(),
anyInt(), anyInt(), anyInt(), any(), any(), any());
when(fileJobService.findAssembleJob(anyLong(), anyString(), anyLong())).thenReturn(null);
when(fileJobService.findAssembleJobsByResultIds(anyString(), any())).thenReturn(Map.of());
when(fileJobService.countUnfinishedAssembleJobs(anyLong(), anyString())).thenReturn(0L);
doAnswer(invocation -> {
events.add("enqueue: taskId=" + invocation.getArgument(0)
+ " resultId=" + invocation.getArgument(2)
+ " scopeKey=" + invocation.<String>getArgument(3));
return null;
}).when(fileJobService).enqueueAssembleResult(anyLong(), anyString(), anyLong(), anyString());
when(ziniao.normalizeShopName(nullable(String.class))).thenAnswer(invocation -> {
String shopName = invocation.<String>getArgument(0);
return shopName == null ? "" : shopName.trim();
});
when(assembly.countRows(any())).thenAnswer(invocation ->
((List<?>) invocation.getArgument(0)).size());
configurePayloadStorage(payloadStorage);
configureChunkMapper(chunkMapper);
configureScopeMapper(scopeMapper);
lenient().when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
service = new ShopDataCrawlTaskService(
fileTaskMapper,
fileResultMapper,
resolveService,
assembly,
cacheService,
mock(OssStorageService.class),
ziniao,
objectMapper,
mock(TaskPressureProperties.class),
fileJobService,
resultItemService,
progressSnapshotService,
lockService,
chunkMapper,
scopeMapper,
payloadStorage,
instanceMetadata,
mock(ShopDataCrawlDailyFileService.class),
mock(ShopDataCrawlItemStoreService.class),
mock(DuplicateCheckRefreshPort.class),
null,
mock(TaskProgressLightAssembler.class),
mock(TaskHeartbeatPositionService.class));
return service;
}
private void configurePayloadStorage(TransientPayloadStorageService storage) {
lenient().when(storage.isSharedWriteEnabled()).thenReturn(sharedWriteEnabled);
lenient().when(storage.storeChunkPayloadVersioned(anyString(), anyLong(), anyString(), any(), anyString()))
.thenAnswer(invocation -> {
String pointer = "rustfs:payload-" + (++nextPayloadId);
rustfsPayloads.put(pointer, invocation.getArgument(4));
events.add("payloadStore: pointer=" + pointer);
return pointer;
});
lenient().when(storage.extractPointer(nullable(String.class))).thenAnswer(invocation -> {
String value = invocation.<String>getArgument(0);
return value != null && value.startsWith("rustfs:") ? value : null;
});
lenient().when(storage.resolvePayload(any(), any())).thenAnswer(invocation -> {
String pointer = invocation.<String>getArgument(0);
events.add("payloadResolve: pointer=" + pointer);
String payload = rustfsPayloads.get(pointer);
if (payload == null) {
throw new IllegalStateException("missing test RustFS payload: " + pointer);
}
return payload;
});
lenient().doAnswer(invocation -> {
String pointer = invocation.<String>getArgument(0);
if (rustfsPayloads.containsKey(pointer)) {
events.add("payloadDelete: pointer=" + pointer);
rustfsPayloads.remove(pointer);
}
return null;
}).when(storage).deletePayloadIfPresent(any());
}
private void configureChunkMapper(TaskChunkMapper chunkMapper) {
lenient().when(chunkMapper.insert(any(TaskChunkEntity.class))).thenAnswer(invocation -> {
TaskChunkEntity chunk = invocation.getArgument(0);
boolean duplicate = chunks.stream().anyMatch(existing ->
Objects.equals(existing.getTaskId(), chunk.getTaskId())
&& Objects.equals(existing.getScopeHash(), chunk.getScopeHash())
&& Objects.equals(existing.getChunkIndex(), chunk.getChunkIndex()));
if (duplicate) {
throw new DuplicateKeyException("duplicate chunk key: "
+ chunk.getScopeHash() + "/" + chunk.getChunkIndex());
}
chunk.setId((long) chunks.size() + 1L);
chunks.add(chunk);
events.add("chunkInsert: idx=" + chunk.getChunkIndex() + "/" + chunk.getChunkTotal()
+ " hash=" + shortHash(chunk.getPayloadHash()) + " scopeKey=" + chunk.getScopeKey());
return 1;
});
lenient().when(chunkMapper.selectOne(any())).thenAnswer(invocation -> {
LambdaQueryWrapper<TaskChunkEntity> query = invocation.getArgument(0);
String scopeHash = queryHash(query);
Integer chunkIndex = queryInt(query);
return chunks.stream()
.filter(chunk -> Objects.equals(scopeHash, chunk.getScopeHash()))
.filter(chunk -> Objects.equals(chunkIndex, chunk.getChunkIndex()))
.findFirst().orElse(null);
});
lenient().when(chunkMapper.selectList(any())).thenAnswer(invocation -> chunks.stream()
.sorted(Comparator.comparing(TaskChunkEntity::getChunkIndex)).toList());
lenient().when(chunkMapper.selectCount(any())).thenAnswer(invocation -> (long) chunks.size());
lenient().when(chunkMapper.delete(any())).thenAnswer(invocation -> {
int removed = chunks.size();
if (removed > 0) {
events.add("chunkDelete: count=" + removed);
}
chunks.clear();
return removed;
});
}
private void configureScopeMapper(TaskScopeStateMapper scopeMapper) {
lenient().when(scopeMapper.insert(any(TaskScopeStateEntity.class))).thenAnswer(invocation -> {
TaskScopeStateEntity scope = invocation.getArgument(0);
scope.setId((long) scopes.size() + 1L);
scopes.add(scope);
events.add("scopeInsert: " + renderScope(scope));
return 1;
});
lenient().when(scopeMapper.selectOne(any())).thenAnswer(invocation -> scopes.stream()
.filter(scope -> Objects.equals(queryHash(invocation.getArgument(0)), scope.getScopeHash()))
.findFirst().orElse(null));
lenient().when(scopeMapper.updateById(any(TaskScopeStateEntity.class))).thenAnswer(invocation -> {
TaskScopeStateEntity scope = invocation.getArgument(0);
if (!scopes.contains(scope)) {
scopes.add(scope);
}
events.add("scopeUpdate: " + renderScope(scope));
return 1;
});
lenient().when(scopeMapper.delete(any())).thenAnswer(invocation -> {
int removed = scopes.size();
if (removed > 0) {
events.add("scopeDelete: count=" + removed);
}
scopes.clear();
return removed;
});
}
}
// ------------------------------------------------------------------
// 渲染(时钟/雪花 id/OSS 直链不进快照)
// ------------------------------------------------------------------
private static String renderRow(FileResultEntity row) {
return "id=" + row.getId()
+ " success=" + row.getSuccess()
+ " error=" + dash(row.getErrorMessage())
+ " filename=" + dash(row.getResultFilename())
+ " urlPresent=" + (row.getResultFileUrl() != null && !row.getResultFileUrl().isBlank())
+ " size=" + row.getResultFileSize()
+ " contentType=" + dash(row.getResultContentType())
+ " rowCount=" + row.getRowCount();
}
private static String renderTask(FileTaskEntity task) {
return "id=" + task.getId()
+ " status=" + dash(task.getStatus())
+ " ok=" + task.getSuccessFileCount()
+ " fail=" + task.getFailedFileCount()
+ " sourceFileCount=" + task.getSourceFileCount()
+ " error=" + dash(task.getErrorMessage())
+ " resultJsonItems=" + (task.getResultJson() == null ? -1
: count(task.getResultJson(), "\"resultId\""))
+ " resultJsonHasRows=" + (task.getResultJson() != null
&& task.getResultJson().contains("\"asin\""))
+ " finishedPresent=" + (task.getFinishedAt() != null);
}
private static String renderScope(TaskScopeStateEntity scope) {
return "received=" + scope.getReceivedChunkCount()
+ " total=" + scope.getChunkTotal()
+ " completed=" + scope.getCompleted()
+ " scopeKey=" + dash(scope.getScopeKey());
}
private static String renderPayload(ShopDataCrawlShopPayloadDto payload) {
if (payload == null) {
return "payload=<null>";
}
int countries = payload.getCountryResults() == null ? 0 : payload.getCountryResults().size();
int rows = 0;
if (payload.getCountryResults() != null) {
for (ShopDataCrawlCountryResultDto country : payload.getCountryResults()) {
rows += country == null || country.getItems() == null ? 0 : country.getItems().size();
}
}
return "shopName=" + dash(payload.getShopName())
+ " error=" + dash(payload.getError())
+ " countries=" + countries + " rows=" + rows
+ " done=" + payload.getShopDone()
+ " chunk=" + payload.getChunkIndex() + "/" + payload.getChunkTotal();
}
/**
* 批量条件更新要认三件事:刷哪些列(顺序)、按哪些行(WHERE 原文)、带哪些值(按 {@code MPGENVAL} 序号)。
*
* <p>WHERE 只能取原文:MyBatis-Plus 对单元素集合走 {@code formatSql("{0}", value)} 直接拼进 SQL、
* 不进 {@code paramNameValuePairs},本支收尾只有首行那一枚 id,靠参数表读不到它。
*/
private static String renderUpdateWrapper(LambdaUpdateWrapper<FileResultEntity> wrapper) {
String sqlSet = wrapper.getSqlSet() == null ? "" : wrapper.getSqlSet();
String columns = Arrays.stream(sqlSet.split(","))
.map(part -> part.split("=")[0].trim())
.collect(Collectors.joining(","));
String params = wrapper.getParamNameValuePairs().entrySet().stream()
.sorted(Comparator.comparingInt(entry -> Integer.parseInt(entry.getKey().substring("MPGENVAL".length()))))
.map(entry -> entry.getKey() + "=" + (entry.getValue() instanceof String text
? "\"" + text + "\"" : String.valueOf(entry.getValue())))
.collect(Collectors.joining(","));
return "columns=" + dash(columns) + " where=" + wrapper.getSqlSegment() + " params=" + params;
}
private static String dash(Object value) {
String text = value == null ? "" : String.valueOf(value);
return text.isBlank() ? "-" : text;
}
private static String shortHash(String value) {
return value == null || value.length() < 12 ? dash(value) : value.substring(0, 12);
}
private static int count(String text, String needle) {
int occurrences = 0;
int index = text.indexOf(needle);
while (index >= 0) {
occurrences++;
index = text.indexOf(needle, index + needle.length());
}
return occurrences;
}
private static Integer queryInt(LambdaQueryWrapper<?> query) {
query.getSqlSegment();
return query.getParamNameValuePairs().values().stream()
.filter(Integer.class::isInstance)
.map(Integer.class::cast)
.findFirst().orElse(null);
}
private static String queryHash(LambdaQueryWrapper<?> query) {
query.getSqlSegment();
return query.getParamNameValuePairs().values().stream()
.filter(String.class::isInstance)
.map(String.class::cast)
.filter(value -> value.length() == 64)
.findFirst().orElse(null);
}
// ------------------------------------------------------------------
// 请求构造
// ------------------------------------------------------------------
private static ShopDataCrawlRowDto row(String date, String asin) {
ShopDataCrawlRowDto row = new ShopDataCrawlRowDto();
row.setDate(date);
row.setAsin(asin);
row.setBrand("Example Brand");
row.setInventorySales("10");
row.setSalesRank("20");
row.setPageViews("30");
row.setUnitsSold("40");
row.setPrice("50");
row.setRecommendedOffer("60");
return row;
}
private static ShopDataCrawlCountryResultDto country(String code, ShopDataCrawlRowDto... rows) {
ShopDataCrawlCountryResultDto country = new ShopDataCrawlCountryResultDto();
country.setCountry(code);
country.setItems(List.of(rows));
return country;
}
/** 旧版增量协议:不带 chunk_index/chunk_total,靠 shopDone 收尾。 */
private static ShopDataCrawlShopPayloadDto legacy(boolean shopDone, String country, ShopDataCrawlRowDto... rows) {
ShopDataCrawlShopPayloadDto payload = new ShopDataCrawlShopPayloadDto();
payload.setShopName(SHOP_NAME);
payload.setShopDone(shopDone);
payload.setCountryResults(country == null
? new ArrayList<>() : new ArrayList<>(List.of(country(country, rows))));
return payload;
}
private static ShopDataCrawlShopPayloadDto chunk(int index, int total, String country, ShopDataCrawlRowDto... rows) {
ShopDataCrawlShopPayloadDto payload = legacy(false, country, rows);
payload.setChunkIndex(index);
payload.setChunkTotal(total);
return payload;
}
private static ShopDataCrawlShopPayloadDto failedShop(String error) {
ShopDataCrawlShopPayloadDto payload = new ShopDataCrawlShopPayloadDto();
payload.setShopName(SHOP_NAME);
payload.setError(error);
return payload;
}
private static ShopDataCrawlSubmitResultRequest request(ShopDataCrawlShopPayloadDto... shops) {
ShopDataCrawlSubmitResultRequest request = new ShopDataCrawlSubmitResultRequest();
request.setShops(List.of(shops));
return request;
}
// ------------------------------------------------------------------
// 用例
// ------------------------------------------------------------------
private interface Emit {
String render() throws Exception;
}
private static void appendCase(StringBuilder sb, String caseName, Emit emit) {
sb.append("== case=").append(caseName).append('\n');
try {
sb.append(emit.render());
} catch (Exception ex) {
sb.append("thrown: ").append(ex.getClass().getSimpleName())
.append(" code=").append(ex instanceof com.nanri.aiimage.common.exception.BusinessException businessException
? dash(businessException.getCode()) : "-")
.append(" message=").append(dash(ex.getMessage())).append('\n');
}
}
private static String submit(Fixture fixture, ShopDataCrawlSubmitResultRequest request) {
fixture.service().submitResult(TASK_ID, request);
StringBuilder sb = new StringBuilder();
fixture.events.forEach(line -> sb.append(line).append('\n'));
return sb.append(counters(fixture)).toString();
}
private static String counters(Fixture fixture) {
return "events=" + fixture.events.size()
+ " heartbeatTouches=" + fixture.heartbeats
+ " lockCloses=" + fixture.lockCloses
+ " chunks=" + fixture.chunks.size()
+ " scopes=" + fixture.scopes.size()
+ " rustfs=" + fixture.rustfsPayloads.size() + '\n';
}
/** 默认世界:RUNNING + 单店一行 + 归属本实例。服务在首次用到时才建(夹具开关得先就位)。 */
private static Fixture running() {
Fixture fixture = new Fixture();
fixture.withRows(SHOP_NAME).withTask("RUNNING", MODULE_TYPE, "instance-a");
return fixture;
}
/** 只取本次调用的流水(分片跨调用场景:前一次提交的事件不算)。 */
private static String submitFreshEvents(Fixture fixture, ShopDataCrawlSubmitResultRequest request) {
fixture.events.clear();
return submit(fixture, request);
}
public static String runAll() throws Exception {
StringBuilder sb = new StringBuilder();
// 1. 旧协议一片到底:合并→行标成功→清合并载荷→进度落库→tryFinalize 收尾链
Fixture happy = running();
appendCase(sb, "happyLegacyDone", () -> submit(happy, request(legacy(true, "UK", row("2026-07-25", "B001")))));
// 2. 店铺报错:先并快照再标失败,任务终态 FAILED,且终态无成功行要走「静默清理分片」那格
Fixture failed = running();
appendCase(sb, "failedShopFinalizesTaskFailed", () -> submit(failed, request(failedShop("店铺打不开"))));
// 3. 没 done 也没分片元数据:行留在待处理(-1),任务表照常刷一次进度
Fixture waiting = running();
appendCase(sb, "legacyWithoutDoneWaits", () -> submit(waiting, request(legacy(false, "UK", row("2026-07-25", "B001")))));
// 4. 回传店名与任务不符:normalizePayloadByShop 直接拒,一行都不许被碰
Fixture mismatched = running();
appendCase(sb, "shopNameMismatchRejected", () -> {
ShopDataCrawlShopPayloadDto other = legacy(true, "UK", row("2026-07-25", "B001"));
other.setShopName("别的店");
return submit(mismatched, request(other));
});
// 5. 一次回传塞两个店铺:本支的合同是「必须且只能一个」,抢锁之前就拒
Fixture twoShops = running();
appendCase(sb, "twoShopsRejectedBeforeLock", () -> submit(twoShops,
request(legacy(true, "UK", row("2026-07-25", "B001")), legacy(true, "DE", row("2026-07-26", "B002")))));
// 6. 整个请求为 null
Fixture nullRequest = running();
appendCase(sb, "nullRequestRejectedBeforeLock", () -> submit(nullRequest, null));
// 7. shops 里夹 null:进到建索引那一步才拒(与第 5 例是两条不同出口)
Fixture nullShop = running();
ShopDataCrawlSubmitResultRequest nullish = new ShopDataCrawlSubmitResultRequest();
nullish.setShops(Arrays.asList((ShopDataCrawlShopPayloadDto) null));
appendCase(sb, "nullShopInListRejected", () -> submit(nullShop, nullish));
// 8. taskId 非法:锁一次都不该被碰
Fixture badId = running();
appendCase(sb, "invalidTaskIdRejectedBeforeLock", () -> {
StringBuilder inner = new StringBuilder();
try {
badId.service().submitResult(0L, request(legacy(true, "UK", row("2026-07-25", "B001"))));
inner.append("thrown: none\n");
} catch (Exception ex) {
inner.append("thrown: ").append(ex.getClass().getSimpleName())
.append(" code=").append(ex instanceof com.nanri.aiimage.common.exception.BusinessException businessException
? dash(businessException.getCode()) : "-")
.append(" message=").append(dash(ex.getMessage())).append('\n');
}
return inner.append(counters(badId)).toString();
});
// 9. 任务已终态:本支是<b>不带 code</b> 的 BusinessException(与孪生两支的 40901 是两种线协议)
Fixture terminal = new Fixture();
terminal.withRows(SHOP_NAME).withTask("SUCCESS", MODULE_TYPE, "instance-a");
appendCase(sb, "terminalTaskRejectedWithoutCode", () -> submit(terminal,
request(legacy(true, "UK", row("2026-07-25", "B001")))));
// 10. 任务属于别的模块:「任务不存在」
Fixture wrongModule = new Fixture();
wrongModule.withRows(SHOP_NAME).withTask("RUNNING", "APPEARANCE_PATENT", "instance-a");
appendCase(sb, "wrongModuleTaskNotFound", () -> submit(wrongModule,
request(legacy(true, "UK", row("2026-07-25", "B001")))));
// 11. 归属别人<b>且</b>已终态:本支现网先查归属 ⇒ 40903,不是终态拒绝(步序钉死格)
Fixture foreign = new Fixture();
foreign.withRows(SHOP_NAME).withTask("SUCCESS", MODULE_TYPE, "instance-b");
appendCase(sb, "ownershipCheckedBeforeTerminal", () -> submit(foreign,
request(legacy(true, "UK", row("2026-07-25", "B001")))));
// 12. 任务里两行:本支的合同是「一支一店」,「任务店铺数据异常」
Fixture twoRows = new Fixture();
twoRows.withRows(SHOP_NAME, "店铺B").withTask("RUNNING", MODULE_TYPE, "instance-a");
appendCase(sb, "twoResultRowsRejected", () -> submit(twoRows,
request(legacy(true, "UK", row("2026-07-25", "B001")))));
// 13. 显式分片、只到一片:落 RustFS+分片行+scope 计数,行状态<b>不动</b>
Fixture partialChunk = running();
appendCase(sb, "partialChunkStored", () -> submit(partialChunk,
request(chunk(1, 2, "DE", row("2026-07-25", "B001")))));
// 14. 第二片先到、第一片后到:完成判定按 chunk_index 顺序合并,不受到达顺序影响
Fixture reordered = running();
reordered.service().submitResult(TASK_ID, request(chunk(2, 2, "UK", row("2026-07-26", "B002"))));
appendCase(sb, "secondChunkArrivesFirst", () -> submitFreshEvents(reordered,
request(chunk(1, 2, "DE", row("2026-07-25", "B001")))));
// 15. 同一片重复回传(内容一致):唯一键冲突被吸收,计数器不重复加
Fixture retried = running();
ShopDataCrawlSubmitResultRequest first = request(chunk(1, 2, "DE", row("2026-07-25", "B001")));
retried.service().submitResult(TASK_ID, first);
appendCase(sb, "identicalChunkRetryAbsorbed", () -> submitFreshEvents(retried, first));
// 16. 同一片换了内容:拒(分片幂等靠内容哈希,不是靠 index)
Fixture conflicting = running();
conflicting.service().submitResult(TASK_ID, request(chunk(1, 2, "DE", row("2026-07-25", "B001"))));
appendCase(sb, "chunkRetryWithDifferentContentRejected", () -> submitFreshEvents(conflicting,
request(chunk(1, 2, "DE", row("2026-07-25", "B099")))));
// 17. chunk_index 与 chunk_total 只传一个:建索引后第一道元数据校验就拒
Fixture halfMetadata = running();
appendCase(sb, "chunkIndexWithoutTotalRejected", () -> {
ShopDataCrawlShopPayloadDto payload = legacy(true, "DE", row("2026-07-25", "B001"));
payload.setChunkIndex(1);
return submit(halfMetadata, request(payload));
});
// 18. chunk_total 中途改口:同一店铺必须一致
Fixture changedTotal = running();
changedTotal.service().submitResult(TASK_ID, request(chunk(1, 2, "DE", row("2026-07-25", "B001"))));
appendCase(sb, "chunkTotalChangeRejected", () -> submitFreshEvents(changedTotal,
request(chunk(2, 3, "UK", row("2026-07-26", "B002")))));
// 19. RustFS 没开:分片一条都不许落库
Fixture noRustfs = running();
noRustfs.sharedWriteEnabled = false;
appendCase(sb, "chunkRejectedWhenRustfsDisabled", () -> submit(noRustfs,
request(chunk(1, 1, "DE", row("2026-07-25", "B001")))));
// 20. 单片即全片:走 loadCompleteResultPayload→行标成功→终态收尾刷结果列(批量条件更新那格)
Fixture singleChunk = running();
appendCase(sb, "completedRowFlushesWorkbookColumns", () -> submit(singleChunk,
request(chunk(1, 1, "UK", row("2026-07-25", "B001"), row("2026-07-26", "B002")))));
return sb.toString();
}
private static String read(File file) throws Exception {
return Files.readString(file.toPath(), StandardCharsets.UTF_8).replace("\r\n", "\n");
}
@Test
void submit_result_snapshot_matches_golden() throws Exception {
assertEquals(read(GOLDEN_SUBMIT), runAll(), "回传段落库流水与 golden 一致");
}
@Test
void submit_result_snapshot_is_reproducible() throws Exception {
assertEquals(runAll(), runAll(), "同一夹具跑两次流水一致(时钟与雪花 id 不得进快照)");
}
@Test
void submit_result_snapshot_golden_is_committed() {
assertTrue(GOLDEN_SUBMIT.isFile(), "golden 文件必须存在并入库: " + GOLDEN_SUBMIT.getAbsolutePath());
assertTrue(GOLDEN_SUBMIT.length() > 0, "golden 文件非空");
}
@Test
void submit_result_snapshot_tamper_is_detected() throws Exception {
String original = read(GOLDEN_SUBMIT);
assertTrue(original.contains("== case=happyLegacyDone"), "golden 内容合法");
try {
Files.writeString(GOLDEN_SUBMIT.toPath(), original + "\n# tampered", StandardCharsets.UTF_8);
assertTrue(!read(GOLDEN_SUBMIT).equals(runAll()), "篡改 golden 后断言应失败");
} finally {
Files.writeString(GOLDEN_SUBMIT.toPath(), original, StandardCharsets.UTF_8);
}
assertEquals(original, read(GOLDEN_SUBMIT), "恢复原始 golden");
}
/** 只在显式点名时写 golden:默认跑测试永远不许改是非源。 */
@Test
@EnabledIfEnvironmentVariable(named = "SHUFU_SNAPSHOT_WRITE", matches = "true")
void write_golden_on_demand() throws Exception {
Files.createDirectories(GOLDEN_SUBMIT.getParentFile().toPath());
Files.writeString(GOLDEN_SUBMIT.toPath(), runAll(), StandardCharsets.UTF_8);
}
}