test(phase-3): 3B 迁移前把 shopdatacrawl 的建单与回传两段钉成 golden
蓝图 §4 Phase 3 首迁实录写的开工前置第①条——「那五支的 submitResult 同样没有 golden, 各自补完才能动」。本笔只补件、不动生产码一支。 回传段 20 例、建单段 14 例,渲染的是副作用**先后**流水而不是分组清单,三格专门钉住 Phase 2/3 已登记的「不许顺手统一」: - ownershipCheckedBeforeTerminal:本支先查归属再判终态(底座 admitSubmission 默认序相反), 套错就 40903 → 「任务已结束」; - terminalTaskRejectedWithoutCode:终态重复提交本支不带 40901,线协议与孪生两支不同; - completedRowFlushesWorkbookColumns:收尾刷结果列走 LambdaUpdateWrapper 批量条件更新 (只刷 success=1 的行、不写 success/errorMessage、objectKey 传 null),publishResultFile 不适用。 建单段另钉一格快照本来看不见的性质:建单时刻一枚复用给头行与每一行结果行,故只断 createdAtEqualsTaskCreatedAt=true 而不打印时钟——底座五参 newTaskHeader 是「逐行各取一次」那一派, 照搬就把这格改成 false。
This commit is contained in:
+487
@@ -0,0 +1,487 @@
|
||||
package com.nanri.aiimage.modules.shopdatacrawl.service.support;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.SerializationFeature;
|
||||
import com.fasterxml.jackson.databind.json.JsonMapper;
|
||||
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.config.InstanceMetadata;
|
||||
import com.nanri.aiimage.config.TaskPressureProperties;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.mapper.ShopDataCrawlCountryPrefMapper;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.mapper.ShopDataCrawlShopCandidateMapper;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCreateTaskRequest;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlTaskItemDto;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlCreateTaskVo;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.vo.ShopDataCrawlResultItemVo;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlDailyFileService;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlExcelAssemblyService;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlItemStoreService;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlResolveService;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskCacheService;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskService;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
|
||||
import com.nanri.aiimage.modules.shopkey.service.ShopManageService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskResultItemService;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoShopMatchResultVo;
|
||||
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
|
||||
|
||||
import java.io.File;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 店铺数据抓取建单(段 1 落库形状)的现网行为快照——Phase 3 迁移 3B 的另一半护身符。
|
||||
*
|
||||
* <p><b>为什么非但要有回传快照、还要有建单快照。</b>3B 要动的不只是 {@code submitResult}:
|
||||
* 建单那截的 task 头行与结果行是底座 {@code newTaskHeader}/{@code newScopeRow} 的直接替换点。
|
||||
* 而本支有一个<b>快照本来量不到</b>的性质——建单时刻 {@code LocalDateTime.now()} 被<b>一枚复用</b>给
|
||||
* 头行和每一行结果行({@code ShopDataCrawlTaskService:574 → :586-587 / :605})。时钟不能进快照
|
||||
* (进了就逐字节对撞不了),所以这里只断言<b>相等关系</b>:每行渲染一枚
|
||||
* {@code createdAtEqualsTaskCreatedAt=}。底座五参版 {@code newTaskHeader} 取的是「默认时区、逐行各取一次」
|
||||
* 那一派,照搬就会把这格从 {@code true} 改成 {@code false}——这条只有这一处量得到。
|
||||
*
|
||||
* <p><b>另外三格被钉住的本支现网形状:</b>
|
||||
* <ul>
|
||||
* <li>{@code success=-1}(本支的 {@code RESULT_PENDING},不是其余六支的 {@code 0});
|
||||
* {@code rowCount} 留 NULL(本支建单不写这一列)。</li>
|
||||
* <li>{@code successFileCount}/{@code failedFileCount} 建单时写 0(底座刻意不初始化,见其注释)。</li>
|
||||
* <li>{@code taskNo} 只渲染前缀:雪花 id 进快照就永远对撞不上。</li>
|
||||
* </ul>
|
||||
*
|
||||
* <p>国家代码校验走的是 {@link ShopDataCrawlResolveService#validateCountryCodes} <b>真实实现</b>
|
||||
* (那段是 static、不碰库),店铺匹配则按现网出口逐例打桩——快照要锁的是本支怎么用这些件,
|
||||
* 不是替它们重写一遍判据。
|
||||
*
|
||||
* golden:src/test/resources/shopdatacrawl/golden/create-task-snapshot.txt
|
||||
* 重新生成:{@code SHUFU_SNAPSHOT_WRITE=true mvn -o test -Dtest=ShopDataCrawlCreateTaskSnapshotTest}
|
||||
*/
|
||||
class ShopDataCrawlCreateTaskSnapshotTest {
|
||||
|
||||
private static final File GOLDEN_CREATE =
|
||||
new File("src/test/resources/shopdatacrawl/golden/create-task-snapshot.txt");
|
||||
private static final String MODULE_TYPE = "SHOP_DATA_CRAWL";
|
||||
|
||||
/** Spring 注入的那只 ObjectMapper ≠ {@code new ObjectMapper()}:Boot 会挂上 jsr310(Phase 0 实测教训)。 */
|
||||
private static ObjectMapper bootLikeObjectMapper() {
|
||||
return JsonMapper.builder()
|
||||
.addModule(new JavaTimeModule())
|
||||
.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS)
|
||||
.build();
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// 夹具
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
private static final class Fixture {
|
||||
private final ObjectMapper objectMapper = bootLikeObjectMapper();
|
||||
private final List<String> events = new ArrayList<>();
|
||||
private final List<FileTaskEntity> taskInserts = new ArrayList<>();
|
||||
private final List<FileResultEntity> resultInserts = new ArrayList<>();
|
||||
private ShopDataCrawlTaskService service;
|
||||
private int heartbeats;
|
||||
/** {@code >=0} 时,第 N+1 次归一化起返回空串——用来够到现网建单循环里那道二次归一化空白闸。 */
|
||||
private int blankNormalizeAfter = -1;
|
||||
private int normalizeCalls;
|
||||
/** 打桩 {@code requireMatchedShop} 的出口:null 表示匹配成功,否则按该文案抛错。 */
|
||||
private String matchFailure;
|
||||
|
||||
private ShopDataCrawlTaskService service() {
|
||||
if (service != null) {
|
||||
return service;
|
||||
}
|
||||
FileTaskMapper fileTaskMapper = mock(FileTaskMapper.class);
|
||||
FileResultMapper fileResultMapper = mock(FileResultMapper.class);
|
||||
ShopDataCrawlTaskCacheService cacheService = mock(ShopDataCrawlTaskCacheService.class);
|
||||
ZiniaoShopSwitchService ziniao = mock(ZiniaoShopSwitchService.class);
|
||||
TaskResultItemService resultItemService = mock(TaskResultItemService.class);
|
||||
TaskProgressSnapshotService progressSnapshotService = mock(TaskProgressSnapshotService.class);
|
||||
InstanceMetadata instanceMetadata = mock(InstanceMetadata.class);
|
||||
|
||||
// 国家代码判据走真实实现(static、不碰库),不为它另造一套形状
|
||||
ShopDataCrawlResolveService realResolve = new ShopDataCrawlResolveService(
|
||||
mock(ShopDataCrawlShopCandidateMapper.class), mock(ShopDataCrawlCountryPrefMapper.class),
|
||||
bootLikeObjectMapper(), ziniao, mock(ShopManageService.class));
|
||||
ShopDataCrawlResolveService resolveService = mock(ShopDataCrawlResolveService.class);
|
||||
when(resolveService.validateCountryCodes(any()))
|
||||
.thenAnswer(invocation -> realResolve.validateCountryCodes(invocation.getArgument(0)));
|
||||
when(resolveService.requireMatchedShop(anyString())).thenAnswer(invocation -> {
|
||||
if (matchFailure != null) {
|
||||
throw new BusinessException(matchFailure);
|
||||
}
|
||||
String shopName = invocation.getArgument(0);
|
||||
return new ZiniaoShopMatchResultVo(true, "shop-" + shopName.trim(), shopName,
|
||||
"公司-" + shopName.trim(), "Amazon", 9L, "https://example.invalid",
|
||||
"MATCHED", "店铺索引匹配成功");
|
||||
});
|
||||
|
||||
doAnswer(invocation -> {
|
||||
FileTaskEntity entity = invocation.getArgument(0);
|
||||
entity.setId(1001L + taskInserts.size());
|
||||
taskInserts.add(entity);
|
||||
events.add("task.insert: " + renderTaskInsert(entity));
|
||||
return 1;
|
||||
}).when(fileTaskMapper).insert(any(FileTaskEntity.class));
|
||||
doAnswer(invocation -> {
|
||||
events.add("task.update: " + renderTaskUpdate(invocation.<FileTaskEntity>getArgument(0)));
|
||||
return 1;
|
||||
}).when(fileTaskMapper).updateById(any(FileTaskEntity.class));
|
||||
doAnswer(invocation -> {
|
||||
FileResultEntity entity = invocation.getArgument(0);
|
||||
entity.setId(2001L + resultInserts.size());
|
||||
resultInserts.add(entity);
|
||||
events.add("result.insert: " + renderResultInsert(entity,
|
||||
taskInserts.isEmpty() ? null : taskInserts.get(0)));
|
||||
return 1;
|
||||
}).when(fileResultMapper).insert(any(FileResultEntity.class));
|
||||
|
||||
doAnswer(invocation -> {
|
||||
heartbeats++;
|
||||
return null;
|
||||
}).when(cacheService).touchTaskHeartbeat(any());
|
||||
doAnswer(invocation -> {
|
||||
events.add("taskCacheSave: id=" + ((FileTaskEntity) invocation.getArgument(0)).getId());
|
||||
return null;
|
||||
}).when(cacheService).saveTaskCache(any(FileTaskEntity.class));
|
||||
doAnswer(invocation -> {
|
||||
events.add("snapshotTable.replace: taskId=" + invocation.getArgument(0)
|
||||
+ " items=" + ((List<?>) invocation.getArgument(2)).size());
|
||||
return null;
|
||||
}).when(resultItemService).replaceTaskSnapshots(anyLong(), anyString(), any(), any());
|
||||
doAnswer(invocation -> {
|
||||
events.add("progressSave: taskId=" + invocation.getArgument(0)
|
||||
+ " status=" + dash(invocation.getArgument(2))
|
||||
+ " total=" + invocation.<Integer>getArgument(3)
|
||||
+ " ok=" + invocation.<Integer>getArgument(4)
|
||||
+ " fail=" + invocation.<Integer>getArgument(5));
|
||||
return null;
|
||||
}).when(progressSnapshotService).save(anyLong(), anyString(), anyString(),
|
||||
org.mockito.ArgumentMatchers.anyInt(), org.mockito.ArgumentMatchers.anyInt(),
|
||||
org.mockito.ArgumentMatchers.anyInt(), any(), any(), any());
|
||||
when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
|
||||
when(ziniao.normalizeShopName(any())).thenAnswer(invocation -> {
|
||||
String shopName = invocation.<String>getArgument(0);
|
||||
normalizeCalls++;
|
||||
if (blankNormalizeAfter >= 0 && normalizeCalls > blankNormalizeAfter) {
|
||||
return "";
|
||||
}
|
||||
return shopName == null ? "" : shopName.trim();
|
||||
});
|
||||
|
||||
service = new ShopDataCrawlTaskService(
|
||||
fileTaskMapper,
|
||||
fileResultMapper,
|
||||
resolveService,
|
||||
mock(ShopDataCrawlExcelAssemblyService.class),
|
||||
cacheService,
|
||||
mock(OssStorageService.class),
|
||||
ziniao,
|
||||
objectMapper,
|
||||
mock(TaskPressureProperties.class),
|
||||
mock(TaskFileJobService.class),
|
||||
resultItemService,
|
||||
progressSnapshotService,
|
||||
mock(TaskDistributedLockService.class),
|
||||
mock(TaskChunkMapper.class),
|
||||
mock(TaskScopeStateMapper.class),
|
||||
mock(TransientPayloadStorageService.class),
|
||||
instanceMetadata,
|
||||
mock(ShopDataCrawlDailyFileService.class),
|
||||
mock(ShopDataCrawlItemStoreService.class),
|
||||
mock(DuplicateCheckRefreshPort.class),
|
||||
null,
|
||||
mock(TaskProgressLightAssembler.class),
|
||||
mock(TaskHeartbeatPositionService.class));
|
||||
return service;
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// 渲染(时钟/雪花 id 不进快照,只断相等关系)
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
private static String renderTaskInsert(FileTaskEntity task) {
|
||||
return "id=" + task.getId()
|
||||
+ " taskNoPrefix=" + prefix(task.getTaskNo())
|
||||
+ " moduleType=" + dash(task.getModuleType())
|
||||
+ " taskMode=" + dash(task.getTaskMode())
|
||||
+ " status=" + dash(task.getStatus())
|
||||
+ " sourceFileCount=" + task.getSourceFileCount()
|
||||
+ " ok=" + task.getSuccessFileCount()
|
||||
+ " fail=" + task.getFailedFileCount()
|
||||
+ " userId=" + dash(task.getUserId())
|
||||
+ " createdBy=" + dash(task.getCreatedBy())
|
||||
+ " deviceId=" + dash(task.getDeviceId())
|
||||
+ " ownerInstanceId=" + dash(task.getOwnerInstanceId())
|
||||
+ " requestJsonPresent=" + (task.getRequestJson() != null)
|
||||
+ " resultJsonPresent=" + (task.getResultJson() != null);
|
||||
}
|
||||
|
||||
private static String renderTaskUpdate(FileTaskEntity task) {
|
||||
return "id=" + task.getId()
|
||||
+ " status=" + dash(task.getStatus())
|
||||
+ " ok=" + task.getSuccessFileCount()
|
||||
+ " fail=" + task.getFailedFileCount()
|
||||
+ " requestJsonHasOwner=" + (task.getRequestJson() != null
|
||||
&& task.getRequestJson().contains("\"ownerInstanceId\":\"instance-a\""))
|
||||
+ " requestJsonItems=" + (task.getRequestJson() == null ? -1
|
||||
: count(task.getRequestJson(), "\"shopName\""))
|
||||
+ " resultJsonItems=" + (task.getResultJson() == null ? -1
|
||||
: count(task.getResultJson(), "\"resultId\""))
|
||||
+ " createdAtEqualsUpdatedAt=" + (task.getCreatedAt() != null
|
||||
&& task.getCreatedAt().equals(task.getUpdatedAt()));
|
||||
}
|
||||
|
||||
private static String renderResultInsert(FileResultEntity row, FileTaskEntity task) {
|
||||
return "id=" + row.getId()
|
||||
+ " taskId=" + dash(row.getTaskId())
|
||||
+ " moduleType=" + dash(row.getModuleType())
|
||||
+ " sourceFilename=" + dash(row.getSourceFilename())
|
||||
+ " sourceFileUrl=" + dash(row.getSourceFileUrl())
|
||||
+ " userId=" + dash(row.getUserId())
|
||||
+ " success=" + dash(row.getSuccess())
|
||||
+ " rowCount=" + dash(row.getRowCount())
|
||||
+ " createdAtEqualsTaskCreatedAt=" + (task != null && row.getCreatedAt() != null
|
||||
&& row.getCreatedAt().equals(task.getCreatedAt()));
|
||||
}
|
||||
|
||||
private static String prefix(String taskNo) {
|
||||
if (taskNo == null) {
|
||||
return "-";
|
||||
}
|
||||
int index = taskNo.lastIndexOf('-');
|
||||
return index < 0 ? taskNo : taskNo.substring(0, index);
|
||||
}
|
||||
|
||||
private static String dash(Object value) {
|
||||
String text = value == null ? "" : String.valueOf(value);
|
||||
return text.isBlank() ? "-" : text;
|
||||
}
|
||||
|
||||
private static int count(String text, String needle) {
|
||||
int occurrences = 0;
|
||||
int index = text.indexOf(needle);
|
||||
while (index >= 0) {
|
||||
occurrences++;
|
||||
index = text.indexOf(needle, index + needle.length());
|
||||
}
|
||||
return occurrences;
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// 请求构造
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
private static ShopDataCrawlTaskItemDto shop(String shopName) {
|
||||
ShopDataCrawlTaskItemDto item = new ShopDataCrawlTaskItemDto();
|
||||
item.setShopName(shopName);
|
||||
item.setShopId("请求里带的旧 id");
|
||||
item.setPlatform("请求里带的旧平台");
|
||||
item.setCompanyName("请求里带的旧公司");
|
||||
item.setMatchStatus("请求里带的旧状态");
|
||||
item.setMatchMessage("请求里带的旧说明");
|
||||
return item;
|
||||
}
|
||||
|
||||
private static ShopDataCrawlCreateTaskRequest request(Long userId,
|
||||
List<ShopDataCrawlTaskItemDto> items,
|
||||
List<String> countryCodes) {
|
||||
ShopDataCrawlCreateTaskRequest request = new ShopDataCrawlCreateTaskRequest();
|
||||
request.setUserId(userId);
|
||||
request.setItems(items);
|
||||
request.setCountryCodes(countryCodes);
|
||||
return request;
|
||||
}
|
||||
|
||||
private static ShopDataCrawlCreateTaskRequest oneShop(Long userId, String shopName, List<String> countryCodes) {
|
||||
return request(userId, new ArrayList<>(List.of(shop(shopName))), countryCodes);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// 用例
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
private static String runCase(String caseName, Fixture fixture, ShopDataCrawlCreateTaskRequest request) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append("== case=").append(caseName).append('\n');
|
||||
try {
|
||||
ShopDataCrawlCreateTaskVo vo = fixture.service().createTask(request, "dev-1");
|
||||
sb.append("vo: taskId=").append(dash(vo.getTaskId()))
|
||||
.append(" items=").append(vo.getItems().size()).append('\n');
|
||||
for (ShopDataCrawlResultItemVo item : vo.getItems()) {
|
||||
sb.append(" item: resultId=").append(dash(item.getResultId()))
|
||||
.append(" taskId=").append(dash(item.getTaskId()))
|
||||
.append(" shopName=").append(dash(item.getShopName()))
|
||||
.append(" shopId=").append(dash(item.getShopId()))
|
||||
.append(" platform=").append(dash(item.getPlatform()))
|
||||
.append(" companyName=").append(dash(item.getCompanyName()))
|
||||
.append(" matched=").append(item.isMatched())
|
||||
.append(" matchStatus=").append(dash(item.getMatchStatus()))
|
||||
.append(" taskStatus=").append(dash(item.getTaskStatus()))
|
||||
.append(" success=").append(dash(item.getSuccess()))
|
||||
.append(" error=").append(dash(item.getError()))
|
||||
.append(" countryCodes=").append(item.getCountryCodes())
|
||||
.append(" countryResults=").append(item.getCountryResults() == null ? -1
|
||||
: item.getCountryResults().size())
|
||||
.append(" downloadUrl=").append(dash(item.getDownloadUrl()))
|
||||
.append('\n');
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
sb.append("thrown: ").append(ex.getClass().getSimpleName())
|
||||
.append(" code=").append(ex instanceof BusinessException businessException
|
||||
? dash(businessException.getCode()) : "-")
|
||||
.append(" message=").append(dash(ex.getMessage())).append('\n');
|
||||
}
|
||||
sb.append("events=").append(fixture.events.size())
|
||||
.append(" taskInserts=").append(fixture.taskInserts.size())
|
||||
.append(" resultInserts=").append(fixture.resultInserts.size())
|
||||
.append(" heartbeatTouches=").append(fixture.heartbeats).append('\n');
|
||||
fixture.events.forEach(line -> sb.append(" evt: ").append(line).append('\n'));
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
public static String runAll() {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
|
||||
// 1. 单店两国家:头行先落(拿到自增主键)→ 缓存+心跳 → 结果行 → requestJson/resultJson 一起刷回
|
||||
Fixture happy = new Fixture();
|
||||
sb.append(runCase("happySingleShopTwoCountries", happy,
|
||||
oneShop(7L, " Demo Shop ", List.of("UK", "DE"))));
|
||||
|
||||
// 2. 请求里带的 shopId/platform/company 一律以紫鸟重新匹配到的为准(这几列会被覆盖)
|
||||
Fixture overwrite = new Fixture();
|
||||
sb.append(runCase("requestFieldsOverwrittenByMatch", overwrite,
|
||||
oneShop(7L, "Demo Shop", List.of("UK"))));
|
||||
|
||||
// 3. items 两枚:本支的合同是「必须且只能一个店铺」,且不碰库
|
||||
Fixture twoItems = new Fixture();
|
||||
sb.append(runCase("twoItemsRejected", twoItems,
|
||||
request(7L, new ArrayList<>(List.of(shop("店铺A"), shop("店铺B"))), List.of("UK"))));
|
||||
|
||||
// 4. items 空列表
|
||||
Fixture emptyItems = new Fixture();
|
||||
sb.append(runCase("emptyItemsRejected", emptyItems,
|
||||
request(7L, new ArrayList<>(), List.of("UK"))));
|
||||
|
||||
// 5. items 为 null
|
||||
Fixture nullItems = new Fixture();
|
||||
sb.append(runCase("nullItemsRejected", nullItems,
|
||||
request(7L, null, List.of("UK"))));
|
||||
|
||||
// 6. 两枚同店名:第一道闸数的是<b>原始</b> items(:552),dedupe(:557)根本没机会救它
|
||||
// ⇒「必须且只能包含一个有效店铺」那句只在 dedupe 把唯一一枚丢光时才到得了(下一例)
|
||||
Fixture duplicatedShops = new Fixture();
|
||||
sb.append(runCase("rawTwoItemsSameShopRejectedFirstGate", duplicatedShops,
|
||||
request(7L, new ArrayList<>(List.of(shop("店铺A"), shop(" 店铺A "))), List.of("UK"))));
|
||||
|
||||
// 7. 店名归一化后为空:去重阶段就被丢掉
|
||||
Fixture blankShop = new Fixture();
|
||||
sb.append(runCase("blankShopNameDropped", blankShop,
|
||||
oneShop(7L, " ", List.of("UK"))));
|
||||
|
||||
// 8. 建单循环里二次归一化变空(夹具把第 3 次调用打空):头行已落库、结果行一行都不写
|
||||
Fixture renormalizedBlank = new Fixture();
|
||||
renormalizedBlank.blankNormalizeAfter = 2;
|
||||
sb.append(runCase("reNormalizedToBlankRejects", renormalizedBlank,
|
||||
oneShop(7L, "Demo Shop", List.of("UK"))));
|
||||
|
||||
// 9. 紫鸟匹配不上:requireMatchedShop 直接抛,建单不留下任何行
|
||||
Fixture unmatched = new Fixture();
|
||||
unmatched.matchFailure = "店铺未匹配";
|
||||
sb.append(runCase("unmatchedShopRejected", unmatched,
|
||||
oneShop(7L, "Demo Shop", List.of("UK"))));
|
||||
|
||||
// 10~12. 国家代码三格判据(真实实现):非法、重复、空项/至少一个
|
||||
Fixture usCountry = new Fixture();
|
||||
sb.append(runCase("usIsNotSupportedCountry", usCountry,
|
||||
oneShop(7L, "Demo Shop", List.of("US"))));
|
||||
Fixture duplicateCountry = new Fixture();
|
||||
sb.append(runCase("duplicateCountryCodeRejected", duplicateCountry,
|
||||
oneShop(7L, "Demo Shop", List.of("uk", "UK"))));
|
||||
Fixture emptyCountries = new Fixture();
|
||||
sb.append(runCase("emptyCountryCodesRejected", emptyCountries,
|
||||
oneShop(7L, "Demo Shop", List.of())));
|
||||
Fixture blankCountry = new Fixture();
|
||||
sb.append(runCase("blankCountryCodeRejected", blankCountry,
|
||||
oneShop(7L, "Demo Shop", Arrays.asList("UK", " "))));
|
||||
|
||||
// 13~14. userId 两格(null 与 0)都走同一句「user_id 不合法」
|
||||
Fixture nullUser = new Fixture();
|
||||
sb.append(runCase("nullUserIdRejected", nullUser,
|
||||
oneShop(null, "Demo Shop", List.of("UK"))));
|
||||
Fixture zeroUser = new Fixture();
|
||||
sb.append(runCase("zeroUserIdRejected", zeroUser,
|
||||
oneShop(0L, "Demo Shop", List.of("UK"))));
|
||||
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
private static String read(File file) throws Exception {
|
||||
return Files.readString(file.toPath(), StandardCharsets.UTF_8).replace("\r\n", "\n");
|
||||
}
|
||||
|
||||
@Test
|
||||
void create_task_snapshot_matches_golden() throws Exception {
|
||||
assertEquals(read(GOLDEN_CREATE), runAll(), "建单落库流水与 golden 一致");
|
||||
}
|
||||
|
||||
@Test
|
||||
void create_task_snapshot_is_reproducible() {
|
||||
assertEquals(runAll(), runAll(), "同一夹具跑两次流水一致(时钟与雪花 id 不得进快照)");
|
||||
}
|
||||
|
||||
@Test
|
||||
void create_task_snapshot_golden_is_committed() {
|
||||
assertTrue(GOLDEN_CREATE.isFile(), "golden 文件必须存在并入库: " + GOLDEN_CREATE.getAbsolutePath());
|
||||
assertTrue(GOLDEN_CREATE.length() > 0, "golden 文件非空");
|
||||
}
|
||||
|
||||
@Test
|
||||
void create_task_snapshot_tamper_is_detected() throws Exception {
|
||||
String original = read(GOLDEN_CREATE);
|
||||
assertTrue(original.contains("createdAtEqualsTaskCreatedAt=true"),
|
||||
"golden 必须钉住「建单时刻一枚复用给头行与每一行」这条底座量不到的形状");
|
||||
try {
|
||||
Files.writeString(GOLDEN_CREATE.toPath(), original + "\n# tampered", StandardCharsets.UTF_8);
|
||||
assertTrue(!read(GOLDEN_CREATE).equals(runAll()), "篡改 golden 后断言应失败");
|
||||
} finally {
|
||||
Files.writeString(GOLDEN_CREATE.toPath(), original, StandardCharsets.UTF_8);
|
||||
}
|
||||
assertEquals(original, read(GOLDEN_CREATE), "恢复原始 golden");
|
||||
}
|
||||
|
||||
/** 只在显式点名时写 golden:默认跑测试永远不许改是非源。 */
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "SHUFU_SNAPSHOT_WRITE", matches = "true")
|
||||
void write_golden_on_demand() throws Exception {
|
||||
Files.createDirectories(GOLDEN_CREATE.getParentFile().toPath());
|
||||
Files.writeString(GOLDEN_CREATE.toPath(), runAll(), StandardCharsets.UTF_8);
|
||||
}
|
||||
}
|
||||
+808
@@ -0,0 +1,808 @@
|
||||
package com.nanri.aiimage.modules.shopdatacrawl.service.support;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.SerializationFeature;
|
||||
import com.fasterxml.jackson.databind.json.JsonMapper;
|
||||
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
|
||||
import com.nanri.aiimage.config.InstanceMetadata;
|
||||
import com.nanri.aiimage.config.TaskPressureProperties;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlCountryResultDto;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlRowDto;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlShopPayloadDto;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.model.dto.ShopDataCrawlSubmitResultRequest;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlDailyFileService;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlExcelAssemblyService;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlItemStoreService;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlResolveService;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskCacheService;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.service.ShopDataCrawlTaskService;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.spi.DuplicateCheckRefreshPort;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
|
||||
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||
import com.nanri.aiimage.modules.task.service.TaskResultItemService;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
import org.apache.ibatis.session.Configuration;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
|
||||
import org.springframework.dao.DuplicateKeyException;
|
||||
|
||||
import java.io.File;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Comparator;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyInt;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.ArgumentMatchers.nullable;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* 店铺数据抓取回传落库(段 3)的现网行为快照——Phase 3 迁移 3B 的护身符。
|
||||
*
|
||||
* <p><b>为什么补这一支。</b>蓝图 §4 Phase 3 首迁实录把话说死了:「那五支的 {@code submitResult}
|
||||
* 同样没有 golden ⇒ 各自补完才能动」。{@code ShopDataCrawlSnapshotTest} 锁的是 Excel 分组与行布局,
|
||||
* 一行库都没碰;建单与回传两段此前只有断言式用例({@code ShopDataCrawlTaskServiceChunkTest}),
|
||||
* 那些用例断的是<b>结论</b>({@code result.getSuccess()==1}),不锁<b>副作用先后</b>。而 3B 要改的
|
||||
* 恰恰是先后:准入步序、合并与标记的顺序、终态那格的收尾链。所以这一支必须落在迁移<b>之前</b>。
|
||||
*
|
||||
* <p><b>这一支同时钉住 3B 的三处「不许顺手统一」</b>(Phase 2/3 逐条登记过的差异,代码注释里也写着):
|
||||
* <ol>
|
||||
* <li>{@code caseOwnershipCheckedBeforeTerminal}:本支现网先查归属再判终态,与底座
|
||||
* {@code admitSubmission} 的默认序相反。若迁移时改成「直接套底座那一派」,
|
||||
* 同一枚既被别人持有又已终态的任务会从 {@code TaskOwnerMismatchException}(40903)
|
||||
* 变成「任务已结束,拒绝重复提交」——本例当场判红。</li>
|
||||
* <li>{@code caseTerminalTaskRejectedWithoutCode}:终态重复提交本支<b>不带</b>
|
||||
* {@code TASK_ALREADY_FINISHED}(40901),线协议与 ShopMatch/PriceTrack 那两支不同
|
||||
* (那两支是 success=true、Python 停止重试;本支是 success=false)。
|
||||
* golden 里那行 {@code code=-} 就是这件事的唯一读数。</li>
|
||||
* <li>{@code caseCompletedRowFlushesWorkbookColumns}:终态收尾的结果列写走
|
||||
* {@code LambdaUpdateWrapper} 批量条件更新(只刷 {@code success=1} 的行、不写
|
||||
* {@code success}/{@code errorMessage}、{@code objectKey} 传 null),
|
||||
* 底座 {@code publishResultFile} 对它<b>不适用</b>。本例把「一条 batchUpdate、零条
|
||||
* {@code result.update}」这个形状钉住,套错件就红。</li>
|
||||
* </ol>
|
||||
*
|
||||
* <p><b>快照吃的是事件流水,不是分组清单。</b>顺序换了就红。时钟、雪花 id、OSS 直链不进快照;
|
||||
* 分片内容哈希取前 12 位(既灵敏又不把 golden 变成一堵十六进制墙)。
|
||||
*
|
||||
* golden:src/test/resources/shopdatacrawl/golden/submit-result-snapshot.txt
|
||||
* 重新生成:{@code SHUFU_SNAPSHOT_WRITE=true mvn -o test -Dtest=ShopDataCrawlSubmitResultSnapshotTest}
|
||||
* (用环境变量而不是 -D:pom 里 {@code <argLine>} 是显式配置,命令行覆盖不动它。
|
||||
* 这条 golden 只许在改动本支<b>之前</b>生成一次,此后它就是是非源)
|
||||
*/
|
||||
class ShopDataCrawlSubmitResultSnapshotTest {
|
||||
|
||||
private static final File GOLDEN_SUBMIT =
|
||||
new File("src/test/resources/shopdatacrawl/golden/submit-result-snapshot.txt");
|
||||
private static final String MODULE_TYPE = "SHOP_DATA_CRAWL";
|
||||
private static final String SHOP_NAME = "Demo Shop";
|
||||
private static final Long TASK_ID = 1001L;
|
||||
|
||||
@BeforeAll
|
||||
static void initializeMybatisMetadata() {
|
||||
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new Configuration(), "");
|
||||
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
|
||||
TableInfoHelper.initTableInfo(assistant, FileResultEntity.class);
|
||||
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
|
||||
TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class);
|
||||
}
|
||||
|
||||
/** Spring 注入的那只 ObjectMapper ≠ {@code new ObjectMapper()}:Boot 会挂上 jsr310(Phase 0 实测教训)。 */
|
||||
private static ObjectMapper bootLikeObjectMapper() {
|
||||
return JsonMapper.builder()
|
||||
.addModule(new JavaTimeModule())
|
||||
.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS)
|
||||
.build();
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// 夹具:一份内存世界(结果行/合并载荷/分片表/scope/RustFS),每次用例全新建一套
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
private static final class Fixture {
|
||||
private final ObjectMapper objectMapper = bootLikeObjectMapper();
|
||||
private final List<String> events = new ArrayList<>();
|
||||
private final Map<String, ShopDataCrawlShopPayloadDto> mergedPayloads = new LinkedHashMap<>();
|
||||
private final List<FileResultEntity> rows = new ArrayList<>();
|
||||
private final List<TaskChunkEntity> chunks = new ArrayList<>();
|
||||
private final List<TaskScopeStateEntity> scopes = new ArrayList<>();
|
||||
private final Map<String, String> rustfsPayloads = new LinkedHashMap<>();
|
||||
private final FileTaskEntity task = new FileTaskEntity();
|
||||
private ShopDataCrawlTaskService service;
|
||||
private int heartbeats;
|
||||
private int lockCloses;
|
||||
private int nextPayloadId;
|
||||
private boolean sharedWriteEnabled = true;
|
||||
|
||||
private Fixture withRows(String... shopNames) {
|
||||
long id = 2001L;
|
||||
for (String shopName : shopNames) {
|
||||
FileResultEntity row = new FileResultEntity();
|
||||
row.setId(id++);
|
||||
row.setTaskId(TASK_ID);
|
||||
row.setModuleType(MODULE_TYPE);
|
||||
row.setSourceFilename(shopName);
|
||||
row.setSourceFileUrl("shop-1");
|
||||
row.setUserId(7L);
|
||||
row.setSuccess(-1);
|
||||
row.setCreatedAt(LocalDateTime.now());
|
||||
rows.add(row);
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
/** {@code owner} 传 null 表示 requestJson 里不带 ownerInstanceId(走「未绑定实例」分支)。 */
|
||||
private Fixture withTask(String status, String moduleType, String owner) {
|
||||
task.setId(TASK_ID);
|
||||
task.setUserId(7L);
|
||||
task.setModuleType(moduleType);
|
||||
task.setStatus(status);
|
||||
task.setRequestJson(owner == null ? "{}" : "{\"ownerInstanceId\":\"" + owner + "\"}");
|
||||
task.setResultJson("[]");
|
||||
task.setCreatedAt(LocalDateTime.now());
|
||||
task.setUpdatedAt(LocalDateTime.now());
|
||||
return this;
|
||||
}
|
||||
|
||||
private ShopDataCrawlTaskService service() {
|
||||
if (service != null) {
|
||||
return service;
|
||||
}
|
||||
FileTaskMapper fileTaskMapper = mock(FileTaskMapper.class);
|
||||
FileResultMapper fileResultMapper = mock(FileResultMapper.class);
|
||||
ShopDataCrawlResolveService resolveService = mock(ShopDataCrawlResolveService.class);
|
||||
ShopDataCrawlExcelAssemblyService assembly = mock(ShopDataCrawlExcelAssemblyService.class);
|
||||
ShopDataCrawlTaskCacheService cacheService = mock(ShopDataCrawlTaskCacheService.class);
|
||||
ZiniaoShopSwitchService ziniao = mock(ZiniaoShopSwitchService.class);
|
||||
TaskFileJobService fileJobService = mock(TaskFileJobService.class);
|
||||
TaskResultItemService resultItemService = mock(TaskResultItemService.class);
|
||||
TaskProgressSnapshotService progressSnapshotService = mock(TaskProgressSnapshotService.class);
|
||||
TaskDistributedLockService lockService = mock(TaskDistributedLockService.class);
|
||||
TaskDistributedLockService.LockHandle lock = mock(TaskDistributedLockService.LockHandle.class);
|
||||
TaskChunkMapper chunkMapper = mock(TaskChunkMapper.class);
|
||||
TaskScopeStateMapper scopeMapper = mock(TaskScopeStateMapper.class);
|
||||
TransientPayloadStorageService payloadStorage = mock(TransientPayloadStorageService.class);
|
||||
InstanceMetadata instanceMetadata = mock(InstanceMetadata.class);
|
||||
|
||||
doAnswer(invocation -> {
|
||||
events.add("result.update: " + renderRow(invocation.getArgument(0)));
|
||||
return 1;
|
||||
}).when(fileResultMapper).updateById(any(FileResultEntity.class));
|
||||
doAnswer(invocation -> {
|
||||
events.add("task.update: " + renderTask(invocation.getArgument(0)));
|
||||
return 1;
|
||||
}).when(fileTaskMapper).updateById(any(FileTaskEntity.class));
|
||||
when(fileTaskMapper.selectById(anyLong())).thenReturn(task);
|
||||
when(fileResultMapper.selectList(any())).thenAnswer(invocation -> List.copyOf(rows));
|
||||
when(fileTaskMapper.selectList(any())).thenAnswer(invocation -> List.of(task));
|
||||
|
||||
// 终态收尾那条「只刷成功行、不动状态位」的批量条件更新(底座 publishResultFile 不适用的那格)
|
||||
doAnswer(invocation -> {
|
||||
LambdaUpdateWrapper<FileResultEntity> wrapper = invocation.getArgument(1);
|
||||
events.add("result.batchUpdate: " + renderUpdateWrapper(wrapper));
|
||||
return 1;
|
||||
}).when(fileResultMapper).update(any(), any());
|
||||
|
||||
when(lockService.acquire(eq(MODULE_TYPE), anyLong())).thenReturn(lock);
|
||||
when(lockService.acquire(eq(MODULE_TYPE), anyLong(), anyLong())).thenReturn(lock);
|
||||
doAnswer(invocation -> {
|
||||
lockCloses++;
|
||||
return null;
|
||||
}).when(lock).close();
|
||||
|
||||
when(cacheService.getTaskCacheBatch(any())).thenReturn(Map.of());
|
||||
doAnswer(invocation -> {
|
||||
heartbeats++;
|
||||
return null;
|
||||
}).when(cacheService).touchTaskHeartbeat(any());
|
||||
when(cacheService.getShopMergedPayload(anyLong(), anyString()))
|
||||
.thenAnswer(invocation -> mergedPayloads.get(invocation.<String>getArgument(1)));
|
||||
when(cacheService.getAllShopMergedPayload(anyLong())).thenAnswer(invocation ->
|
||||
new LinkedHashMap<>(mergedPayloads));
|
||||
doAnswer(invocation -> {
|
||||
String shopKey = invocation.getArgument(1);
|
||||
ShopDataCrawlShopPayloadDto payload = invocation.getArgument(2);
|
||||
mergedPayloads.put(shopKey, payload);
|
||||
events.add("mergedSave: taskId=" + invocation.<Long>getArgument(0)
|
||||
+ " shopKey=" + shopKey + " " + renderPayload(payload));
|
||||
return null;
|
||||
}).when(cacheService).saveShopMergedPayload(anyLong(), anyString(), any());
|
||||
doAnswer(invocation -> {
|
||||
events.add("mergedRemove: taskId=" + invocation.getArgument(0)
|
||||
+ " shopKey=" + invocation.<String>getArgument(1));
|
||||
mergedPayloads.remove(invocation.<String>getArgument(1));
|
||||
return null;
|
||||
}).when(cacheService).removeShopMergedPayload(anyLong(), anyString());
|
||||
doAnswer(invocation -> {
|
||||
events.add("taskCacheSave: id=" + ((FileTaskEntity) invocation.getArgument(0)).getId());
|
||||
return null;
|
||||
}).when(cacheService).saveTaskCache(any(FileTaskEntity.class));
|
||||
doAnswer(invocation -> {
|
||||
events.add("taskCacheDelete: taskId=" + invocation.getArgument(0));
|
||||
return null;
|
||||
}).when(cacheService).deleteTaskCache(any());
|
||||
|
||||
doAnswer(invocation -> {
|
||||
events.add("snapshotTable.replace: taskId=" + invocation.getArgument(0)
|
||||
+ " items=" + ((List<?>) invocation.getArgument(2)).size());
|
||||
return null;
|
||||
}).when(resultItemService).replaceTaskSnapshots(anyLong(), anyString(), any(), any());
|
||||
when(resultItemService.listResultSnapshots(anyLong(), anyString(), any())).thenReturn(List.of());
|
||||
doAnswer(invocation -> {
|
||||
events.add("progressSave: taskId=" + invocation.getArgument(0)
|
||||
+ " status=" + dash(invocation.getArgument(2))
|
||||
+ " total=" + invocation.<Integer>getArgument(3)
|
||||
+ " ok=" + invocation.<Integer>getArgument(4)
|
||||
+ " fail=" + invocation.<Integer>getArgument(5));
|
||||
return null;
|
||||
}).when(progressSnapshotService).save(anyLong(), anyString(), anyString(),
|
||||
anyInt(), anyInt(), anyInt(), any(), any(), any());
|
||||
|
||||
when(fileJobService.findAssembleJob(anyLong(), anyString(), anyLong())).thenReturn(null);
|
||||
when(fileJobService.findAssembleJobsByResultIds(anyString(), any())).thenReturn(Map.of());
|
||||
when(fileJobService.countUnfinishedAssembleJobs(anyLong(), anyString())).thenReturn(0L);
|
||||
doAnswer(invocation -> {
|
||||
events.add("enqueue: taskId=" + invocation.getArgument(0)
|
||||
+ " resultId=" + invocation.getArgument(2)
|
||||
+ " scopeKey=" + invocation.<String>getArgument(3));
|
||||
return null;
|
||||
}).when(fileJobService).enqueueAssembleResult(anyLong(), anyString(), anyLong(), anyString());
|
||||
|
||||
when(ziniao.normalizeShopName(nullable(String.class))).thenAnswer(invocation -> {
|
||||
String shopName = invocation.<String>getArgument(0);
|
||||
return shopName == null ? "" : shopName.trim();
|
||||
});
|
||||
when(assembly.countRows(any())).thenAnswer(invocation ->
|
||||
((List<?>) invocation.getArgument(0)).size());
|
||||
|
||||
configurePayloadStorage(payloadStorage);
|
||||
configureChunkMapper(chunkMapper);
|
||||
configureScopeMapper(scopeMapper);
|
||||
|
||||
lenient().when(instanceMetadata.getInstanceId()).thenReturn("instance-a");
|
||||
|
||||
service = new ShopDataCrawlTaskService(
|
||||
fileTaskMapper,
|
||||
fileResultMapper,
|
||||
resolveService,
|
||||
assembly,
|
||||
cacheService,
|
||||
mock(OssStorageService.class),
|
||||
ziniao,
|
||||
objectMapper,
|
||||
mock(TaskPressureProperties.class),
|
||||
fileJobService,
|
||||
resultItemService,
|
||||
progressSnapshotService,
|
||||
lockService,
|
||||
chunkMapper,
|
||||
scopeMapper,
|
||||
payloadStorage,
|
||||
instanceMetadata,
|
||||
mock(ShopDataCrawlDailyFileService.class),
|
||||
mock(ShopDataCrawlItemStoreService.class),
|
||||
mock(DuplicateCheckRefreshPort.class),
|
||||
null,
|
||||
mock(TaskProgressLightAssembler.class),
|
||||
mock(TaskHeartbeatPositionService.class));
|
||||
return service;
|
||||
}
|
||||
|
||||
private void configurePayloadStorage(TransientPayloadStorageService storage) {
|
||||
lenient().when(storage.isSharedWriteEnabled()).thenReturn(sharedWriteEnabled);
|
||||
lenient().when(storage.storeChunkPayloadVersioned(anyString(), anyLong(), anyString(), any(), anyString()))
|
||||
.thenAnswer(invocation -> {
|
||||
String pointer = "rustfs:payload-" + (++nextPayloadId);
|
||||
rustfsPayloads.put(pointer, invocation.getArgument(4));
|
||||
events.add("payloadStore: pointer=" + pointer);
|
||||
return pointer;
|
||||
});
|
||||
lenient().when(storage.extractPointer(nullable(String.class))).thenAnswer(invocation -> {
|
||||
String value = invocation.<String>getArgument(0);
|
||||
return value != null && value.startsWith("rustfs:") ? value : null;
|
||||
});
|
||||
lenient().when(storage.resolvePayload(any(), any())).thenAnswer(invocation -> {
|
||||
String pointer = invocation.<String>getArgument(0);
|
||||
events.add("payloadResolve: pointer=" + pointer);
|
||||
String payload = rustfsPayloads.get(pointer);
|
||||
if (payload == null) {
|
||||
throw new IllegalStateException("missing test RustFS payload: " + pointer);
|
||||
}
|
||||
return payload;
|
||||
});
|
||||
lenient().doAnswer(invocation -> {
|
||||
String pointer = invocation.<String>getArgument(0);
|
||||
if (rustfsPayloads.containsKey(pointer)) {
|
||||
events.add("payloadDelete: pointer=" + pointer);
|
||||
rustfsPayloads.remove(pointer);
|
||||
}
|
||||
return null;
|
||||
}).when(storage).deletePayloadIfPresent(any());
|
||||
}
|
||||
|
||||
private void configureChunkMapper(TaskChunkMapper chunkMapper) {
|
||||
lenient().when(chunkMapper.insert(any(TaskChunkEntity.class))).thenAnswer(invocation -> {
|
||||
TaskChunkEntity chunk = invocation.getArgument(0);
|
||||
boolean duplicate = chunks.stream().anyMatch(existing ->
|
||||
Objects.equals(existing.getTaskId(), chunk.getTaskId())
|
||||
&& Objects.equals(existing.getScopeHash(), chunk.getScopeHash())
|
||||
&& Objects.equals(existing.getChunkIndex(), chunk.getChunkIndex()));
|
||||
if (duplicate) {
|
||||
throw new DuplicateKeyException("duplicate chunk key: "
|
||||
+ chunk.getScopeHash() + "/" + chunk.getChunkIndex());
|
||||
}
|
||||
chunk.setId((long) chunks.size() + 1L);
|
||||
chunks.add(chunk);
|
||||
events.add("chunkInsert: idx=" + chunk.getChunkIndex() + "/" + chunk.getChunkTotal()
|
||||
+ " hash=" + shortHash(chunk.getPayloadHash()) + " scopeKey=" + chunk.getScopeKey());
|
||||
return 1;
|
||||
});
|
||||
lenient().when(chunkMapper.selectOne(any())).thenAnswer(invocation -> {
|
||||
LambdaQueryWrapper<TaskChunkEntity> query = invocation.getArgument(0);
|
||||
String scopeHash = queryHash(query);
|
||||
Integer chunkIndex = queryInt(query);
|
||||
return chunks.stream()
|
||||
.filter(chunk -> Objects.equals(scopeHash, chunk.getScopeHash()))
|
||||
.filter(chunk -> Objects.equals(chunkIndex, chunk.getChunkIndex()))
|
||||
.findFirst().orElse(null);
|
||||
});
|
||||
lenient().when(chunkMapper.selectList(any())).thenAnswer(invocation -> chunks.stream()
|
||||
.sorted(Comparator.comparing(TaskChunkEntity::getChunkIndex)).toList());
|
||||
lenient().when(chunkMapper.selectCount(any())).thenAnswer(invocation -> (long) chunks.size());
|
||||
lenient().when(chunkMapper.delete(any())).thenAnswer(invocation -> {
|
||||
int removed = chunks.size();
|
||||
if (removed > 0) {
|
||||
events.add("chunkDelete: count=" + removed);
|
||||
}
|
||||
chunks.clear();
|
||||
return removed;
|
||||
});
|
||||
}
|
||||
|
||||
private void configureScopeMapper(TaskScopeStateMapper scopeMapper) {
|
||||
lenient().when(scopeMapper.insert(any(TaskScopeStateEntity.class))).thenAnswer(invocation -> {
|
||||
TaskScopeStateEntity scope = invocation.getArgument(0);
|
||||
scope.setId((long) scopes.size() + 1L);
|
||||
scopes.add(scope);
|
||||
events.add("scopeInsert: " + renderScope(scope));
|
||||
return 1;
|
||||
});
|
||||
lenient().when(scopeMapper.selectOne(any())).thenAnswer(invocation -> scopes.stream()
|
||||
.filter(scope -> Objects.equals(queryHash(invocation.getArgument(0)), scope.getScopeHash()))
|
||||
.findFirst().orElse(null));
|
||||
lenient().when(scopeMapper.updateById(any(TaskScopeStateEntity.class))).thenAnswer(invocation -> {
|
||||
TaskScopeStateEntity scope = invocation.getArgument(0);
|
||||
if (!scopes.contains(scope)) {
|
||||
scopes.add(scope);
|
||||
}
|
||||
events.add("scopeUpdate: " + renderScope(scope));
|
||||
return 1;
|
||||
});
|
||||
lenient().when(scopeMapper.delete(any())).thenAnswer(invocation -> {
|
||||
int removed = scopes.size();
|
||||
if (removed > 0) {
|
||||
events.add("scopeDelete: count=" + removed);
|
||||
}
|
||||
scopes.clear();
|
||||
return removed;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// 渲染(时钟/雪花 id/OSS 直链不进快照)
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
private static String renderRow(FileResultEntity row) {
|
||||
return "id=" + row.getId()
|
||||
+ " success=" + row.getSuccess()
|
||||
+ " error=" + dash(row.getErrorMessage())
|
||||
+ " filename=" + dash(row.getResultFilename())
|
||||
+ " urlPresent=" + (row.getResultFileUrl() != null && !row.getResultFileUrl().isBlank())
|
||||
+ " size=" + row.getResultFileSize()
|
||||
+ " contentType=" + dash(row.getResultContentType())
|
||||
+ " rowCount=" + row.getRowCount();
|
||||
}
|
||||
|
||||
private static String renderTask(FileTaskEntity task) {
|
||||
return "id=" + task.getId()
|
||||
+ " status=" + dash(task.getStatus())
|
||||
+ " ok=" + task.getSuccessFileCount()
|
||||
+ " fail=" + task.getFailedFileCount()
|
||||
+ " sourceFileCount=" + task.getSourceFileCount()
|
||||
+ " error=" + dash(task.getErrorMessage())
|
||||
+ " resultJsonItems=" + (task.getResultJson() == null ? -1
|
||||
: count(task.getResultJson(), "\"resultId\""))
|
||||
+ " resultJsonHasRows=" + (task.getResultJson() != null
|
||||
&& task.getResultJson().contains("\"asin\""))
|
||||
+ " finishedPresent=" + (task.getFinishedAt() != null);
|
||||
}
|
||||
|
||||
private static String renderScope(TaskScopeStateEntity scope) {
|
||||
return "received=" + scope.getReceivedChunkCount()
|
||||
+ " total=" + scope.getChunkTotal()
|
||||
+ " completed=" + scope.getCompleted()
|
||||
+ " scopeKey=" + dash(scope.getScopeKey());
|
||||
}
|
||||
|
||||
private static String renderPayload(ShopDataCrawlShopPayloadDto payload) {
|
||||
if (payload == null) {
|
||||
return "payload=<null>";
|
||||
}
|
||||
int countries = payload.getCountryResults() == null ? 0 : payload.getCountryResults().size();
|
||||
int rows = 0;
|
||||
if (payload.getCountryResults() != null) {
|
||||
for (ShopDataCrawlCountryResultDto country : payload.getCountryResults()) {
|
||||
rows += country == null || country.getItems() == null ? 0 : country.getItems().size();
|
||||
}
|
||||
}
|
||||
return "shopName=" + dash(payload.getShopName())
|
||||
+ " error=" + dash(payload.getError())
|
||||
+ " countries=" + countries + " rows=" + rows
|
||||
+ " done=" + payload.getShopDone()
|
||||
+ " chunk=" + payload.getChunkIndex() + "/" + payload.getChunkTotal();
|
||||
}
|
||||
|
||||
/**
|
||||
* 批量条件更新要认三件事:刷哪些列(顺序)、按哪些行(WHERE 原文)、带哪些值(按 {@code MPGENVAL} 序号)。
|
||||
*
|
||||
* <p>WHERE 只能取原文:MyBatis-Plus 对单元素集合走 {@code formatSql("{0}", value)} 直接拼进 SQL、
|
||||
* 不进 {@code paramNameValuePairs},本支收尾只有首行那一枚 id,靠参数表读不到它。
|
||||
*/
|
||||
private static String renderUpdateWrapper(LambdaUpdateWrapper<FileResultEntity> wrapper) {
|
||||
String sqlSet = wrapper.getSqlSet() == null ? "" : wrapper.getSqlSet();
|
||||
String columns = Arrays.stream(sqlSet.split(","))
|
||||
.map(part -> part.split("=")[0].trim())
|
||||
.collect(Collectors.joining(","));
|
||||
String params = wrapper.getParamNameValuePairs().entrySet().stream()
|
||||
.sorted(Comparator.comparingInt(entry -> Integer.parseInt(entry.getKey().substring("MPGENVAL".length()))))
|
||||
.map(entry -> entry.getKey() + "=" + (entry.getValue() instanceof String text
|
||||
? "\"" + text + "\"" : String.valueOf(entry.getValue())))
|
||||
.collect(Collectors.joining(","));
|
||||
return "columns=" + dash(columns) + " where=" + wrapper.getSqlSegment() + " params=" + params;
|
||||
}
|
||||
|
||||
private static String dash(Object value) {
|
||||
String text = value == null ? "" : String.valueOf(value);
|
||||
return text.isBlank() ? "-" : text;
|
||||
}
|
||||
|
||||
private static String shortHash(String value) {
|
||||
return value == null || value.length() < 12 ? dash(value) : value.substring(0, 12);
|
||||
}
|
||||
|
||||
private static int count(String text, String needle) {
|
||||
int occurrences = 0;
|
||||
int index = text.indexOf(needle);
|
||||
while (index >= 0) {
|
||||
occurrences++;
|
||||
index = text.indexOf(needle, index + needle.length());
|
||||
}
|
||||
return occurrences;
|
||||
}
|
||||
|
||||
private static Integer queryInt(LambdaQueryWrapper<?> query) {
|
||||
query.getSqlSegment();
|
||||
return query.getParamNameValuePairs().values().stream()
|
||||
.filter(Integer.class::isInstance)
|
||||
.map(Integer.class::cast)
|
||||
.findFirst().orElse(null);
|
||||
}
|
||||
|
||||
private static String queryHash(LambdaQueryWrapper<?> query) {
|
||||
query.getSqlSegment();
|
||||
return query.getParamNameValuePairs().values().stream()
|
||||
.filter(String.class::isInstance)
|
||||
.map(String.class::cast)
|
||||
.filter(value -> value.length() == 64)
|
||||
.findFirst().orElse(null);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// 请求构造
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
private static ShopDataCrawlRowDto row(String date, String asin) {
|
||||
ShopDataCrawlRowDto row = new ShopDataCrawlRowDto();
|
||||
row.setDate(date);
|
||||
row.setAsin(asin);
|
||||
row.setBrand("Example Brand");
|
||||
row.setInventorySales("10");
|
||||
row.setSalesRank("20");
|
||||
row.setPageViews("30");
|
||||
row.setUnitsSold("40");
|
||||
row.setPrice("50");
|
||||
row.setRecommendedOffer("60");
|
||||
return row;
|
||||
}
|
||||
|
||||
private static ShopDataCrawlCountryResultDto country(String code, ShopDataCrawlRowDto... rows) {
|
||||
ShopDataCrawlCountryResultDto country = new ShopDataCrawlCountryResultDto();
|
||||
country.setCountry(code);
|
||||
country.setItems(List.of(rows));
|
||||
return country;
|
||||
}
|
||||
|
||||
/** 旧版增量协议:不带 chunk_index/chunk_total,靠 shopDone 收尾。 */
|
||||
private static ShopDataCrawlShopPayloadDto legacy(boolean shopDone, String country, ShopDataCrawlRowDto... rows) {
|
||||
ShopDataCrawlShopPayloadDto payload = new ShopDataCrawlShopPayloadDto();
|
||||
payload.setShopName(SHOP_NAME);
|
||||
payload.setShopDone(shopDone);
|
||||
payload.setCountryResults(country == null
|
||||
? new ArrayList<>() : new ArrayList<>(List.of(country(country, rows))));
|
||||
return payload;
|
||||
}
|
||||
|
||||
private static ShopDataCrawlShopPayloadDto chunk(int index, int total, String country, ShopDataCrawlRowDto... rows) {
|
||||
ShopDataCrawlShopPayloadDto payload = legacy(false, country, rows);
|
||||
payload.setChunkIndex(index);
|
||||
payload.setChunkTotal(total);
|
||||
return payload;
|
||||
}
|
||||
|
||||
private static ShopDataCrawlShopPayloadDto failedShop(String error) {
|
||||
ShopDataCrawlShopPayloadDto payload = new ShopDataCrawlShopPayloadDto();
|
||||
payload.setShopName(SHOP_NAME);
|
||||
payload.setError(error);
|
||||
return payload;
|
||||
}
|
||||
|
||||
private static ShopDataCrawlSubmitResultRequest request(ShopDataCrawlShopPayloadDto... shops) {
|
||||
ShopDataCrawlSubmitResultRequest request = new ShopDataCrawlSubmitResultRequest();
|
||||
request.setShops(List.of(shops));
|
||||
return request;
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// 用例
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
private interface Emit {
|
||||
String render() throws Exception;
|
||||
}
|
||||
|
||||
private static void appendCase(StringBuilder sb, String caseName, Emit emit) {
|
||||
sb.append("== case=").append(caseName).append('\n');
|
||||
try {
|
||||
sb.append(emit.render());
|
||||
} catch (Exception ex) {
|
||||
sb.append("thrown: ").append(ex.getClass().getSimpleName())
|
||||
.append(" code=").append(ex instanceof com.nanri.aiimage.common.exception.BusinessException businessException
|
||||
? dash(businessException.getCode()) : "-")
|
||||
.append(" message=").append(dash(ex.getMessage())).append('\n');
|
||||
}
|
||||
}
|
||||
|
||||
private static String submit(Fixture fixture, ShopDataCrawlSubmitResultRequest request) {
|
||||
fixture.service().submitResult(TASK_ID, request);
|
||||
StringBuilder sb = new StringBuilder();
|
||||
fixture.events.forEach(line -> sb.append(line).append('\n'));
|
||||
return sb.append(counters(fixture)).toString();
|
||||
}
|
||||
|
||||
private static String counters(Fixture fixture) {
|
||||
return "events=" + fixture.events.size()
|
||||
+ " heartbeatTouches=" + fixture.heartbeats
|
||||
+ " lockCloses=" + fixture.lockCloses
|
||||
+ " chunks=" + fixture.chunks.size()
|
||||
+ " scopes=" + fixture.scopes.size()
|
||||
+ " rustfs=" + fixture.rustfsPayloads.size() + '\n';
|
||||
}
|
||||
|
||||
/** 默认世界:RUNNING + 单店一行 + 归属本实例。服务在首次用到时才建(夹具开关得先就位)。 */
|
||||
private static Fixture running() {
|
||||
Fixture fixture = new Fixture();
|
||||
fixture.withRows(SHOP_NAME).withTask("RUNNING", MODULE_TYPE, "instance-a");
|
||||
return fixture;
|
||||
}
|
||||
|
||||
/** 只取本次调用的流水(分片跨调用场景:前一次提交的事件不算)。 */
|
||||
private static String submitFreshEvents(Fixture fixture, ShopDataCrawlSubmitResultRequest request) {
|
||||
fixture.events.clear();
|
||||
return submit(fixture, request);
|
||||
}
|
||||
|
||||
public static String runAll() throws Exception {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
|
||||
// 1. 旧协议一片到底:合并→行标成功→清合并载荷→进度落库→tryFinalize 收尾链
|
||||
Fixture happy = running();
|
||||
appendCase(sb, "happyLegacyDone", () -> submit(happy, request(legacy(true, "UK", row("2026-07-25", "B001")))));
|
||||
|
||||
// 2. 店铺报错:先并快照再标失败,任务终态 FAILED,且终态无成功行要走「静默清理分片」那格
|
||||
Fixture failed = running();
|
||||
appendCase(sb, "failedShopFinalizesTaskFailed", () -> submit(failed, request(failedShop("店铺打不开"))));
|
||||
|
||||
// 3. 没 done 也没分片元数据:行留在待处理(-1),任务表照常刷一次进度
|
||||
Fixture waiting = running();
|
||||
appendCase(sb, "legacyWithoutDoneWaits", () -> submit(waiting, request(legacy(false, "UK", row("2026-07-25", "B001")))));
|
||||
|
||||
// 4. 回传店名与任务不符:normalizePayloadByShop 直接拒,一行都不许被碰
|
||||
Fixture mismatched = running();
|
||||
appendCase(sb, "shopNameMismatchRejected", () -> {
|
||||
ShopDataCrawlShopPayloadDto other = legacy(true, "UK", row("2026-07-25", "B001"));
|
||||
other.setShopName("别的店");
|
||||
return submit(mismatched, request(other));
|
||||
});
|
||||
|
||||
// 5. 一次回传塞两个店铺:本支的合同是「必须且只能一个」,抢锁之前就拒
|
||||
Fixture twoShops = running();
|
||||
appendCase(sb, "twoShopsRejectedBeforeLock", () -> submit(twoShops,
|
||||
request(legacy(true, "UK", row("2026-07-25", "B001")), legacy(true, "DE", row("2026-07-26", "B002")))));
|
||||
|
||||
// 6. 整个请求为 null
|
||||
Fixture nullRequest = running();
|
||||
appendCase(sb, "nullRequestRejectedBeforeLock", () -> submit(nullRequest, null));
|
||||
|
||||
// 7. shops 里夹 null:进到建索引那一步才拒(与第 5 例是两条不同出口)
|
||||
Fixture nullShop = running();
|
||||
ShopDataCrawlSubmitResultRequest nullish = new ShopDataCrawlSubmitResultRequest();
|
||||
nullish.setShops(Arrays.asList((ShopDataCrawlShopPayloadDto) null));
|
||||
appendCase(sb, "nullShopInListRejected", () -> submit(nullShop, nullish));
|
||||
|
||||
// 8. taskId 非法:锁一次都不该被碰
|
||||
Fixture badId = running();
|
||||
appendCase(sb, "invalidTaskIdRejectedBeforeLock", () -> {
|
||||
StringBuilder inner = new StringBuilder();
|
||||
try {
|
||||
badId.service().submitResult(0L, request(legacy(true, "UK", row("2026-07-25", "B001"))));
|
||||
inner.append("thrown: none\n");
|
||||
} catch (Exception ex) {
|
||||
inner.append("thrown: ").append(ex.getClass().getSimpleName())
|
||||
.append(" code=").append(ex instanceof com.nanri.aiimage.common.exception.BusinessException businessException
|
||||
? dash(businessException.getCode()) : "-")
|
||||
.append(" message=").append(dash(ex.getMessage())).append('\n');
|
||||
}
|
||||
return inner.append(counters(badId)).toString();
|
||||
});
|
||||
|
||||
// 9. 任务已终态:本支是<b>不带 code</b> 的 BusinessException(与孪生两支的 40901 是两种线协议)
|
||||
Fixture terminal = new Fixture();
|
||||
terminal.withRows(SHOP_NAME).withTask("SUCCESS", MODULE_TYPE, "instance-a");
|
||||
appendCase(sb, "terminalTaskRejectedWithoutCode", () -> submit(terminal,
|
||||
request(legacy(true, "UK", row("2026-07-25", "B001")))));
|
||||
|
||||
// 10. 任务属于别的模块:「任务不存在」
|
||||
Fixture wrongModule = new Fixture();
|
||||
wrongModule.withRows(SHOP_NAME).withTask("RUNNING", "APPEARANCE_PATENT", "instance-a");
|
||||
appendCase(sb, "wrongModuleTaskNotFound", () -> submit(wrongModule,
|
||||
request(legacy(true, "UK", row("2026-07-25", "B001")))));
|
||||
|
||||
// 11. 归属别人<b>且</b>已终态:本支现网先查归属 ⇒ 40903,不是终态拒绝(步序钉死格)
|
||||
Fixture foreign = new Fixture();
|
||||
foreign.withRows(SHOP_NAME).withTask("SUCCESS", MODULE_TYPE, "instance-b");
|
||||
appendCase(sb, "ownershipCheckedBeforeTerminal", () -> submit(foreign,
|
||||
request(legacy(true, "UK", row("2026-07-25", "B001")))));
|
||||
|
||||
// 12. 任务里两行:本支的合同是「一支一店」,「任务店铺数据异常」
|
||||
Fixture twoRows = new Fixture();
|
||||
twoRows.withRows(SHOP_NAME, "店铺B").withTask("RUNNING", MODULE_TYPE, "instance-a");
|
||||
appendCase(sb, "twoResultRowsRejected", () -> submit(twoRows,
|
||||
request(legacy(true, "UK", row("2026-07-25", "B001")))));
|
||||
|
||||
// 13. 显式分片、只到一片:落 RustFS+分片行+scope 计数,行状态<b>不动</b>
|
||||
Fixture partialChunk = running();
|
||||
appendCase(sb, "partialChunkStored", () -> submit(partialChunk,
|
||||
request(chunk(1, 2, "DE", row("2026-07-25", "B001")))));
|
||||
|
||||
// 14. 第二片先到、第一片后到:完成判定按 chunk_index 顺序合并,不受到达顺序影响
|
||||
Fixture reordered = running();
|
||||
reordered.service().submitResult(TASK_ID, request(chunk(2, 2, "UK", row("2026-07-26", "B002"))));
|
||||
appendCase(sb, "secondChunkArrivesFirst", () -> submitFreshEvents(reordered,
|
||||
request(chunk(1, 2, "DE", row("2026-07-25", "B001")))));
|
||||
|
||||
// 15. 同一片重复回传(内容一致):唯一键冲突被吸收,计数器不重复加
|
||||
Fixture retried = running();
|
||||
ShopDataCrawlSubmitResultRequest first = request(chunk(1, 2, "DE", row("2026-07-25", "B001")));
|
||||
retried.service().submitResult(TASK_ID, first);
|
||||
appendCase(sb, "identicalChunkRetryAbsorbed", () -> submitFreshEvents(retried, first));
|
||||
|
||||
// 16. 同一片换了内容:拒(分片幂等靠内容哈希,不是靠 index)
|
||||
Fixture conflicting = running();
|
||||
conflicting.service().submitResult(TASK_ID, request(chunk(1, 2, "DE", row("2026-07-25", "B001"))));
|
||||
appendCase(sb, "chunkRetryWithDifferentContentRejected", () -> submitFreshEvents(conflicting,
|
||||
request(chunk(1, 2, "DE", row("2026-07-25", "B099")))));
|
||||
|
||||
// 17. chunk_index 与 chunk_total 只传一个:建索引后第一道元数据校验就拒
|
||||
Fixture halfMetadata = running();
|
||||
appendCase(sb, "chunkIndexWithoutTotalRejected", () -> {
|
||||
ShopDataCrawlShopPayloadDto payload = legacy(true, "DE", row("2026-07-25", "B001"));
|
||||
payload.setChunkIndex(1);
|
||||
return submit(halfMetadata, request(payload));
|
||||
});
|
||||
|
||||
// 18. chunk_total 中途改口:同一店铺必须一致
|
||||
Fixture changedTotal = running();
|
||||
changedTotal.service().submitResult(TASK_ID, request(chunk(1, 2, "DE", row("2026-07-25", "B001"))));
|
||||
appendCase(sb, "chunkTotalChangeRejected", () -> submitFreshEvents(changedTotal,
|
||||
request(chunk(2, 3, "UK", row("2026-07-26", "B002")))));
|
||||
|
||||
// 19. RustFS 没开:分片一条都不许落库
|
||||
Fixture noRustfs = running();
|
||||
noRustfs.sharedWriteEnabled = false;
|
||||
appendCase(sb, "chunkRejectedWhenRustfsDisabled", () -> submit(noRustfs,
|
||||
request(chunk(1, 1, "DE", row("2026-07-25", "B001")))));
|
||||
|
||||
// 20. 单片即全片:走 loadCompleteResultPayload→行标成功→终态收尾刷结果列(批量条件更新那格)
|
||||
Fixture singleChunk = running();
|
||||
appendCase(sb, "completedRowFlushesWorkbookColumns", () -> submit(singleChunk,
|
||||
request(chunk(1, 1, "UK", row("2026-07-25", "B001"), row("2026-07-26", "B002")))));
|
||||
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
private static String read(File file) throws Exception {
|
||||
return Files.readString(file.toPath(), StandardCharsets.UTF_8).replace("\r\n", "\n");
|
||||
}
|
||||
|
||||
@Test
|
||||
void submit_result_snapshot_matches_golden() throws Exception {
|
||||
assertEquals(read(GOLDEN_SUBMIT), runAll(), "回传段落库流水与 golden 一致");
|
||||
}
|
||||
|
||||
@Test
|
||||
void submit_result_snapshot_is_reproducible() throws Exception {
|
||||
assertEquals(runAll(), runAll(), "同一夹具跑两次流水一致(时钟与雪花 id 不得进快照)");
|
||||
}
|
||||
|
||||
@Test
|
||||
void submit_result_snapshot_golden_is_committed() {
|
||||
assertTrue(GOLDEN_SUBMIT.isFile(), "golden 文件必须存在并入库: " + GOLDEN_SUBMIT.getAbsolutePath());
|
||||
assertTrue(GOLDEN_SUBMIT.length() > 0, "golden 文件非空");
|
||||
}
|
||||
|
||||
@Test
|
||||
void submit_result_snapshot_tamper_is_detected() throws Exception {
|
||||
String original = read(GOLDEN_SUBMIT);
|
||||
assertTrue(original.contains("== case=happyLegacyDone"), "golden 内容合法");
|
||||
try {
|
||||
Files.writeString(GOLDEN_SUBMIT.toPath(), original + "\n# tampered", StandardCharsets.UTF_8);
|
||||
assertTrue(!read(GOLDEN_SUBMIT).equals(runAll()), "篡改 golden 后断言应失败");
|
||||
} finally {
|
||||
Files.writeString(GOLDEN_SUBMIT.toPath(), original, StandardCharsets.UTF_8);
|
||||
}
|
||||
assertEquals(original, read(GOLDEN_SUBMIT), "恢复原始 golden");
|
||||
}
|
||||
|
||||
/** 只在显式点名时写 golden:默认跑测试永远不许改是非源。 */
|
||||
@Test
|
||||
@EnabledIfEnvironmentVariable(named = "SHUFU_SNAPSHOT_WRITE", matches = "true")
|
||||
void write_golden_on_demand() throws Exception {
|
||||
Files.createDirectories(GOLDEN_SUBMIT.getParentFile().toPath());
|
||||
Files.writeString(GOLDEN_SUBMIT.toPath(), runAll(), StandardCharsets.UTF_8);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
== case=happySingleShopTwoCountries
|
||||
vo: taskId=1001 items=1
|
||||
item: resultId=2001 taskId=1001 shopName=Demo Shop shopId=shop-Demo Shop platform=Amazon companyName=公司-Demo Shop matched=true matchStatus=MATCHED taskStatus=RUNNING success=- error=- countryCodes=[UK, DE] countryResults=0 downloadUrl=-
|
||||
events=7 taskInserts=1 resultInserts=1 heartbeatTouches=1
|
||||
evt: task.insert: id=1001 taskNoPrefix=SHOP_DATA_CRAWL moduleType=SHOP_DATA_CRAWL taskMode=PYTHON_QUEUE status=RUNNING sourceFileCount=1 ok=0 fail=0 userId=7 createdBy=user:7 deviceId=dev-1 ownerInstanceId=- requestJsonPresent=false resultJsonPresent=false
|
||||
evt: taskCacheSave: id=1001
|
||||
evt: result.insert: id=2001 taskId=1001 moduleType=SHOP_DATA_CRAWL sourceFilename=Demo Shop sourceFileUrl=shop-Demo Shop userId=7 success=-1 rowCount=- createdAtEqualsTaskCreatedAt=true
|
||||
evt: snapshotTable.replace: taskId=1001 items=1
|
||||
evt: progressSave: taskId=1001 status=RUNNING total=1 ok=0 fail=0
|
||||
evt: task.update: id=1001 status=RUNNING ok=0 fail=0 requestJsonHasOwner=true requestJsonItems=1 resultJsonItems=1 createdAtEqualsUpdatedAt=true
|
||||
evt: taskCacheSave: id=1001
|
||||
== case=requestFieldsOverwrittenByMatch
|
||||
vo: taskId=1001 items=1
|
||||
item: resultId=2001 taskId=1001 shopName=Demo Shop shopId=shop-Demo Shop platform=Amazon companyName=公司-Demo Shop matched=true matchStatus=MATCHED taskStatus=RUNNING success=- error=- countryCodes=[UK] countryResults=0 downloadUrl=-
|
||||
events=7 taskInserts=1 resultInserts=1 heartbeatTouches=1
|
||||
evt: task.insert: id=1001 taskNoPrefix=SHOP_DATA_CRAWL moduleType=SHOP_DATA_CRAWL taskMode=PYTHON_QUEUE status=RUNNING sourceFileCount=1 ok=0 fail=0 userId=7 createdBy=user:7 deviceId=dev-1 ownerInstanceId=- requestJsonPresent=false resultJsonPresent=false
|
||||
evt: taskCacheSave: id=1001
|
||||
evt: result.insert: id=2001 taskId=1001 moduleType=SHOP_DATA_CRAWL sourceFilename=Demo Shop sourceFileUrl=shop-Demo Shop userId=7 success=-1 rowCount=- createdAtEqualsTaskCreatedAt=true
|
||||
evt: snapshotTable.replace: taskId=1001 items=1
|
||||
evt: progressSave: taskId=1001 status=RUNNING total=1 ok=0 fail=0
|
||||
evt: task.update: id=1001 status=RUNNING ok=0 fail=0 requestJsonHasOwner=true requestJsonItems=1 resultJsonItems=1 createdAtEqualsUpdatedAt=true
|
||||
evt: taskCacheSave: id=1001
|
||||
== case=twoItemsRejected
|
||||
thrown: BusinessException code=- message=items 必须且只能包含一个店铺
|
||||
events=0 taskInserts=0 resultInserts=0 heartbeatTouches=0
|
||||
== case=emptyItemsRejected
|
||||
thrown: BusinessException code=- message=items 必须且只能包含一个店铺
|
||||
events=0 taskInserts=0 resultInserts=0 heartbeatTouches=0
|
||||
== case=nullItemsRejected
|
||||
thrown: BusinessException code=- message=items 必须且只能包含一个店铺
|
||||
events=0 taskInserts=0 resultInserts=0 heartbeatTouches=0
|
||||
== case=rawTwoItemsSameShopRejectedFirstGate
|
||||
thrown: BusinessException code=- message=items 必须且只能包含一个店铺
|
||||
events=0 taskInserts=0 resultInserts=0 heartbeatTouches=0
|
||||
== case=blankShopNameDropped
|
||||
thrown: BusinessException code=- message=items 必须且只能包含一个有效店铺
|
||||
events=0 taskInserts=0 resultInserts=0 heartbeatTouches=0
|
||||
== case=reNormalizedToBlankRejects
|
||||
thrown: BusinessException code=- message=任务数据已失效,请刷新后重试
|
||||
events=2 taskInserts=1 resultInserts=0 heartbeatTouches=1
|
||||
evt: task.insert: id=1001 taskNoPrefix=SHOP_DATA_CRAWL moduleType=SHOP_DATA_CRAWL taskMode=PYTHON_QUEUE status=RUNNING sourceFileCount=1 ok=0 fail=0 userId=7 createdBy=user:7 deviceId=dev-1 ownerInstanceId=- requestJsonPresent=false resultJsonPresent=false
|
||||
evt: taskCacheSave: id=1001
|
||||
== case=unmatchedShopRejected
|
||||
thrown: BusinessException code=- message=店铺未匹配
|
||||
events=0 taskInserts=0 resultInserts=0 heartbeatTouches=0
|
||||
== case=usIsNotSupportedCountry
|
||||
thrown: BusinessException code=- message=非法国家代码: US
|
||||
events=0 taskInserts=0 resultInserts=0 heartbeatTouches=0
|
||||
== case=duplicateCountryCodeRejected
|
||||
thrown: BusinessException code=- message=country_codes 存在重复: UK
|
||||
events=0 taskInserts=0 resultInserts=0 heartbeatTouches=0
|
||||
== case=emptyCountryCodesRejected
|
||||
thrown: BusinessException code=- message=country_codes 至少选择 1 个国家
|
||||
events=0 taskInserts=0 resultInserts=0 heartbeatTouches=0
|
||||
== case=blankCountryCodeRejected
|
||||
thrown: BusinessException code=- message=country_codes 含空项
|
||||
events=0 taskInserts=0 resultInserts=0 heartbeatTouches=0
|
||||
== case=nullUserIdRejected
|
||||
thrown: BusinessException code=- message=user_id 不合法
|
||||
events=0 taskInserts=0 resultInserts=0 heartbeatTouches=0
|
||||
== case=zeroUserIdRejected
|
||||
thrown: BusinessException code=- message=user_id 不合法
|
||||
events=0 taskInserts=0 resultInserts=0 heartbeatTouches=0
|
||||
@@ -0,0 +1,138 @@
|
||||
== case=happyLegacyDone
|
||||
taskCacheSave: id=1001
|
||||
mergedSave: taskId=1001 shopKey=Demo Shop shopName=Demo Shop error=- countries=1 rows=1 done=true chunk=null/null
|
||||
result.update: id=2001 success=1 error=- filename=- urlPresent=false size=null contentType=- rowCount=null
|
||||
mergedRemove: taskId=1001 shopKey=Demo Shop
|
||||
snapshotTable.replace: taskId=1001 items=1
|
||||
progressSave: taskId=1001 status=RUNNING total=1 ok=1 fail=0
|
||||
task.update: id=1001 status=RUNNING ok=null fail=null sourceFileCount=null error=- resultJsonItems=1 resultJsonHasRows=true finishedPresent=false
|
||||
taskCacheSave: id=1001
|
||||
taskCacheSave: id=1001
|
||||
task.update: id=1001 status=SUCCESS ok=1 fail=0 sourceFileCount=null error=- resultJsonItems=1 resultJsonHasRows=true finishedPresent=true
|
||||
result.batchUpdate: columns=resultFilename,resultFileUrl,resultFileSize,resultContentType,rowCount where=(id IN (#{ew.paramNameValuePairs.MPGENVAL6}) AND success = #{ew.paramNameValuePairs.MPGENVAL7}) params=MPGENVAL1="店铺数据抓取-Demo Shop-1001.xlsx",MPGENVAL2=null,MPGENVAL3=0,MPGENVAL4="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",MPGENVAL5=1
|
||||
enqueue: taskId=1001 resultId=2001 scopeKey=task:1001:owner:instance-a
|
||||
snapshotTable.replace: taskId=1001 items=1
|
||||
progressSave: taskId=1001 status=SUCCESS total=1 ok=1 fail=0
|
||||
task.update: id=1001 status=SUCCESS ok=1 fail=0 sourceFileCount=null error=- resultJsonItems=1 resultJsonHasRows=true finishedPresent=true
|
||||
taskCacheDelete: taskId=1001
|
||||
events=16 heartbeatTouches=1 lockCloses=2 chunks=0 scopes=0 rustfs=0
|
||||
== case=failedShopFinalizesTaskFailed
|
||||
taskCacheSave: id=1001
|
||||
result.update: id=2001 success=0 error=店铺打不开 filename=- urlPresent=false size=0 contentType=- rowCount=0
|
||||
mergedRemove: taskId=1001 shopKey=Demo Shop
|
||||
snapshotTable.replace: taskId=1001 items=1
|
||||
progressSave: taskId=1001 status=RUNNING total=1 ok=0 fail=1
|
||||
task.update: id=1001 status=RUNNING ok=null fail=null sourceFileCount=null error=- resultJsonItems=1 resultJsonHasRows=false finishedPresent=false
|
||||
taskCacheSave: id=1001
|
||||
taskCacheSave: id=1001
|
||||
snapshotTable.replace: taskId=1001 items=1
|
||||
progressSave: taskId=1001 status=FAILED total=1 ok=0 fail=1
|
||||
task.update: id=1001 status=FAILED ok=0 fail=1 sourceFileCount=null error=Demo Shop: 店铺打不开 resultJsonItems=1 resultJsonHasRows=false finishedPresent=true
|
||||
snapshotTable.replace: taskId=1001 items=1
|
||||
progressSave: taskId=1001 status=FAILED total=1 ok=0 fail=1
|
||||
task.update: id=1001 status=FAILED ok=0 fail=1 sourceFileCount=null error=Demo Shop: 店铺打不开 resultJsonItems=1 resultJsonHasRows=false finishedPresent=true
|
||||
taskCacheDelete: taskId=1001
|
||||
events=15 heartbeatTouches=1 lockCloses=2 chunks=0 scopes=0 rustfs=0
|
||||
== case=legacyWithoutDoneWaits
|
||||
taskCacheSave: id=1001
|
||||
mergedSave: taskId=1001 shopKey=Demo Shop shopName=Demo Shop error=- countries=1 rows=1 done=null chunk=null/null
|
||||
snapshotTable.replace: taskId=1001 items=1
|
||||
progressSave: taskId=1001 status=RUNNING total=1 ok=0 fail=0
|
||||
task.update: id=1001 status=RUNNING ok=null fail=null sourceFileCount=null error=- resultJsonItems=1 resultJsonHasRows=true finishedPresent=false
|
||||
taskCacheSave: id=1001
|
||||
taskCacheSave: id=1001
|
||||
task.update: id=1001 status=RUNNING ok=0 fail=0 sourceFileCount=null error=- resultJsonItems=1 resultJsonHasRows=true finishedPresent=false
|
||||
taskCacheSave: id=1001
|
||||
events=9 heartbeatTouches=1 lockCloses=2 chunks=0 scopes=0 rustfs=0
|
||||
== case=shopNameMismatchRejected
|
||||
thrown: BusinessException code=- message=回传店铺与任务不匹配
|
||||
== case=twoShopsRejectedBeforeLock
|
||||
thrown: BusinessException code=- message=shops 必须且只能包含一个店铺结果
|
||||
== case=nullRequestRejectedBeforeLock
|
||||
thrown: BusinessException code=- message=shops 必须且只能包含一个店铺结果
|
||||
== case=nullShopInListRejected
|
||||
thrown: BusinessException code=- message=shops 包含空结果
|
||||
== case=invalidTaskIdRejectedBeforeLock
|
||||
thrown: BusinessException code=- message=taskId 不合法
|
||||
events=0 heartbeatTouches=0 lockCloses=0 chunks=0 scopes=0 rustfs=0
|
||||
== case=terminalTaskRejectedWithoutCode
|
||||
thrown: BusinessException code=- message=任务已结束,拒绝重复提交
|
||||
== case=wrongModuleTaskNotFound
|
||||
thrown: BusinessException code=- message=任务不存在
|
||||
== case=ownershipCheckedBeforeTerminal
|
||||
thrown: TaskOwnerMismatchException code=40903 message=该任务已绑定到另一台服务实例处理,请通过原实例继续处理
|
||||
== case=twoResultRowsRejected
|
||||
thrown: BusinessException code=- message=任务店铺数据异常
|
||||
== case=partialChunkStored
|
||||
taskCacheSave: id=1001
|
||||
payloadStore: pointer=rustfs:payload-1
|
||||
chunkInsert: idx=1/2 hash=eafc7b7e1451 scopeKey=result-chunks:Demo Shop
|
||||
scopeInsert: received=1 total=2 completed=0 scopeKey=result-chunks:Demo Shop
|
||||
task.update: id=1001 status=RUNNING ok=null fail=null sourceFileCount=null error=- resultJsonItems=0 resultJsonHasRows=false finishedPresent=false
|
||||
taskCacheSave: id=1001
|
||||
taskCacheSave: id=1001
|
||||
task.update: id=1001 status=RUNNING ok=0 fail=0 sourceFileCount=null error=- resultJsonItems=0 resultJsonHasRows=false finishedPresent=false
|
||||
taskCacheSave: id=1001
|
||||
events=9 heartbeatTouches=1 lockCloses=2 chunks=1 scopes=1 rustfs=1
|
||||
== case=secondChunkArrivesFirst
|
||||
taskCacheSave: id=1001
|
||||
payloadStore: pointer=rustfs:payload-2
|
||||
chunkInsert: idx=1/2 hash=eafc7b7e1451 scopeKey=result-chunks:Demo Shop
|
||||
scopeUpdate: received=2 total=2 completed=1 scopeKey=result-chunks:Demo Shop
|
||||
payloadResolve: pointer=rustfs:payload-2
|
||||
payloadResolve: pointer=rustfs:payload-1
|
||||
result.update: id=2001 success=1 error=- filename=- urlPresent=false size=null contentType=- rowCount=null
|
||||
mergedRemove: taskId=1001 shopKey=Demo Shop
|
||||
snapshotTable.replace: taskId=1001 items=1
|
||||
progressSave: taskId=1001 status=RUNNING total=1 ok=1 fail=0
|
||||
task.update: id=1001 status=RUNNING ok=0 fail=0 sourceFileCount=null error=- resultJsonItems=1 resultJsonHasRows=true finishedPresent=false
|
||||
taskCacheSave: id=1001
|
||||
taskCacheSave: id=1001
|
||||
task.update: id=1001 status=SUCCESS ok=1 fail=0 sourceFileCount=null error=- resultJsonItems=1 resultJsonHasRows=true finishedPresent=true
|
||||
result.batchUpdate: columns=resultFilename,resultFileUrl,resultFileSize,resultContentType,rowCount where=(id IN (#{ew.paramNameValuePairs.MPGENVAL6}) AND success = #{ew.paramNameValuePairs.MPGENVAL7}) params=MPGENVAL1="店铺数据抓取-Demo Shop-1001.xlsx",MPGENVAL2=null,MPGENVAL3=0,MPGENVAL4="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",MPGENVAL5=1
|
||||
enqueue: taskId=1001 resultId=2001 scopeKey=task:1001:owner:instance-a
|
||||
snapshotTable.replace: taskId=1001 items=1
|
||||
progressSave: taskId=1001 status=SUCCESS total=1 ok=1 fail=0
|
||||
task.update: id=1001 status=SUCCESS ok=1 fail=0 sourceFileCount=null error=- resultJsonItems=1 resultJsonHasRows=true finishedPresent=true
|
||||
taskCacheDelete: taskId=1001
|
||||
events=20 heartbeatTouches=2 lockCloses=4 chunks=2 scopes=1 rustfs=2
|
||||
== case=identicalChunkRetryAbsorbed
|
||||
taskCacheSave: id=1001
|
||||
payloadStore: pointer=rustfs:payload-2
|
||||
payloadDelete: pointer=rustfs:payload-2
|
||||
scopeUpdate: received=1 total=2 completed=0 scopeKey=result-chunks:Demo Shop
|
||||
task.update: id=1001 status=RUNNING ok=0 fail=0 sourceFileCount=null error=- resultJsonItems=0 resultJsonHasRows=false finishedPresent=false
|
||||
taskCacheSave: id=1001
|
||||
taskCacheSave: id=1001
|
||||
task.update: id=1001 status=RUNNING ok=0 fail=0 sourceFileCount=null error=- resultJsonItems=0 resultJsonHasRows=false finishedPresent=false
|
||||
taskCacheSave: id=1001
|
||||
events=9 heartbeatTouches=2 lockCloses=4 chunks=1 scopes=1 rustfs=1
|
||||
== case=chunkRetryWithDifferentContentRejected
|
||||
thrown: BusinessException code=- message=同一 chunk_index 已回传不同内容,拒绝覆盖
|
||||
== case=chunkIndexWithoutTotalRejected
|
||||
thrown: BusinessException code=- message=chunk_index 与 chunk_total 必须同时传入
|
||||
== case=chunkTotalChangeRejected
|
||||
thrown: BusinessException code=- message=同一店铺的 chunk_total 必须保持一致
|
||||
== case=chunkRejectedWhenRustfsDisabled
|
||||
thrown: BusinessException code=- message=RustFS 未配置,店铺数据抓取结果分片暂不可接收
|
||||
== case=completedRowFlushesWorkbookColumns
|
||||
taskCacheSave: id=1001
|
||||
payloadStore: pointer=rustfs:payload-1
|
||||
chunkInsert: idx=1/1 hash=b6c082cd9d58 scopeKey=result-chunks:Demo Shop
|
||||
scopeInsert: received=1 total=1 completed=1 scopeKey=result-chunks:Demo Shop
|
||||
payloadResolve: pointer=rustfs:payload-1
|
||||
result.update: id=2001 success=1 error=- filename=- urlPresent=false size=null contentType=- rowCount=null
|
||||
mergedRemove: taskId=1001 shopKey=Demo Shop
|
||||
snapshotTable.replace: taskId=1001 items=1
|
||||
progressSave: taskId=1001 status=RUNNING total=1 ok=1 fail=0
|
||||
task.update: id=1001 status=RUNNING ok=null fail=null sourceFileCount=null error=- resultJsonItems=1 resultJsonHasRows=true finishedPresent=false
|
||||
taskCacheSave: id=1001
|
||||
taskCacheSave: id=1001
|
||||
task.update: id=1001 status=SUCCESS ok=1 fail=0 sourceFileCount=null error=- resultJsonItems=1 resultJsonHasRows=true finishedPresent=true
|
||||
result.batchUpdate: columns=resultFilename,resultFileUrl,resultFileSize,resultContentType,rowCount where=(id IN (#{ew.paramNameValuePairs.MPGENVAL6}) AND success = #{ew.paramNameValuePairs.MPGENVAL7}) params=MPGENVAL1="店铺数据抓取-Demo Shop-1001.xlsx",MPGENVAL2=null,MPGENVAL3=0,MPGENVAL4="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",MPGENVAL5=1
|
||||
enqueue: taskId=1001 resultId=2001 scopeKey=task:1001:owner:instance-a
|
||||
snapshotTable.replace: taskId=1001 items=1
|
||||
progressSave: taskId=1001 status=SUCCESS total=1 ok=1 fail=0
|
||||
task.update: id=1001 status=SUCCESS ok=1 fail=0 sourceFileCount=null error=- resultJsonItems=1 resultJsonHasRows=true finishedPresent=true
|
||||
taskCacheDelete: taskId=1001
|
||||
events=19 heartbeatTouches=1 lockCloses=2 chunks=1 scopes=1 rustfs=1
|
||||
Reference in New Issue
Block a user