test(phase-3): 3C 前置——补 collectdata 回传段 golden,钉住「两段准入」的现网步序
CollectDataSnapshotTest 只锁到建单+解析+载荷,submitResult 那一段按 §5「先补再动」
必须先在非源上落一笔。本笔吃真实入口 service.submitResult,golden 270 行/15 段 case,
渲染的是调用顺序流水而非分组清单。
这支与另五支最不一样的是**准入发生两次**:锁外预检(快速失败,不为终态任务白跑
去重与品牌检测)+锁内重读复核。于是流水把四个锁外副作用(filter/brandFilter/
asinCache.write/recheck.enqueue)也记下来,terminalRejectedBeforeLock 与
taskNotFoundBeforeLock 两格读出来是 lockAcquire=0 outsideLock=0 ——
「预检在取锁与全部预处理之前」这条形状从此改不动了。
钉住的其他现网形状:
- 重复分片:锁外预处理照跑、锁内 dup 命中后一条库都不写,vo.currentChunkRows=0
- 结果行按需新建(sourceFilename 空串那一形),scopeKey 用 submissionId 回落 "task:{id}"
- done=true 才挂最终组装;带 error 且已收到分片 ⇒ 标失败也照常组装(28599 那起的账)
- 空 ASIN 的行不进过滤链但仍进 chunk payload(结果文件 sheet 要全量)
- requireRustfsPayload 的形状:指针不是 rustfs: 前缀 ⇒ 先 deletePayloadIfPresent 再抛,
顺序反了就是留垃圾对象(chunkPointerNotRustfs 那格钉住)
- 锁竞争抛的是 code=40902 +「任务正在处理,请稍后再试」——与底座件的
「任务正在处理中,请稍后再试」差一个字,本笔只登记不统一
- 这支没有「taskId 不合法」那格:非法 id 一路走到「任务不存在」(invalidTaskId... 那格)
长内容只进长度与 sha256 前 12 位(resultJson/chunk payload),时钟与雪花 id 不进快照;
同一夹具跑两次+跨两个 JVM 复跑逐字节相同(golden 由写模式生成后,非写模式复跑一字未动)。
门禁:collectdata 全面+棘轮/边界/面数见证 定向 320 例/0 红,BUILD SUCCESS。
This commit is contained in:
+581
@@ -0,0 +1,581 @@
|
|||||||
|
package com.nanri.aiimage.modules.collectdata.service.support;
|
||||||
|
|
||||||
|
import cn.hutool.crypto.digest.DigestUtil;
|
||||||
|
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||||
|
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import com.nanri.aiimage.common.exception.BusinessException;
|
||||||
|
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||||
|
import com.nanri.aiimage.modules.asincache.service.AsinResultCacheService;
|
||||||
|
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataCountryPrefMapper;
|
||||||
|
import com.nanri.aiimage.modules.collectdata.mapper.CollectDataItemMapper;
|
||||||
|
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataSubmitResultRequest;
|
||||||
|
import com.nanri.aiimage.modules.collectdata.model.dto.CollectDataSubmitRowDto;
|
||||||
|
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataResultRowVo;
|
||||||
|
import com.nanri.aiimage.modules.collectdata.model.vo.CollectDataSubmitResultVo;
|
||||||
|
import com.nanri.aiimage.modules.collectdata.service.CollectDataBrandRecheckService;
|
||||||
|
import com.nanri.aiimage.modules.collectdata.service.CollectDataExcelAssemblyService;
|
||||||
|
import com.nanri.aiimage.modules.collectdata.service.CollectDataService;
|
||||||
|
import com.nanri.aiimage.modules.collectdata.util.CollectDataBatchQuery;
|
||||||
|
import com.nanri.aiimage.modules.collectdata.util.CollectDataBrandBatchFilter;
|
||||||
|
import com.nanri.aiimage.modules.collectdata.util.CollectDataInvalidAsinBatchWriter;
|
||||||
|
import com.nanri.aiimage.modules.collectdata.util.CollectDataResultDetailCodec;
|
||||||
|
import com.nanri.aiimage.modules.collectdata.util.CollectDataResultDetailReader;
|
||||||
|
import com.nanri.aiimage.modules.collectdata.util.CollectDataResultItemBatchWriter;
|
||||||
|
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||||
|
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||||
|
import com.nanri.aiimage.modules.invalidasin.mapper.InvalidAsinDataMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.TaskResultItemMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.service.StaleTaskHeartbeatPolicy;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskHeartbeatPositionService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||||
|
import com.nanri.aiimage.modules.notification.service.NotificationDispatchService;
|
||||||
|
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||||
|
import org.junit.jupiter.api.BeforeAll;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
|
||||||
|
import org.mockito.Answers;
|
||||||
|
import org.springframework.transaction.PlatformTransactionManager;
|
||||||
|
import org.springframework.transaction.support.TransactionTemplate;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
import static org.mockito.ArgumentMatchers.any;
|
||||||
|
import static org.mockito.ArgumentMatchers.anyInt;
|
||||||
|
import static org.mockito.ArgumentMatchers.anyList;
|
||||||
|
import static org.mockito.ArgumentMatchers.anyLong;
|
||||||
|
import static org.mockito.ArgumentMatchers.anyString;
|
||||||
|
import static org.mockito.Mockito.doAnswer;
|
||||||
|
import static org.mockito.Mockito.mock;
|
||||||
|
import static org.mockito.Mockito.when;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 采集数据回传落库(段 3)的现网行为快照——3C 余下那一半({@code collectdata})接底座的护身符。
|
||||||
|
*
|
||||||
|
* <p><b>为什么先补这一支。</b>{@code CollectDataSnapshotTest}(1B 落的)只锁到
|
||||||
|
* 「建单+解析+载荷」,蓝图 §5 那句「{@code submitResult} 出结果段没打快照,Phase 3 要动那一段
|
||||||
|
* 须先补再动」对这支同样成立。3C 要改的正是那一段里的准入两格(预检与锁内重读都是
|
||||||
|
* 「加载+模块校验」)、终态那格({@code IDEMPOTENT} 形)与组装 job 挂接({@code :1137})。
|
||||||
|
*
|
||||||
|
* <p><b>这支最特别的是准入发生了两次。</b>锁外预检(快速失败,不为终态任务白跑品牌检测)
|
||||||
|
* +锁内重读复核——两段都是「任务不存在/任务已结束」,但前者在<b>取锁与全部预处理之前</b>。
|
||||||
|
* 所以流水里把 {@code filter}/{@code brandFilter}/{@code asinCache}/{@code recheck} 这些
|
||||||
|
* 锁外副作用也记下来:预检被挪到锁内、或者预处理被挪到预检之前,这两格立刻红。
|
||||||
|
*
|
||||||
|
* <p><b>替身只实现这条链真吃到的部分</b>:任务行/结果行/分片行都是内存活实体(重读看得见上一次
|
||||||
|
* 写入),{@code TransientPayloadStorageService} 按「返回 rustfs: 指针」实现——因为
|
||||||
|
* {@code requireRustfsPayload} 判的就是指针前缀,返回别的会让整段走删除分支。
|
||||||
|
* 时钟、雪花 id、OSS 直链不进快照;长内容(chunk payload、resultJson)只进长度与 sha256 前 12 位。
|
||||||
|
*
|
||||||
|
* golden:src/test/resources/collectdata/golden/submit-result-snapshot.txt
|
||||||
|
* 重新生成:{@code SHUFU_SNAPSHOT_WRITE=true mvn -o test -Dtest=CollectDataSubmitResultSnapshotTest}
|
||||||
|
*/
|
||||||
|
class CollectDataSubmitResultSnapshotTest {
|
||||||
|
|
||||||
|
private static final File GOLDEN_SUBMIT =
|
||||||
|
new File("src/test/resources/collectdata/golden/submit-result-snapshot.txt");
|
||||||
|
private static final long TASK_ID = 1001L;
|
||||||
|
private static final long RESULT_ID = 2001L;
|
||||||
|
|
||||||
|
@BeforeAll
|
||||||
|
static void initLambdaCache() {
|
||||||
|
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
|
||||||
|
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
|
||||||
|
TableInfoHelper.initTableInfo(assistant, FileResultEntity.class);
|
||||||
|
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
|
||||||
|
TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- 现网副作用替身 ----
|
||||||
|
|
||||||
|
private static final class Fixture {
|
||||||
|
private final List<String> events = new ArrayList<>();
|
||||||
|
private final Map<String, TaskChunkEntity> chunks = new LinkedHashMap<>();
|
||||||
|
private FileTaskEntity task;
|
||||||
|
private FileResultEntity result;
|
||||||
|
private TaskScopeStateEntity scope;
|
||||||
|
private boolean rustfsEnabled = true;
|
||||||
|
private boolean rustfsPointer = true;
|
||||||
|
private boolean lockAvailable = true;
|
||||||
|
private long receivedChunkCount = 0L;
|
||||||
|
private CollectDataSubmitResultVo vo;
|
||||||
|
private CollectDataService service;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static Fixture running(String status) {
|
||||||
|
Fixture fixture = new Fixture();
|
||||||
|
fixture.task = task(status);
|
||||||
|
fixture.result = resultRow();
|
||||||
|
return fixture;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static FileTaskEntity task(String status) {
|
||||||
|
FileTaskEntity task = new FileTaskEntity();
|
||||||
|
task.setId(TASK_ID);
|
||||||
|
task.setModuleType("COLLECT_DATA");
|
||||||
|
task.setStatus(status);
|
||||||
|
task.setTaskMode("PYTHON_QUEUE");
|
||||||
|
task.setUserId(7L);
|
||||||
|
task.setSourceFileCount(1);
|
||||||
|
task.setResultJson("{\"totalRows\":0,\"receivedRows\":0,\"finalRowCount\":0}");
|
||||||
|
return task;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static FileResultEntity resultRow() {
|
||||||
|
FileResultEntity row = new FileResultEntity();
|
||||||
|
row.setId(RESULT_ID);
|
||||||
|
row.setTaskId(TASK_ID);
|
||||||
|
row.setModuleType("COLLECT_DATA");
|
||||||
|
row.setSourceFilename("");
|
||||||
|
row.setRowCount(0);
|
||||||
|
row.setSuccess(0);
|
||||||
|
row.setUserId(7L);
|
||||||
|
return row;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static CollectDataSubmitRowDto rawRow(String asin, String keyword) {
|
||||||
|
CollectDataSubmitRowDto dto = new CollectDataSubmitRowDto();
|
||||||
|
dto.setAsin(asin);
|
||||||
|
dto.setBrand(asin.isBlank() ? "" : "brand-" + asin);
|
||||||
|
dto.setKeyword(keyword);
|
||||||
|
dto.setPrice("19.90");
|
||||||
|
dto.setSellerName("seller-A");
|
||||||
|
dto.setDeliveryMethod("FBA");
|
||||||
|
dto.setPage(1);
|
||||||
|
return dto;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static CollectDataSubmitResultRequest request(Integer chunkIndex, Integer chunkTotal,
|
||||||
|
Boolean done, String error,
|
||||||
|
CollectDataSubmitRowDto... rows) {
|
||||||
|
CollectDataSubmitResultRequest request = new CollectDataSubmitResultRequest();
|
||||||
|
request.setSubmissionId("sub-1");
|
||||||
|
request.setChunkIndex(chunkIndex);
|
||||||
|
request.setChunkTotal(chunkTotal);
|
||||||
|
request.setDone(done);
|
||||||
|
request.setError(error);
|
||||||
|
request.setItems(new ArrayList<>(List.of(rows)));
|
||||||
|
return request;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static CollectDataResultRowVo vo(String asin, String keyword) {
|
||||||
|
CollectDataResultRowVo row = new CollectDataResultRowVo();
|
||||||
|
row.setAsin(asin);
|
||||||
|
row.setKeyword(keyword);
|
||||||
|
row.setBrand(asin.isBlank() ? "" : "brand-" + asin);
|
||||||
|
row.setPrice("19.90");
|
||||||
|
row.setSellerName("seller-A");
|
||||||
|
row.setDeliveryMethod("FBA");
|
||||||
|
row.setPage(1);
|
||||||
|
return row;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static CollectDataService newService(Fixture f) {
|
||||||
|
FileTaskMapper fileTaskMapper = mock(FileTaskMapper.class);
|
||||||
|
FileResultMapper fileResultMapper = mock(FileResultMapper.class);
|
||||||
|
TaskChunkMapper taskChunkMapper = mock(TaskChunkMapper.class);
|
||||||
|
TaskScopeStateMapper taskScopeStateMapper = mock(TaskScopeStateMapper.class);
|
||||||
|
TaskDistributedLockService lockService = mock(TaskDistributedLockService.class);
|
||||||
|
TaskFileJobService taskFileJobService = mock(TaskFileJobService.class);
|
||||||
|
TransientPayloadStorageService payloadStorage = mock(TransientPayloadStorageService.class);
|
||||||
|
CollectDataBatchQuery batchQuery = mock(CollectDataBatchQuery.class);
|
||||||
|
CollectDataBrandBatchFilter brandFilter = mock(CollectDataBrandBatchFilter.class);
|
||||||
|
CollectDataInvalidAsinBatchWriter invalidAsinBatchWriter = mock(CollectDataInvalidAsinBatchWriter.class);
|
||||||
|
CollectDataResultItemBatchWriter resultItemBatchWriter = mock(CollectDataResultItemBatchWriter.class);
|
||||||
|
AsinResultCacheService asinCache = mock(AsinResultCacheService.class);
|
||||||
|
CollectDataBrandRecheckService recheck = mock(CollectDataBrandRecheckService.class);
|
||||||
|
ObjectMapper objectMapper = new ObjectMapper();
|
||||||
|
|
||||||
|
when(payloadStorage.isSharedWriteEnabled()).thenAnswer(invocation -> {
|
||||||
|
f.events.add("storage.ensure sharedWrite=" + f.rustfsEnabled);
|
||||||
|
return f.rustfsEnabled;
|
||||||
|
});
|
||||||
|
when(payloadStorage.extractPointer(any())).thenAnswer(invocation -> invocation.getArgument(0));
|
||||||
|
when(payloadStorage.storeResultPayload(anyString(), anyLong(), anyString(), anyString(), anyString()))
|
||||||
|
.thenAnswer(invocation -> {
|
||||||
|
String json = invocation.getArgument(4);
|
||||||
|
f.events.add("payload.storeResult key=" + invocation.getArgument(3)
|
||||||
|
+ " len=" + json.length() + " hash=" + hash(json));
|
||||||
|
return "rustfs:result-" + invocation.getArgument(3);
|
||||||
|
});
|
||||||
|
when(payloadStorage.storeChunkPayloadVersioned(anyString(), anyLong(), anyString(), anyInt(), anyString()))
|
||||||
|
.thenAnswer(invocation -> {
|
||||||
|
String json = invocation.getArgument(4);
|
||||||
|
f.events.add("payload.storeChunk idx=" + invocation.getArgument(3)
|
||||||
|
+ " len=" + json.length() + " hash=" + hash(json));
|
||||||
|
return (f.rustfsPointer ? "rustfs:chunk-" : "local:chunk-") + invocation.getArgument(3);
|
||||||
|
});
|
||||||
|
doAnswer(invocation -> {
|
||||||
|
f.events.add("payload.delete pointer=" + invocation.getArgument(0));
|
||||||
|
return null;
|
||||||
|
}).when(payloadStorage).deletePayloadIfPresent(anyString());
|
||||||
|
|
||||||
|
when(fileTaskMapper.selectById(any())).thenAnswer(invocation -> {
|
||||||
|
f.events.add("task.load id=" + invocation.getArgument(0) + " status="
|
||||||
|
+ dash(f.task == null ? null : f.task.getStatus()) + " module="
|
||||||
|
+ dash(f.task == null ? null : f.task.getModuleType()));
|
||||||
|
return f.task;
|
||||||
|
});
|
||||||
|
doAnswer(invocation -> {
|
||||||
|
FileTaskEntity task = invocation.getArgument(0);
|
||||||
|
String resultJson = task.getResultJson() == null ? "" : task.getResultJson();
|
||||||
|
f.events.add("task.update id=" + task.getId()
|
||||||
|
+ " status=" + dash(task.getStatus())
|
||||||
|
+ " errorMessage=" + dash(task.getErrorMessage())
|
||||||
|
+ " successFileCount=" + dash(task.getSuccessFileCount())
|
||||||
|
+ " failedFileCount=" + dash(task.getFailedFileCount())
|
||||||
|
+ " finishedAt=" + (task.getFinishedAt() == null ? "-" : "set")
|
||||||
|
+ " resultJson.len=" + resultJson.length() + " resultJson.hash=" + hash(resultJson));
|
||||||
|
return 1;
|
||||||
|
}).when(fileTaskMapper).updateById(any(FileTaskEntity.class));
|
||||||
|
|
||||||
|
when(fileResultMapper.selectOne(any())).thenAnswer(invocation -> {
|
||||||
|
f.events.add("result.selectOne -> id=" + dash(f.result == null ? null : f.result.getId()));
|
||||||
|
return f.result;
|
||||||
|
});
|
||||||
|
doAnswer(invocation -> {
|
||||||
|
FileResultEntity row = invocation.getArgument(0);
|
||||||
|
row.setId(RESULT_ID);
|
||||||
|
f.result = row;
|
||||||
|
f.events.add("result.insert taskId=" + row.getTaskId() + " sourceFilename="
|
||||||
|
+ dash(row.getSourceFilename()) + " rowCount=" + dash(row.getRowCount())
|
||||||
|
+ " success=" + dash(row.getSuccess()));
|
||||||
|
return 1;
|
||||||
|
}).when(fileResultMapper).insert(any(FileResultEntity.class));
|
||||||
|
doAnswer(invocation -> {
|
||||||
|
FileResultEntity row = invocation.getArgument(0);
|
||||||
|
f.events.add("result.update id=" + row.getId()
|
||||||
|
+ " success=" + dash(row.getSuccess())
|
||||||
|
+ " rowCount=" + dash(row.getRowCount())
|
||||||
|
+ " resultFilename=" + dash(row.getResultFilename())
|
||||||
|
+ " resultFileUrl=" + dash(row.getResultFileUrl())
|
||||||
|
+ " fileSize=" + dash(row.getResultFileSize())
|
||||||
|
+ " errorMessage=" + dash(row.getErrorMessage()));
|
||||||
|
return 1;
|
||||||
|
}).when(fileResultMapper).updateById(any(FileResultEntity.class));
|
||||||
|
|
||||||
|
when(taskChunkMapper.selectOne(any())).thenAnswer(invocation -> {
|
||||||
|
TaskChunkEntity existing = f.chunks.isEmpty() ? null : f.chunks.values().iterator().next();
|
||||||
|
f.events.add("chunk.selectOne -> " + (existing == null ? "none" : "idx=" + existing.getChunkIndex()));
|
||||||
|
return existing;
|
||||||
|
});
|
||||||
|
when(taskChunkMapper.selectCount(any())).thenAnswer(invocation -> {
|
||||||
|
f.events.add("chunk.count -> " + f.receivedChunkCount);
|
||||||
|
return f.receivedChunkCount;
|
||||||
|
});
|
||||||
|
doAnswer(invocation -> {
|
||||||
|
TaskChunkEntity chunk = invocation.getArgument(0);
|
||||||
|
f.chunks.put(String.valueOf(chunk.getChunkIndex()), chunk);
|
||||||
|
f.events.add("chunk.insert scope=" + dash(chunk.getScopeKey())
|
||||||
|
+ " idx=" + chunk.getChunkIndex() + "/" + chunk.getChunkTotal()
|
||||||
|
+ " payload=" + dash(chunk.getPayloadJson())
|
||||||
|
+ " payloadHash=" + dash(chunk.getPayloadHash()));
|
||||||
|
return 1;
|
||||||
|
}).when(taskChunkMapper).insert(any(TaskChunkEntity.class));
|
||||||
|
|
||||||
|
when(taskScopeStateMapper.selectOne(any())).thenAnswer(invocation -> {
|
||||||
|
f.events.add("scope.selectOne -> " + (f.scope == null ? "none" : "id=" + f.scope.getId()));
|
||||||
|
return f.scope;
|
||||||
|
});
|
||||||
|
doAnswer(invocation -> {
|
||||||
|
TaskScopeStateEntity scope = invocation.getArgument(0);
|
||||||
|
scope.setId(4001L);
|
||||||
|
f.scope = scope;
|
||||||
|
f.events.add("scope.insert chunkTotal=" + scope.getChunkTotal()
|
||||||
|
+ " received=" + scope.getReceivedChunkCount()
|
||||||
|
+ " completed=" + scope.getCompleted()
|
||||||
|
+ " lastError=" + dash(scope.getLastError())
|
||||||
|
+ " stateJson=" + dash(scope.getStateJson()));
|
||||||
|
return 1;
|
||||||
|
}).when(taskScopeStateMapper).insert(any(TaskScopeStateEntity.class));
|
||||||
|
doAnswer(invocation -> {
|
||||||
|
TaskScopeStateEntity scope = invocation.getArgument(0);
|
||||||
|
f.events.add("scope.update chunkTotal=" + scope.getChunkTotal()
|
||||||
|
+ " received=" + scope.getReceivedChunkCount()
|
||||||
|
+ " completed=" + scope.getCompleted());
|
||||||
|
return 1;
|
||||||
|
}).when(taskScopeStateMapper).updateById(any(TaskScopeStateEntity.class));
|
||||||
|
|
||||||
|
when(lockService.acquire(anyString(), anyLong(), anyLong())).thenAnswer(invocation -> {
|
||||||
|
long waitMillis = invocation.getArgument(2);
|
||||||
|
if (!f.lockAvailable) {
|
||||||
|
f.events.add("lock.acquire taskId=" + invocation.getArgument(1)
|
||||||
|
+ " waitMillis=" + waitMillis + " -> busy");
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
f.events.add("lock.acquire taskId=" + invocation.getArgument(1) + " waitMillis=" + waitMillis);
|
||||||
|
return mock(TaskDistributedLockService.LockHandle.class);
|
||||||
|
});
|
||||||
|
|
||||||
|
when(batchQuery.filter(anyList(), any())).thenAnswer(invocation -> {
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
List<CollectDataResultRowVo> rows = invocation.getArgument(0);
|
||||||
|
List<CollectDataResultRowVo> kept = new ArrayList<>(rows);
|
||||||
|
f.events.add("outsideLock filter rows=" + kept.size() + " owner=" + invocation.getArgument(1)
|
||||||
|
+ " dedupe=0 invalid=0");
|
||||||
|
return new CollectDataBatchQuery.FilterResult(kept, 0, 0);
|
||||||
|
});
|
||||||
|
when(brandFilter.filter(anyList())).thenAnswer(invocation -> {
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
List<CollectDataResultRowVo> rows = invocation.getArgument(0);
|
||||||
|
f.events.add("outsideLock brandFilter accepted=" + rows.size() + " rejected=0 queryFailed=0");
|
||||||
|
return new CollectDataBrandBatchFilter.BrandBatchOutcome(List.of(), List.of(), rows);
|
||||||
|
});
|
||||||
|
doAnswer(invocation -> {
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
List<CollectDataResultRowVo> rows = invocation.getArgument(2);
|
||||||
|
f.events.add("outsideLock asinCache.write country=" + dash(invocation.getArgument(0))
|
||||||
|
+ " taskId=" + invocation.getArgument(1) + " rows=" + rows.size());
|
||||||
|
return null;
|
||||||
|
}).when(asinCache).writeCollectDataRows(any(), anyString(), anyList());
|
||||||
|
doAnswer(invocation -> {
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
List<CollectDataResultRowVo> failed = invocation.getArgument(1);
|
||||||
|
f.events.add("outsideLock recheck.enqueue n=" + failed.size());
|
||||||
|
return null;
|
||||||
|
}).when(recheck).enqueue(any(FileTaskEntity.class), anyList());
|
||||||
|
doAnswer(invocation -> {
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
List<CollectDataResultRowVo> rejected = invocation.getArgument(0);
|
||||||
|
f.events.add("invalidAsin.writeBatch n=" + rejected.size() + " owner=" + invocation.getArgument(1));
|
||||||
|
return null;
|
||||||
|
}).when(invalidAsinBatchWriter).writeBatch(anyList(), any());
|
||||||
|
|
||||||
|
when(resultItemBatchWriter.upsertAccepted(anyLong(), any(), anyString(), anyInt(), anyList(), anyString()))
|
||||||
|
.thenAnswer(invocation -> {
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
List<CollectDataResultRowVo> accepted = invocation.getArgument(4);
|
||||||
|
f.events.add("resultItem.upsert chunk=" + invocation.getArgument(3)
|
||||||
|
+ " accepted=" + accepted.size() + " ref=" + invocation.getArgument(5));
|
||||||
|
return new CollectDataResultItemBatchWriter.UpsertCounts(accepted.size(), 0, accepted.size());
|
||||||
|
});
|
||||||
|
|
||||||
|
f.service = new CollectDataService(mock(LocalFileStorageService.class), fileTaskMapper, fileResultMapper,
|
||||||
|
mock(CollectDataItemMapper.class), mock(CollectDataCountryPrefMapper.class),
|
||||||
|
mock(InvalidAsinDataMapper.class), taskChunkMapper, taskScopeStateMapper,
|
||||||
|
mock(TaskResultItemMapper.class), lockService, taskFileJobService,
|
||||||
|
mock(TaskHeartbeatPositionService.class), payloadStorage,
|
||||||
|
mock(CollectDataExcelAssemblyService.class), mock(OssStorageService.class), objectMapper,
|
||||||
|
new TransactionTemplate(mock(PlatformTransactionManager.class)),
|
||||||
|
mock(DistributedJobLockService.class, Answers.RETURNS_MOCKS), batchQuery, brandFilter,
|
||||||
|
invalidAsinBatchWriter, new CollectDataResultDetailCodec(objectMapper),
|
||||||
|
resultItemBatchWriter, mock(CollectDataResultDetailReader.class),
|
||||||
|
mock(TaskProgressLightAssembler.class), mock(NotificationDispatchService.class),
|
||||||
|
asinCache, recheck, mock(StaleTaskHeartbeatPolicy.class));
|
||||||
|
return f.service;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- 渲染 ----
|
||||||
|
|
||||||
|
private static String dash(Object value) {
|
||||||
|
String text = value == null ? "" : String.valueOf(value);
|
||||||
|
return text.isBlank() ? "-" : text.replace('\n', ' ');
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String hash(String value) {
|
||||||
|
return value.isEmpty() ? "-" : DigestUtil.sha256Hex(value).substring(0, 12);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String counters(Fixture f) {
|
||||||
|
return "counters lockAcquire=" + count(f, "lock.acquire")
|
||||||
|
+ " outsideLock=" + count(f, "outsideLock")
|
||||||
|
+ " taskUpdate=" + count(f, "task.update")
|
||||||
|
+ " resultUpdate=" + count(f, "result.update")
|
||||||
|
+ " resultInsert=" + count(f, "result.insert")
|
||||||
|
+ " chunkInsert=" + count(f, "chunk.insert")
|
||||||
|
+ " scopeWrite=" + (count(f, "scope.insert") + count(f, "scope.update"))
|
||||||
|
+ " jobEnqueue=" + count(f, "job.enqueue")
|
||||||
|
+ '\n';
|
||||||
|
}
|
||||||
|
|
||||||
|
private static long count(Fixture f, String prefix) {
|
||||||
|
return f.events.stream().filter(event -> event.startsWith(prefix)).count();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String submit(Fixture f, Long taskId, CollectDataSubmitResultRequest request) {
|
||||||
|
String outcome;
|
||||||
|
try {
|
||||||
|
f.vo = f.service.submitResult(taskId, request);
|
||||||
|
outcome = "thrown: none\n";
|
||||||
|
} catch (BusinessException ex) {
|
||||||
|
outcome = "thrown: BusinessException code=" + dash(ex.getCode())
|
||||||
|
+ " message=" + dash(ex.getMessage()) + '\n';
|
||||||
|
} catch (Exception ex) {
|
||||||
|
outcome = "thrown: " + ex.getClass().getSimpleName()
|
||||||
|
+ " message=" + dash(ex.getMessage()) + '\n';
|
||||||
|
}
|
||||||
|
StringBuilder sb = new StringBuilder();
|
||||||
|
for (String event : f.events) {
|
||||||
|
sb.append(event).append('\n');
|
||||||
|
}
|
||||||
|
sb.append(outcome);
|
||||||
|
if (f.vo != null) {
|
||||||
|
sb.append("vo: taskId=").append(dash(f.vo.getTaskId()))
|
||||||
|
.append(" resultId=").append(dash(f.vo.getResultId()))
|
||||||
|
.append(" chunk=").append(dash(f.vo.getChunkIndex())).append('/').append(dash(f.vo.getChunkTotal()))
|
||||||
|
.append(" done=").append(dash(f.vo.getDone()))
|
||||||
|
.append(" receivedRows=").append(dash(f.vo.getReceivedRows()))
|
||||||
|
.append(" currentChunkRows=").append(dash(f.vo.getCurrentChunkRows()))
|
||||||
|
.append(" dedupeFiltered=").append(dash(f.vo.getDedupeFilteredCount()))
|
||||||
|
.append(" invalidFiltered=").append(dash(f.vo.getInvalidFilteredCount()))
|
||||||
|
.append(" brandRejected=").append(dash(f.vo.getBrandRejectedCount()))
|
||||||
|
.append(" brandQueryFailed=").append(dash(f.vo.getBrandQueryFailedCount()))
|
||||||
|
.append(" finalRows=").append(dash(f.vo.getFinalRowCount()))
|
||||||
|
.append(" taskStatus=").append(dash(f.vo.getTaskStatus())).append('\n');
|
||||||
|
}
|
||||||
|
return sb.append(counters(f)).toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void appendCase(StringBuilder sb, String name, Fixture fixture,
|
||||||
|
Long taskId, CollectDataSubmitResultRequest request) {
|
||||||
|
sb.append("== case=").append(name).append('\n');
|
||||||
|
newService(fixture);
|
||||||
|
sb.append(submit(fixture, taskId, request));
|
||||||
|
sb.append('\n');
|
||||||
|
}
|
||||||
|
|
||||||
|
static String runAll() {
|
||||||
|
StringBuilder sb = new StringBuilder();
|
||||||
|
|
||||||
|
// 1. 常态:非末片,锁外预处理+锁内落分片与作用域状态,任务表只刷 updatedAt
|
||||||
|
appendCase(sb, "middleChunk", running("RUNNING"), TASK_ID,
|
||||||
|
request(1, 3, false, null, rawRow("B011", "kw-a"), rawRow("B012", "kw-a")));
|
||||||
|
|
||||||
|
// 2. 末片 done=true:挂最终组装(结果行+任务表+一条 job,scopeKey 是 task:{id} 那一形)
|
||||||
|
appendCase(sb, "doneChunkEnqueuesWorkbook", running("RUNNING"), TASK_ID,
|
||||||
|
request(3, 3, true, null, rawRow("B013", "kw-b")));
|
||||||
|
|
||||||
|
// 3. 重复分片:锁外预处理照跑(现网如此),锁内 dup 命中 ⇒ 一条库都不写、currentChunkRows=0
|
||||||
|
Fixture duplicate = running("RUNNING");
|
||||||
|
TaskChunkEntity existing = new TaskChunkEntity();
|
||||||
|
existing.setChunkIndex(1);
|
||||||
|
existing.setChunkTotal(3);
|
||||||
|
duplicate.chunks.put("1", existing);
|
||||||
|
appendCase(sb, "duplicateChunkIgnored", duplicate, TASK_ID,
|
||||||
|
request(1, 3, false, null, rawRow("B014", "kw-c")));
|
||||||
|
|
||||||
|
// 4. 没有结果行:ensureTaskResult 先插一行再走后续(结果行的 sourceFilename 是空串那一形)
|
||||||
|
Fixture noRow = running("RUNNING");
|
||||||
|
noRow.result = null;
|
||||||
|
appendCase(sb, "resultRowCreatedOnDemand", noRow, TASK_ID, request(1, 1, true, null, rawRow("B015", "kw-d")));
|
||||||
|
|
||||||
|
// 5. 带 error:标失败并照常组装(已收到分片 ⇒ 用户要拿得到部分结果)
|
||||||
|
Fixture failedWithChunks = running("RUNNING");
|
||||||
|
failedWithChunks.receivedChunkCount = 2L;
|
||||||
|
appendCase(sb, "errorWithReceivedChunks", failedWithChunks, TASK_ID,
|
||||||
|
request(2, 3, false, "Python 采集异常", rawRow("B016", "kw-e")));
|
||||||
|
|
||||||
|
// 6. 带 error 且一个分片都没收到:只标失败,不组装
|
||||||
|
appendCase(sb, "errorWithoutChunks", running("RUNNING"), TASK_ID,
|
||||||
|
request(1, 2, false, "Python 采集异常", rawRow("B017", "kw-f")));
|
||||||
|
|
||||||
|
// 7. PENDING 首片:锁内先把任务复位成 RUNNING
|
||||||
|
appendCase(sb, "pendingTaskResetToRunning", running("PENDING"), TASK_ID,
|
||||||
|
request(1, 2, false, null, rawRow("B018", "kw-g")));
|
||||||
|
|
||||||
|
// 8. 空 ASIN 的行不进过滤链但仍进 chunk payload(结果文件 sheet 要全量)
|
||||||
|
appendCase(sb, "blankAsinRowKeptOutOfFilter", running("RUNNING"), TASK_ID,
|
||||||
|
request(1, 1, false, null, rawRow("", "kw-h")));
|
||||||
|
|
||||||
|
// 9. 终态任务重复回传:预检就拒,且发生在取锁与全部锁外预处理之前
|
||||||
|
Fixture terminal = running("SUCCESS");
|
||||||
|
appendCase(sb, "terminalRejectedBeforeLock", terminal, TASK_ID,
|
||||||
|
request(1, 2, false, null, rawRow("B019", "kw-i")));
|
||||||
|
|
||||||
|
// 10. 任务不存在/模块不符:同一句「任务不存在」,零锁零预处理
|
||||||
|
Fixture missingTask = running("RUNNING");
|
||||||
|
missingTask.task = null;
|
||||||
|
appendCase(sb, "taskNotFoundBeforeLock", missingTask, TASK_ID,
|
||||||
|
request(1, 2, false, null, rawRow("B020", "kw-j")));
|
||||||
|
Fixture wrongModule = running("RUNNING");
|
||||||
|
wrongModule.task = task("RUNNING");
|
||||||
|
wrongModule.task.setModuleType("APPEARANCE_PATENT");
|
||||||
|
appendCase(sb, "wrongModuleBeforeLock", wrongModule, TASK_ID,
|
||||||
|
request(1, 2, false, null, rawRow("B021", "kw-k")));
|
||||||
|
|
||||||
|
// 11. 请求体为空/RustFS 未配置:连预检都不做
|
||||||
|
appendCase(sb, "nullRequest", running("RUNNING"), TASK_ID, null);
|
||||||
|
Fixture rustfsOff = running("RUNNING");
|
||||||
|
rustfsOff.rustfsEnabled = false;
|
||||||
|
appendCase(sb, "rustfsDisabled", rustfsOff, TASK_ID, request(1, 1, false, null, rawRow("B022", "kw-l")));
|
||||||
|
|
||||||
|
// 12. 分片指针没落进 RustFS:先删已写的成品再抛(顺序反了就是留垃圾对象)
|
||||||
|
Fixture badPointer = running("RUNNING");
|
||||||
|
badPointer.rustfsPointer = false;
|
||||||
|
appendCase(sb, "chunkPointerNotRustfs", badPointer, TASK_ID,
|
||||||
|
request(1, 1, false, null, rawRow("B023", "kw-m")));
|
||||||
|
|
||||||
|
// 13. 锁竞争:40902 那一形(文案与底座件不同,见蓝图 3C 实录)
|
||||||
|
Fixture busy = running("RUNNING");
|
||||||
|
busy.lockAvailable = false;
|
||||||
|
appendCase(sb, "lockBusyRejected", busy, TASK_ID, request(1, 2, false, null, rawRow("B024", "kw-n")));
|
||||||
|
|
||||||
|
// 14. taskId 非法:这支没做「taskId 不合法」那格校验——非法 id 走的是「任务不存在」
|
||||||
|
appendCase(sb, "invalidTaskIdFallsThroughToNotFound", running("RUNNING"), 0L,
|
||||||
|
request(1, 2, false, null, rawRow("B025", "kw-o")));
|
||||||
|
|
||||||
|
return sb.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- 用例 ----
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void submit_result_snapshot_matches_golden() throws Exception {
|
||||||
|
assertEquals(read(GOLDEN_SUBMIT), runAll(), "回传段落库流水与 golden 一致");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void submit_result_snapshot_is_reproducible() {
|
||||||
|
assertEquals(runAll(), runAll(), "同一夹具跑两次流水一致(时钟与雪花 id 不得进快照)");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void submit_result_snapshot_golden_is_committed() {
|
||||||
|
assertTrue(GOLDEN_SUBMIT.isFile(), "golden 文件必须存在并入库: " + GOLDEN_SUBMIT.getAbsolutePath());
|
||||||
|
assertTrue(GOLDEN_SUBMIT.length() > 0, "golden 文件非空");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void submit_result_snapshot_tamper_is_detected() throws Exception {
|
||||||
|
String original = read(GOLDEN_SUBMIT);
|
||||||
|
assertTrue(original.contains("== case=middleChunk"), "golden 内容合法");
|
||||||
|
try {
|
||||||
|
Files.writeString(GOLDEN_SUBMIT.toPath(), original + "\n# tampered", StandardCharsets.UTF_8);
|
||||||
|
assertTrue(!read(GOLDEN_SUBMIT).equals(runAll()), "篡改 golden 后断言应失败");
|
||||||
|
} finally {
|
||||||
|
Files.writeString(GOLDEN_SUBMIT.toPath(), original, StandardCharsets.UTF_8);
|
||||||
|
}
|
||||||
|
assertEquals(original, read(GOLDEN_SUBMIT), "恢复原始 golden");
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 只在显式点名时写 golden:默认跑测试永远不许改是非源。 */
|
||||||
|
@Test
|
||||||
|
@EnabledIfEnvironmentVariable(named = "SHUFU_SNAPSHOT_WRITE", matches = "true")
|
||||||
|
void write_golden_on_demand() throws Exception {
|
||||||
|
Files.createDirectories(GOLDEN_SUBMIT.getParentFile().toPath());
|
||||||
|
Files.writeString(GOLDEN_SUBMIT.toPath(), runAll(), StandardCharsets.UTF_8);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String read(File file) throws Exception {
|
||||||
|
return Files.readString(file.toPath(), StandardCharsets.UTF_8).replace("\r\n", "\n");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,270 @@
|
|||||||
|
== case=middleChunk
|
||||||
|
storage.ensure sharedWrite=true
|
||||||
|
task.load id=1001 status=RUNNING module=COLLECT_DATA
|
||||||
|
outsideLock filter rows=2 owner=7 dedupe=0 invalid=0
|
||||||
|
outsideLock brandFilter accepted=2 rejected=0 queryFailed=0
|
||||||
|
outsideLock asinCache.write country=- taskId=1001 rows=2
|
||||||
|
outsideLock recheck.enqueue n=0
|
||||||
|
lock.acquire taskId=1001 waitMillis=5000
|
||||||
|
task.load id=1001 status=RUNNING module=COLLECT_DATA
|
||||||
|
result.selectOne -> id=2001
|
||||||
|
chunk.selectOne -> none
|
||||||
|
payload.storeResult key=chunk-1 len=253 hash=8640b8fb328e
|
||||||
|
resultItem.upsert chunk=1 accepted=2 ref=rustfs:result-chunk-1
|
||||||
|
payload.storeChunk idx=1 len=253 hash=8640b8fb328e
|
||||||
|
chunk.insert scope=sub-1 idx=1/3 payload=rustfs:chunk-1 payloadHash=8640b8fb328efe229676242654815f1ff673b047ff74be70619e2ef8e5a74783
|
||||||
|
scope.selectOne -> none
|
||||||
|
chunk.count -> 0
|
||||||
|
scope.insert chunkTotal=3 received=0 completed=0 lastError=- stateJson={"phase":"RECEIVED"}
|
||||||
|
task.update id=1001 status=RUNNING errorMessage=- successFileCount=- failedFileCount=- finishedAt=- resultJson.len=333 resultJson.hash=f635be79fec3
|
||||||
|
task.load id=1001 status=RUNNING module=COLLECT_DATA
|
||||||
|
thrown: none
|
||||||
|
vo: taskId=1001 resultId=2001 chunk=1/3 done=false receivedRows=2 currentChunkRows=2 dedupeFiltered=0 invalidFiltered=0 brandRejected=0 brandQueryFailed=0 finalRows=2 taskStatus=RUNNING
|
||||||
|
counters lockAcquire=1 outsideLock=4 taskUpdate=1 resultUpdate=0 resultInsert=0 chunkInsert=1 scopeWrite=1 jobEnqueue=0
|
||||||
|
|
||||||
|
== case=doneChunkEnqueuesWorkbook
|
||||||
|
storage.ensure sharedWrite=true
|
||||||
|
task.load id=1001 status=RUNNING module=COLLECT_DATA
|
||||||
|
outsideLock filter rows=1 owner=7 dedupe=0 invalid=0
|
||||||
|
outsideLock brandFilter accepted=1 rejected=0 queryFailed=0
|
||||||
|
outsideLock asinCache.write country=- taskId=1001 rows=1
|
||||||
|
outsideLock recheck.enqueue n=0
|
||||||
|
lock.acquire taskId=1001 waitMillis=5000
|
||||||
|
task.load id=1001 status=RUNNING module=COLLECT_DATA
|
||||||
|
result.selectOne -> id=2001
|
||||||
|
chunk.selectOne -> none
|
||||||
|
payload.storeResult key=chunk-3 len=127 hash=c3d3dc56797d
|
||||||
|
resultItem.upsert chunk=3 accepted=1 ref=rustfs:result-chunk-3
|
||||||
|
payload.storeChunk idx=3 len=127 hash=c3d3dc56797d
|
||||||
|
chunk.insert scope=sub-1 idx=3/3 payload=rustfs:chunk-3 payloadHash=c3d3dc56797d71f2eb35b6c7a0228fff4135195e8758a2e80e849a36e15f556c
|
||||||
|
scope.selectOne -> none
|
||||||
|
chunk.count -> 0
|
||||||
|
scope.insert chunkTotal=3 received=0 completed=1 lastError=- stateJson={"phase":"RECEIVED"}
|
||||||
|
result.update id=2001 success=0 rowCount=1 resultFilename=collect-data-1001-result.xlsx resultFileUrl=- fileSize=0 errorMessage=-
|
||||||
|
task.update id=1001 status=RUNNING errorMessage=- successFileCount=- failedFileCount=- finishedAt=- resultJson.len=333 resultJson.hash=2f13867b410c
|
||||||
|
task.load id=1001 status=RUNNING module=COLLECT_DATA
|
||||||
|
thrown: none
|
||||||
|
vo: taskId=1001 resultId=2001 chunk=3/3 done=true receivedRows=1 currentChunkRows=1 dedupeFiltered=0 invalidFiltered=0 brandRejected=0 brandQueryFailed=0 finalRows=1 taskStatus=RUNNING
|
||||||
|
counters lockAcquire=1 outsideLock=4 taskUpdate=1 resultUpdate=1 resultInsert=0 chunkInsert=1 scopeWrite=1 jobEnqueue=0
|
||||||
|
|
||||||
|
== case=duplicateChunkIgnored
|
||||||
|
storage.ensure sharedWrite=true
|
||||||
|
task.load id=1001 status=RUNNING module=COLLECT_DATA
|
||||||
|
outsideLock filter rows=1 owner=7 dedupe=0 invalid=0
|
||||||
|
outsideLock brandFilter accepted=1 rejected=0 queryFailed=0
|
||||||
|
outsideLock asinCache.write country=- taskId=1001 rows=1
|
||||||
|
outsideLock recheck.enqueue n=0
|
||||||
|
lock.acquire taskId=1001 waitMillis=5000
|
||||||
|
task.load id=1001 status=RUNNING module=COLLECT_DATA
|
||||||
|
result.selectOne -> id=2001
|
||||||
|
chunk.selectOne -> idx=1
|
||||||
|
thrown: none
|
||||||
|
vo: taskId=1001 resultId=2001 chunk=1/3 done=false receivedRows=0 currentChunkRows=0 dedupeFiltered=0 invalidFiltered=0 brandRejected=0 brandQueryFailed=0 finalRows=0 taskStatus=RUNNING
|
||||||
|
counters lockAcquire=1 outsideLock=4 taskUpdate=0 resultUpdate=0 resultInsert=0 chunkInsert=0 scopeWrite=0 jobEnqueue=0
|
||||||
|
|
||||||
|
== case=resultRowCreatedOnDemand
|
||||||
|
storage.ensure sharedWrite=true
|
||||||
|
task.load id=1001 status=RUNNING module=COLLECT_DATA
|
||||||
|
outsideLock filter rows=1 owner=7 dedupe=0 invalid=0
|
||||||
|
outsideLock brandFilter accepted=1 rejected=0 queryFailed=0
|
||||||
|
outsideLock asinCache.write country=- taskId=1001 rows=1
|
||||||
|
outsideLock recheck.enqueue n=0
|
||||||
|
lock.acquire taskId=1001 waitMillis=5000
|
||||||
|
task.load id=1001 status=RUNNING module=COLLECT_DATA
|
||||||
|
result.selectOne -> id=-
|
||||||
|
result.insert taskId=1001 sourceFilename=- rowCount=0 success=-
|
||||||
|
chunk.selectOne -> none
|
||||||
|
payload.storeResult key=chunk-1 len=127 hash=23c6b2cbef4e
|
||||||
|
resultItem.upsert chunk=1 accepted=1 ref=rustfs:result-chunk-1
|
||||||
|
payload.storeChunk idx=1 len=127 hash=23c6b2cbef4e
|
||||||
|
chunk.insert scope=sub-1 idx=1/1 payload=rustfs:chunk-1 payloadHash=23c6b2cbef4e8aec3d03e19cd8ad481749cfd1aea7addf69357cb6ea842f0365
|
||||||
|
scope.selectOne -> none
|
||||||
|
chunk.count -> 0
|
||||||
|
scope.insert chunkTotal=1 received=0 completed=1 lastError=- stateJson={"phase":"RECEIVED"}
|
||||||
|
result.update id=2001 success=- rowCount=1 resultFilename=collect-data-1001-result.xlsx resultFileUrl=- fileSize=0 errorMessage=-
|
||||||
|
task.update id=1001 status=RUNNING errorMessage=- successFileCount=- failedFileCount=- finishedAt=- resultJson.len=333 resultJson.hash=2f13867b410c
|
||||||
|
task.load id=1001 status=RUNNING module=COLLECT_DATA
|
||||||
|
thrown: none
|
||||||
|
vo: taskId=1001 resultId=2001 chunk=1/1 done=true receivedRows=1 currentChunkRows=1 dedupeFiltered=0 invalidFiltered=0 brandRejected=0 brandQueryFailed=0 finalRows=1 taskStatus=RUNNING
|
||||||
|
counters lockAcquire=1 outsideLock=4 taskUpdate=1 resultUpdate=1 resultInsert=1 chunkInsert=1 scopeWrite=1 jobEnqueue=0
|
||||||
|
|
||||||
|
== case=errorWithReceivedChunks
|
||||||
|
storage.ensure sharedWrite=true
|
||||||
|
task.load id=1001 status=RUNNING module=COLLECT_DATA
|
||||||
|
outsideLock filter rows=1 owner=7 dedupe=0 invalid=0
|
||||||
|
outsideLock brandFilter accepted=1 rejected=0 queryFailed=0
|
||||||
|
outsideLock asinCache.write country=- taskId=1001 rows=1
|
||||||
|
outsideLock recheck.enqueue n=0
|
||||||
|
lock.acquire taskId=1001 waitMillis=5000
|
||||||
|
task.load id=1001 status=RUNNING module=COLLECT_DATA
|
||||||
|
result.selectOne -> id=2001
|
||||||
|
chunk.selectOne -> none
|
||||||
|
payload.storeResult key=chunk-2 len=127 hash=ead7a0034044
|
||||||
|
resultItem.upsert chunk=2 accepted=1 ref=rustfs:result-chunk-2
|
||||||
|
payload.storeChunk idx=2 len=127 hash=ead7a0034044
|
||||||
|
chunk.insert scope=sub-1 idx=2/3 payload=rustfs:chunk-2 payloadHash=ead7a00340449db788bc72ac7726b1c98b2d0bf42c9dd7e3a5f806fea91b32de
|
||||||
|
scope.selectOne -> none
|
||||||
|
chunk.count -> 2
|
||||||
|
scope.insert chunkTotal=3 received=2 completed=0 lastError=Python 采集异常 stateJson={"phase":"RECEIVED"}
|
||||||
|
result.update id=2001 success=0 rowCount=1 resultFilename=- resultFileUrl=- fileSize=- errorMessage=Python 采集异常
|
||||||
|
task.update id=1001 status=FAILED errorMessage=Python 采集异常 successFileCount=- failedFileCount=1 finishedAt=set resultJson.len=333 resultJson.hash=2f13867b410c
|
||||||
|
chunk.count -> 2
|
||||||
|
result.update id=2001 success=0 rowCount=1 resultFilename=collect-data-1001-result.xlsx resultFileUrl=- fileSize=0 errorMessage=Python 采集异常
|
||||||
|
task.update id=1001 status=FAILED errorMessage=Python 采集异常 successFileCount=- failedFileCount=1 finishedAt=set resultJson.len=333 resultJson.hash=2f13867b410c
|
||||||
|
task.load id=1001 status=FAILED module=COLLECT_DATA
|
||||||
|
thrown: none
|
||||||
|
vo: taskId=1001 resultId=2001 chunk=2/3 done=false receivedRows=1 currentChunkRows=1 dedupeFiltered=0 invalidFiltered=0 brandRejected=0 brandQueryFailed=0 finalRows=1 taskStatus=FAILED
|
||||||
|
counters lockAcquire=1 outsideLock=4 taskUpdate=2 resultUpdate=2 resultInsert=0 chunkInsert=1 scopeWrite=1 jobEnqueue=0
|
||||||
|
|
||||||
|
== case=errorWithoutChunks
|
||||||
|
storage.ensure sharedWrite=true
|
||||||
|
task.load id=1001 status=RUNNING module=COLLECT_DATA
|
||||||
|
outsideLock filter rows=1 owner=7 dedupe=0 invalid=0
|
||||||
|
outsideLock brandFilter accepted=1 rejected=0 queryFailed=0
|
||||||
|
outsideLock asinCache.write country=- taskId=1001 rows=1
|
||||||
|
outsideLock recheck.enqueue n=0
|
||||||
|
lock.acquire taskId=1001 waitMillis=5000
|
||||||
|
task.load id=1001 status=RUNNING module=COLLECT_DATA
|
||||||
|
result.selectOne -> id=2001
|
||||||
|
chunk.selectOne -> none
|
||||||
|
payload.storeResult key=chunk-1 len=127 hash=e1eb2ee676be
|
||||||
|
resultItem.upsert chunk=1 accepted=1 ref=rustfs:result-chunk-1
|
||||||
|
payload.storeChunk idx=1 len=127 hash=e1eb2ee676be
|
||||||
|
chunk.insert scope=sub-1 idx=1/2 payload=rustfs:chunk-1 payloadHash=e1eb2ee676be676c79b78c1ff6103b0a6afea0c4fc1c71be60a6119497e1b33c
|
||||||
|
scope.selectOne -> none
|
||||||
|
chunk.count -> 0
|
||||||
|
scope.insert chunkTotal=2 received=0 completed=0 lastError=Python 采集异常 stateJson={"phase":"RECEIVED"}
|
||||||
|
result.update id=2001 success=0 rowCount=1 resultFilename=- resultFileUrl=- fileSize=- errorMessage=Python 采集异常
|
||||||
|
task.update id=1001 status=FAILED errorMessage=Python 采集异常 successFileCount=- failedFileCount=1 finishedAt=set resultJson.len=333 resultJson.hash=2f13867b410c
|
||||||
|
chunk.count -> 0
|
||||||
|
task.load id=1001 status=FAILED module=COLLECT_DATA
|
||||||
|
thrown: none
|
||||||
|
vo: taskId=1001 resultId=2001 chunk=1/2 done=false receivedRows=1 currentChunkRows=1 dedupeFiltered=0 invalidFiltered=0 brandRejected=0 brandQueryFailed=0 finalRows=1 taskStatus=FAILED
|
||||||
|
counters lockAcquire=1 outsideLock=4 taskUpdate=1 resultUpdate=1 resultInsert=0 chunkInsert=1 scopeWrite=1 jobEnqueue=0
|
||||||
|
|
||||||
|
== case=pendingTaskResetToRunning
|
||||||
|
storage.ensure sharedWrite=true
|
||||||
|
task.load id=1001 status=PENDING module=COLLECT_DATA
|
||||||
|
outsideLock filter rows=1 owner=7 dedupe=0 invalid=0
|
||||||
|
outsideLock brandFilter accepted=1 rejected=0 queryFailed=0
|
||||||
|
outsideLock asinCache.write country=- taskId=1001 rows=1
|
||||||
|
outsideLock recheck.enqueue n=0
|
||||||
|
lock.acquire taskId=1001 waitMillis=5000
|
||||||
|
task.load id=1001 status=PENDING module=COLLECT_DATA
|
||||||
|
task.update id=1001 status=RUNNING errorMessage=- successFileCount=- failedFileCount=- finishedAt=- resultJson.len=50 resultJson.hash=4e7ad4d07deb
|
||||||
|
result.selectOne -> id=2001
|
||||||
|
chunk.selectOne -> none
|
||||||
|
payload.storeResult key=chunk-1 len=127 hash=a55ecd3678de
|
||||||
|
resultItem.upsert chunk=1 accepted=1 ref=rustfs:result-chunk-1
|
||||||
|
payload.storeChunk idx=1 len=127 hash=a55ecd3678de
|
||||||
|
chunk.insert scope=sub-1 idx=1/2 payload=rustfs:chunk-1 payloadHash=a55ecd3678decea68f9e2a7b478fdb4e42575fecb11bd2aa15dd849f42a26e7d
|
||||||
|
scope.selectOne -> none
|
||||||
|
chunk.count -> 0
|
||||||
|
scope.insert chunkTotal=2 received=0 completed=0 lastError=- stateJson={"phase":"RECEIVED"}
|
||||||
|
task.update id=1001 status=RUNNING errorMessage=- successFileCount=- failedFileCount=- finishedAt=- resultJson.len=333 resultJson.hash=2f13867b410c
|
||||||
|
task.load id=1001 status=RUNNING module=COLLECT_DATA
|
||||||
|
thrown: none
|
||||||
|
vo: taskId=1001 resultId=2001 chunk=1/2 done=false receivedRows=1 currentChunkRows=1 dedupeFiltered=0 invalidFiltered=0 brandRejected=0 brandQueryFailed=0 finalRows=1 taskStatus=RUNNING
|
||||||
|
counters lockAcquire=1 outsideLock=4 taskUpdate=2 resultUpdate=0 resultInsert=0 chunkInsert=1 scopeWrite=1 jobEnqueue=0
|
||||||
|
|
||||||
|
== case=blankAsinRowKeptOutOfFilter
|
||||||
|
storage.ensure sharedWrite=true
|
||||||
|
task.load id=1001 status=RUNNING module=COLLECT_DATA
|
||||||
|
outsideLock filter rows=0 owner=7 dedupe=0 invalid=0
|
||||||
|
outsideLock brandFilter accepted=0 rejected=0 queryFailed=0
|
||||||
|
outsideLock recheck.enqueue n=0
|
||||||
|
lock.acquire taskId=1001 waitMillis=5000
|
||||||
|
task.load id=1001 status=RUNNING module=COLLECT_DATA
|
||||||
|
result.selectOne -> id=2001
|
||||||
|
chunk.selectOne -> none
|
||||||
|
payload.storeChunk idx=1 len=113 hash=e0365b1c5568
|
||||||
|
chunk.insert scope=sub-1 idx=1/1 payload=rustfs:chunk-1 payloadHash=e0365b1c556879ffbb03ada23f9b4d53a2cbc9cb025e885bef262febfec13cd5
|
||||||
|
scope.selectOne -> none
|
||||||
|
chunk.count -> 0
|
||||||
|
scope.insert chunkTotal=1 received=0 completed=0 lastError=- stateJson={"phase":"RECEIVED"}
|
||||||
|
task.update id=1001 status=RUNNING errorMessage=- successFileCount=- failedFileCount=- finishedAt=- resultJson.len=333 resultJson.hash=53ca52c40ab0
|
||||||
|
task.load id=1001 status=RUNNING module=COLLECT_DATA
|
||||||
|
thrown: none
|
||||||
|
vo: taskId=1001 resultId=2001 chunk=1/1 done=false receivedRows=1 currentChunkRows=1 dedupeFiltered=0 invalidFiltered=0 brandRejected=0 brandQueryFailed=0 finalRows=0 taskStatus=RUNNING
|
||||||
|
counters lockAcquire=1 outsideLock=3 taskUpdate=1 resultUpdate=0 resultInsert=0 chunkInsert=1 scopeWrite=1 jobEnqueue=0
|
||||||
|
|
||||||
|
== case=terminalRejectedBeforeLock
|
||||||
|
storage.ensure sharedWrite=true
|
||||||
|
task.load id=1001 status=SUCCESS module=COLLECT_DATA
|
||||||
|
thrown: BusinessException code=40901 message=任务已结束,拒绝重复提交
|
||||||
|
counters lockAcquire=0 outsideLock=0 taskUpdate=0 resultUpdate=0 resultInsert=0 chunkInsert=0 scopeWrite=0 jobEnqueue=0
|
||||||
|
|
||||||
|
== case=taskNotFoundBeforeLock
|
||||||
|
storage.ensure sharedWrite=true
|
||||||
|
task.load id=1001 status=- module=-
|
||||||
|
thrown: BusinessException code=- message=任务不存在
|
||||||
|
counters lockAcquire=0 outsideLock=0 taskUpdate=0 resultUpdate=0 resultInsert=0 chunkInsert=0 scopeWrite=0 jobEnqueue=0
|
||||||
|
|
||||||
|
== case=wrongModuleBeforeLock
|
||||||
|
storage.ensure sharedWrite=true
|
||||||
|
task.load id=1001 status=RUNNING module=APPEARANCE_PATENT
|
||||||
|
thrown: BusinessException code=- message=任务不存在
|
||||||
|
counters lockAcquire=0 outsideLock=0 taskUpdate=0 resultUpdate=0 resultInsert=0 chunkInsert=0 scopeWrite=0 jobEnqueue=0
|
||||||
|
|
||||||
|
== case=nullRequest
|
||||||
|
thrown: BusinessException code=- message=请求内容为空
|
||||||
|
counters lockAcquire=0 outsideLock=0 taskUpdate=0 resultUpdate=0 resultInsert=0 chunkInsert=0 scopeWrite=0 jobEnqueue=0
|
||||||
|
|
||||||
|
== case=rustfsDisabled
|
||||||
|
storage.ensure sharedWrite=false
|
||||||
|
thrown: BusinessException code=- message=RustFS 未配置,采集结果回传暂不可接收
|
||||||
|
counters lockAcquire=0 outsideLock=0 taskUpdate=0 resultUpdate=0 resultInsert=0 chunkInsert=0 scopeWrite=0 jobEnqueue=0
|
||||||
|
|
||||||
|
== case=chunkPointerNotRustfs
|
||||||
|
storage.ensure sharedWrite=true
|
||||||
|
task.load id=1001 status=RUNNING module=COLLECT_DATA
|
||||||
|
outsideLock filter rows=1 owner=7 dedupe=0 invalid=0
|
||||||
|
outsideLock brandFilter accepted=1 rejected=0 queryFailed=0
|
||||||
|
outsideLock asinCache.write country=- taskId=1001 rows=1
|
||||||
|
outsideLock recheck.enqueue n=0
|
||||||
|
lock.acquire taskId=1001 waitMillis=5000
|
||||||
|
task.load id=1001 status=RUNNING module=COLLECT_DATA
|
||||||
|
result.selectOne -> id=2001
|
||||||
|
chunk.selectOne -> none
|
||||||
|
payload.storeResult key=chunk-1 len=127 hash=b40fdfc4dc5b
|
||||||
|
resultItem.upsert chunk=1 accepted=1 ref=rustfs:result-chunk-1
|
||||||
|
payload.storeChunk idx=1 len=127 hash=b40fdfc4dc5b
|
||||||
|
payload.delete pointer=local:chunk-1
|
||||||
|
thrown: BusinessException code=- message=采集结果分片必须写入 RustFS
|
||||||
|
counters lockAcquire=1 outsideLock=4 taskUpdate=0 resultUpdate=0 resultInsert=0 chunkInsert=0 scopeWrite=0 jobEnqueue=0
|
||||||
|
|
||||||
|
== case=lockBusyRejected
|
||||||
|
storage.ensure sharedWrite=true
|
||||||
|
task.load id=1001 status=RUNNING module=COLLECT_DATA
|
||||||
|
outsideLock filter rows=1 owner=7 dedupe=0 invalid=0
|
||||||
|
outsideLock brandFilter accepted=1 rejected=0 queryFailed=0
|
||||||
|
outsideLock asinCache.write country=- taskId=1001 rows=1
|
||||||
|
outsideLock recheck.enqueue n=0
|
||||||
|
lock.acquire taskId=1001 waitMillis=5000 -> busy
|
||||||
|
thrown: BusinessException code=40902 message=任务正在处理,请稍后再试
|
||||||
|
counters lockAcquire=1 outsideLock=4 taskUpdate=0 resultUpdate=0 resultInsert=0 chunkInsert=0 scopeWrite=0 jobEnqueue=0
|
||||||
|
|
||||||
|
== case=invalidTaskIdFallsThroughToNotFound
|
||||||
|
storage.ensure sharedWrite=true
|
||||||
|
task.load id=0 status=RUNNING module=COLLECT_DATA
|
||||||
|
outsideLock filter rows=1 owner=7 dedupe=0 invalid=0
|
||||||
|
outsideLock brandFilter accepted=1 rejected=0 queryFailed=0
|
||||||
|
outsideLock asinCache.write country=- taskId=0 rows=1
|
||||||
|
outsideLock recheck.enqueue n=0
|
||||||
|
lock.acquire taskId=0 waitMillis=5000
|
||||||
|
task.load id=0 status=RUNNING module=COLLECT_DATA
|
||||||
|
result.selectOne -> id=2001
|
||||||
|
chunk.selectOne -> none
|
||||||
|
payload.storeResult key=chunk-1 len=127 hash=291c0ca80f2c
|
||||||
|
resultItem.upsert chunk=1 accepted=1 ref=rustfs:result-chunk-1
|
||||||
|
payload.storeChunk idx=1 len=127 hash=291c0ca80f2c
|
||||||
|
chunk.insert scope=sub-1 idx=1/2 payload=rustfs:chunk-1 payloadHash=291c0ca80f2cd6a87cff2dad9e7dbf122f6111e9dc930c0f8dcbcc7f74bf8f46
|
||||||
|
scope.selectOne -> none
|
||||||
|
chunk.count -> 0
|
||||||
|
scope.insert chunkTotal=2 received=0 completed=0 lastError=- stateJson={"phase":"RECEIVED"}
|
||||||
|
task.update id=1001 status=RUNNING errorMessage=- successFileCount=- failedFileCount=- finishedAt=- resultJson.len=333 resultJson.hash=2f13867b410c
|
||||||
|
task.load id=0 status=RUNNING module=COLLECT_DATA
|
||||||
|
thrown: none
|
||||||
|
vo: taskId=1001 resultId=2001 chunk=1/2 done=false receivedRows=1 currentChunkRows=1 dedupeFiltered=0 invalidFiltered=0 brandRejected=0 brandQueryFailed=0 finalRows=1 taskStatus=RUNNING
|
||||||
|
counters lockAcquire=1 outsideLock=4 taskUpdate=1 resultUpdate=0 resultInsert=0 chunkInsert=1 scopeWrite=1 jobEnqueue=0
|
||||||
|
|
||||||
Reference in New Issue
Block a user