super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(delete-brand): 分片缺失降级组装收尾+失败兜底生成可下载结果

Browse Source 
- 结果分片缺失 ≤ missingChunkFallbackLimit(默认20) 时降级组装，缺失行标「未回传」，任务不再整体白跑
- 超阈值失败/组装异常时通过 ResultFileJobHandler.fallbackAssembleOnFailure 兜底生成部分结果文件
- 组装失败文案带缺失分片号明细（如 缺失分片: [247]（已收 4283/4284））
- 新增 GET /api/delete-brand/tasks/{taskId}/missing-chunks，补传后自动恢复终态失败组装 job
- stale-check 失败文案改为含最后心跳时间与最近组装失败真实原因（修复误导性「回传长时间无响应」）
- 新增降级合并/缺失分片单测，Handler/Worker 日志审计 stage 白名单补 FALLBACK
This commit is contained in:
黄自达
2026-09-08 13:18:17 +08:00
parent d9e3f34d37
commit f2906e8d6f
13 changed files with 581 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-java/src/test/java/com/nanri/aiimage/config/TaskResultFileJobWorkerLogAuditTest.java
+1 -1
View File
@@ -100,7 +100,7 @@ class TaskResultFileJobWorkerLogAuditTest {
}
// 阶段字面量来自允许集合（防新增阶段拼写漂移）
List<String> allowed = List.of("SKIP_OWNER", "REQUEUE", "WAIT_LLM", "DEFER", "SUCCESS",
"ORPHAN", "FAILED", "FINALIZE", "HEARTBEAT");
"ORPHAN", "FAILED", "FINALIZE", "HEARTBEAT", "FALLBACK");
for (String l : perJob) {
if (l.contains("stage=")) {
int idx = l.indexOf("stage=");
backend-java/src/test/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandRunServiceFallbackTest.java
+188
View File
@@ -0,0 +1,188 @@
package com.nanri.aiimage.modules.deletebrand.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.DeleteBrandProgressProperties;
import com.nanri.aiimage.config.TaskPressureProperties;
import com.nanri.aiimage.modules.deletebrand.model.cache.DeleteBrandParsedFileCacheDto;
import com.nanri.aiimage.modules.deletebrand.model.dto.DeleteBrandCountryResultItemDto;
import com.nanri.aiimage.modules.deletebrand.model.dto.DeleteBrandProcessedCountryDto;
import com.nanri.aiimage.modules.deletebrand.model.dto.DeleteBrandResultFileDto;
import com.nanri.aiimage.modules.deletebrand.model.vo.DeleteBrandCountryAsinVo;
import com.nanri.aiimage.modules.deletebrand.model.vo.DeleteBrandCountryGroupVo;
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskProgressLightAssembler;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
import org.junit.jupiter.api.Test;
import org.springframework.test.util.ReflectionTestUtils;
import org.springframework.transaction.PlatformTransactionManager;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
/**
* 删除品牌分片缺失降级组装（修复：缺分片不整任务白跑）纯逻辑单测。
*/
class DeleteBrandRunServiceFallbackTest {
private final DeleteBrandRunService service = new DeleteBrandRunService(
mock(FileTaskMapper.class),
mock(FileResultMapper.class),
mock(LocalFileStorageService.class),
mock(DeleteBrandTaskCacheService.class),
mock(DeleteBrandTaskStorageService.class),
mock(ZiniaoShopSwitchService.class),
mock(OssStorageService.class),
mock(ObjectMapper.class),
mock(DeleteBrandProgressProperties.class),
mock(TaskPressureProperties.class),
mock(TaskDistributedLockService.class),
mock(TaskFileJobService.class),
mock(PlatformTransactionManager.class),
mock(TaskProgressLightAssembler.class));
private DeleteBrandResultFileDto chunk(int index, int total, String asin, String status) {
DeleteBrandResultFileDto dto = new DeleteBrandResultFileDto();
dto.setFileKey("scope-key");
dto.setSourceFilename("魏振峰.xlsx");
dto.setChunkIndex(index);
dto.setChunkTotal(total);
dto.setProcessedRows(index);
dto.setTotalRows(total);
dto.setCurrentCountry("US");
dto.setCurrentAsin(asin);
DeleteBrandProcessedCountryDto country = new DeleteBrandProcessedCountryDto();
country.setCountry("US");
DeleteBrandCountryResultItemDto item = new DeleteBrandCountryResultItemDto();
item.setAsin(asin);
item.setStatus(status);
country.setItems(List.of(item));
dto.setCountries(List.of(country));
return dto;
}
private DeleteBrandParsedFileCacheDto parsedFile(String... asins) {
DeleteBrandParsedFileCacheDto parsed = new DeleteBrandParsedFileCacheDto();
parsed.setFileKey("scope-key");
parsed.setSourceFilename("魏振峰.xlsx");
parsed.setTotalRows(asins.length);
DeleteBrandCountryGroupVo country = new DeleteBrandCountryGroupVo();
country.setCountry("US");
for (String asin : asins) {
DeleteBrandCountryAsinVo asinVo = new DeleteBrandCountryAsinVo();
asinVo.setAsin(asin);
country.getItems().add(asinVo);
}
parsed.setCountries(List.of(country));
return parsed;
}
@SuppressWarnings("unchecked")
private List<Integer> missingChunkIndexes(List<DeleteBrandResultFileDto> chunks) {
return (List<Integer>) ReflectionTestUtils.invokeMethod(service, "missingChunkIndexes", chunks);
}
@SuppressWarnings("unchecked")
private Object mergeChunks(DeleteBrandParsedFileCacheDto parsedFile, List<DeleteBrandResultFileDto> chunks, boolean allowMissing) {
return ReflectionTestUtils.invokeMethod(service, "mergeChunks", parsedFile, chunks, allowMissing);
}
@Test
void missingChunkIndexesFindsGap() {
assertEquals(List.of(3), missingChunkIndexes(List.of(chunk(1, 4, "A", "成功"), chunk(2, 4, "B", "成功"), chunk(4, 4, "D", "成功"))));
}
@Test
void missingChunkIndexesMiddleGapOf27171Scenario() {
// 事故现场：4284 个分片、只缺 247 一个
List<DeleteBrandResultFileDto> chunks = new java.util.ArrayList<>();
for (int i = 1; i <= 4284; i++) {
if (i == 247) {
continue;
}
chunks.add(chunk(i, 4284, "B0" + i, "成功"));
}
assertEquals(List.of(247), missingChunkIndexes(chunks));
}
@Test
void missingChunkIndexesEmptyAndUnknownTotal() {
assertEquals(List.of(), missingChunkIndexes(List.of()));
DeleteBrandResultFileDto noTotal = new DeleteBrandResultFileDto();
noTotal.setChunkIndex(1);
assertEquals(List.of(), missingChunkIndexes(List.of(noTotal)));
}
@Test
void mergeChunksWithAllowMissingFillsPlaceholderStatus() {
// parsed 有 2 个 ASIN，但只回传了 1 个（缺 B0DELETED）
DeleteBrandParsedFileCacheDto parsed = parsedFile("B0FINE", "B0DELETED");
List<DeleteBrandResultFileDto> chunks = List.of(chunk(1, 2, "B0FINE", "成功"));
Object merged = mergeChunks(parsed, chunks, true);
assertEquals(2, mergedAsins(merged).size());
assertEquals("成功", mergedAsins(merged).get("B0FINE"));
assertEquals(DeleteBrandRunService.MISSING_CHUNK_STATUS, mergedAsins(merged).get("B0DELETED"));
}
@Test
void mergeChunksStrictStillThrowsOnMissingAsin() {
DeleteBrandParsedFileCacheDto parsed = parsedFile("B0FINE", "B0DELETED");
List<DeleteBrandResultFileDto> chunks = List.of(chunk(1, 2, "B0FINE", "成功"));
BusinessException ex = assertThrows(BusinessException.class,
() -> mergeChunks(parsed, chunks, false));
assertTrue(ex.getMessage().contains("缺少 ASIN 结果"));
assertTrue(ex.getMessage().contains("B0DELETED"));
}
@Test
void mergeChunksAllowMissingWholeCountryBecomesPlaceholder() {
DeleteBrandParsedFileCacheDto parsed = parsedFile("B0FINE", "B0DELETED");
List<DeleteBrandResultFileDto> chunks = List.of();
Object merged = mergeChunks(parsed, chunks, true);
assertEquals(2, mergedAsins(merged).size());
assertEquals(DeleteBrandRunService.MISSING_CHUNK_STATUS, mergedAsins(merged).get("B0FINE"));
assertEquals(DeleteBrandRunService.MISSING_CHUNK_STATUS, mergedAsins(merged).get("B0DELETED"));
}
@Test
void mergeChunksStrictThrowsOnMissingCountry() {
DeleteBrandParsedFileCacheDto parsed = parsedFile("B0FINE");
BusinessException ex = assertThrows(BusinessException.class,
() -> mergeChunks(parsed, List.of(), false));
assertTrue(ex.getMessage().contains("缺少国家结果"));
}
@Test
void mergeChunksInconsistentChunkTotalStillThrows() {
DeleteBrandParsedFileCacheDto parsed = parsedFile("B0FINE");
BusinessException ex = assertThrows(BusinessException.class,
() -> mergeChunks(parsed, List.of(chunk(1, 3, "B0FINE", "成功"), chunk(2, 4, "X", "成功")), true));
assertTrue(ex.getMessage().contains("chunkTotal 不一致"));
}
/**
* 把 mergeChunks 返回的 MergedDeleteBrandFile（record）转成 asin→status 映射便于断言。
*/
private Map<String, String> mergedAsins(Object merged) {
LinkedHashMap<String, String> map = new LinkedHashMap<>();
List<?> countries = (List<?>) ReflectionTestUtils.getField(merged, "countries");
for (Object country : countries) {
List<?> items = (List<?>) ReflectionTestUtils.getField(country, "items");
for (Object item : items) {
map.put((String) ReflectionTestUtils.getField(item, "asin"), (String) ReflectionTestUtils.getField(item, "status"));
}
}
return map;
}
}
backend-java/src/test/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandStaleTaskServiceTest.java
+36 -1
View File
@@ -8,6 +8,7 @@ import com.nanri.aiimage.config.DeleteBrandProgressProperties;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
@@ -26,7 +27,9 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.isNull;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
@@ -46,6 +49,7 @@ class DeleteBrandStaleTaskServiceTest {
@Mock private DeleteBrandRunService deleteBrandRunService;
@Mock private DeleteBrandProgressProperties deleteBrandProgressProperties;
@Mock private TaskDistributedLockService taskDistributedLockService;
@Mock private TaskFileJobService taskFileJobService;
@BeforeAll
static void initializeTableInfo() {
@@ -178,17 +182,48 @@ class DeleteBrandStaleTaskServiceTest {
verify(deleteBrandRunService, never()).tryFinalizeTask(anyLong(), anyBoolean());
}
@Test
void staleFailReasonIncludesHeartbeatAndAssembleError() {
FileTaskEntity t1 = runningTask(801L);
when(fileTaskMapper.selectList(any())).thenReturn(List.of(t1));
when(fileTaskMapper.selectBatchIds(any())).thenReturn(List.of(runningTask(801L)));
when(fileTaskMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
when(deleteBrandTaskCacheService.getProgress(801L))
.thenReturn(Map.of("last_heartbeat_at", System.currentTimeMillis() - 35 * 60_000L));
lockAvailable();
// 必须放在 lockAvailable() 之后：后设置的 stub 覆盖前面的 lenient 默认值
when(taskFileJobService.latestFailedAssembleError(801L, "DELETE_BRAND"))
.thenReturn("结果分片未完整: 240d70221b104a3c8140555efed4fd21，缺失分片: [247]（已收 4283/4284）");
DeleteBrandStaleTaskService service = service();
failStaleDeleteBrandTasks(service);
@SuppressWarnings({"rawtypes", "unchecked"})
ArgumentCaptor<LambdaUpdateWrapper> update = ArgumentCaptor.forClass(LambdaUpdateWrapper.class);
verify(fileTaskMapper).update(isNull(), update.capture());
// 组装后的 SQL 只含占位符，真实文案在 paramNameValuePairs 中
List<String> values = update.getValue().getParamNameValuePairs().values().stream()
.map(String::valueOf)
.toList();
String errorMessage = values.stream().filter(v -> v.contains("最后心跳")).findFirst().orElse("");
// 文案必须包含真实原因（最后心跳 + 最近一次组装失败），不能只写"回传长时间无响应"
assertTrue(errorMessage.contains("最后心跳"), errorMessage);
assertTrue(errorMessage.contains("最近一次组装失败"), errorMessage);
assertTrue(errorMessage.contains("缺失分片"), errorMessage);
}
private DeleteBrandStaleTaskService service() {
return new DeleteBrandStaleTaskService(
fileTaskMapper, deleteBrandTaskCacheService, deleteBrandTaskStorageService, deleteBrandRunService,
null, null, null, null, null, null, null, null, null, null, null, null, null, null, null,
deleteBrandProgressProperties, null, taskDistributedLockService, null);
deleteBrandProgressProperties, null, taskDistributedLockService, taskFileJobService);
}
private void lockAvailable() {
when(deleteBrandProgressProperties.getHeartbeatTimeoutMinutes()).thenReturn(15L);
when(taskDistributedLockService.acquire(any(), any(), anyLong()))
.thenReturn(mock(TaskDistributedLockService.LockHandle.class));
lenient().when(taskFileJobService.latestFailedAssembleError(any(), anyString())).thenReturn(null);
}
private static void failStaleDeleteBrandTasks(DeleteBrandStaleTaskService service) {
backend-java/src/test/java/com/nanri/aiimage/modules/task/service/DeleteBrandResultFileJobHandlerTest.java
+11
View File
@@ -10,6 +10,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* DeleteBrandResultFileJobHandler 测试（任务 70）。
@@ -64,6 +65,16 @@ class DeleteBrandResultFileJobHandlerTest {
assertFalse(handler.isOwnerScoped());
}
@Test
void fallbackAssembleDelegates() {
TaskFileJobEntity job = new TaskFileJobEntity();
job.setTaskId(27171L);
job.setResultId(30195L);
when(deleteBrandRunService.fallbackAssembleResultFile(job)).thenReturn(true);
assertEquals(true, handler.fallbackAssembleOnFailure(job, "结果分片未完整"));
verify(deleteBrandRunService).fallbackAssembleResultFile(job);
}
@Test
void nullJobGuarded() {
assertTrue(handler.process(null));