fix(品牌检测): 熔断中止落真实原因+部分结果可下载,熔断加持续时长门槛

- 熔断改为「连续失败持续 6 分钟未恢复」才中止:原 8 次即中止,5 线程一轮
  就能凑满,重跑机制没机会生效导致任务失败率过高(任务 2309 复盘)
- 期间每 30s 冷却重试,重跑轮次 10→30;限流窗口恢复后任务自动跑完
- 失败终态上报内部接口 /api/internal/brand/tasks/{id}/abort:落真实原因 +
  用已收分片部分组装结果(未检测品牌单独成 sheet),不再悬挂到心跳超时被
  判「前端长时间无响应」且已跑数据无法下载
- 组装前从分片重建聚合(缓存快照可能缺后加字段如 keptBrands)
- 前端:失败任务有结果文件即显示「下载结果」
This commit is contained in:
2026-09-14 17:04:19 +08:00
parent 2d5ca99068
commit b2b9983b07
7 changed files with 500 additions and 18 deletions
@@ -0,0 +1,249 @@
package com.nanri.aiimage.modules.brand.service;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.conditions.Wrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.service.DistributedJobLockService;
import com.nanri.aiimage.config.BrandProgressProperties;
import com.nanri.aiimage.config.StorageProperties;
import com.nanri.aiimage.modules.brand.mapper.BrandCrawlTaskMapper;
import com.nanri.aiimage.modules.brand.model.dto.BrandCrawlResultFileDto;
import com.nanri.aiimage.modules.brand.model.dto.BrandFileAggregateCacheDto;
import com.nanri.aiimage.modules.brand.model.dto.BrandParsedFileCacheDto;
import com.nanri.aiimage.modules.brand.model.dto.BrandSourceFileDto;
import com.nanri.aiimage.modules.brand.model.entity.BrandCrawlTaskEntity;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.task.service.TaskDistributedLockService;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.api.io.TempDir;
import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.isNull;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* 品牌任务中止上报(abortTask):熔断中止时用已收到的分片部分组装结果,
* 落 failed + 真实原因,避免任务悬挂到心跳超时、已跑出的数据无法下载。
*/
@ExtendWith(MockitoExtension.class)
class BrandTaskAbortServiceTest {
@Mock private BrandCrawlTaskMapper brandCrawlTaskMapper;
@Mock private OssStorageService ossStorageService;
@Mock private StorageProperties storageProperties;
@Mock private BrandProgressProperties brandProgressProperties;
@Mock private BrandTaskProgressCacheService brandTaskProgressCacheService;
@Mock private BrandTaskStorageService brandTaskStorageService;
@Mock private DistributedJobLockService distributedJobLockService;
@Mock private TaskDistributedLockService taskDistributedLockService;
@Mock private TaskFileJobService taskFileJobService;
@Mock private TaskProgressSnapshotService taskProgressSnapshotService;
@TempDir
Path tempDir;
@BeforeAll
static void initializeTableInfo() {
TableInfoHelper.initTableInfo(
new MapperBuilderAssistant(new MybatisConfiguration(), ""),
BrandCrawlTaskEntity.class);
}
private BrandTaskService service() {
return new BrandTaskService(brandCrawlTaskMapper, ossStorageService, storageProperties,
brandProgressProperties, brandTaskProgressCacheService, brandTaskStorageService,
distributedJobLockService, taskDistributedLockService, new ObjectMapper(),
taskFileJobService, taskProgressSnapshotService);
}
private BrandCrawlTaskEntity task(String status) {
return task(status, "https://oss.example/source.xlsx");
}
private BrandCrawlTaskEntity task(String status, String fileUrl) {
BrandCrawlTaskEntity entity = new BrandCrawlTaskEntity();
entity.setId(2309L);
entity.setUserId(1050L);
entity.setStatus(status);
entity.setStrategy("Simple");
BrandSourceFileDto source = new BrandSourceFileDto();
source.setFileUrl(fileUrl);
source.setOriginalFilename("待检测品牌.xlsx");
entity.setFilePaths(new ObjectMapper().valueToTree(List.of(source)).toString());
return entity;
}
@SuppressWarnings("unchecked")
private LambdaUpdateWrapper<BrandCrawlTaskEntity> captureUpdateWrapper() {
ArgumentCaptor<Wrapper<BrandCrawlTaskEntity>> captor = ArgumentCaptor.forClass(Wrapper.class);
verify(brandCrawlTaskMapper).update(isNull(), captor.capture());
return (LambdaUpdateWrapper<BrandCrawlTaskEntity>) captor.getValue();
}
@Test
void terminalTaskIsSkippedIdempotently() {
when(brandCrawlTaskMapper.selectById(2309L)).thenReturn(task("success"));
TaskDistributedLockService.LockHandle handle = mock(TaskDistributedLockService.LockHandle.class);
when(taskDistributedLockService.acquire(anyString(), any(), anyLong())).thenReturn(handle);
Map<String, Object> result = service().abortTask(2309L, "限流中止");
assertEquals("success", result.get("status"));
assertFalse((Boolean) result.get("resultGenerated"));
verify(brandCrawlTaskMapper, never()).update(any(), any());
verify(brandTaskProgressCacheService, never()).markFailed(anyLong(), anyString());
}
@Test
void abortWithoutChunksMarksFailedOnly() {
when(brandCrawlTaskMapper.selectById(2309L)).thenReturn(task("running"));
TaskDistributedLockService.LockHandle handle = mock(TaskDistributedLockService.LockHandle.class);
when(taskDistributedLockService.acquire(anyString(), any(), anyLong())).thenReturn(handle);
when(brandTaskStorageService.getParsedPayload(2309L)).thenReturn(List.of());
when(brandTaskStorageService.getAllFileAggregates(2309L)).thenReturn(Map.of());
when(brandCrawlTaskMapper.update(isNull(), any())).thenReturn(1);
Map<String, Object> result = service().abortTask(2309L, "连续 8 次请求被 WIPO 限流");
assertEquals("failed", result.get("status"));
assertFalse((Boolean) result.get("resultGenerated"));
LambdaUpdateWrapper<BrandCrawlTaskEntity> wrapper = captureUpdateWrapper();
assertFalse(wrapper.getSqlSet().contains("result_paths"));
verify(brandTaskProgressCacheService).markFailed(2309L, "连续 8 次请求被 WIPO 限流");
verify(ossStorageService, never()).uploadResultFile(any(), anyString());
}
@Test
void abortWithChunksAssemblesPartialResult() throws Exception {
assertPartialAssembly("running");
}
/** 存量补救:已被心跳兜底判失败、无结果文件的旧任务(如任务 2309)补组装。 */
@Test
void failedTaskWithoutResultIsRepairedByAssembly() throws Exception {
assertPartialAssembly("failed");
}
@Test
void failedTaskWithExistingResultIsSkipped() {
BrandCrawlTaskEntity entity = task("failed");
entity.setResultPaths("{\"zip_url\":\"https://oss.example/existing.zip\"}");
when(brandCrawlTaskMapper.selectById(2309L)).thenReturn(entity);
TaskDistributedLockService.LockHandle handle = mock(TaskDistributedLockService.LockHandle.class);
when(taskDistributedLockService.acquire(anyString(), any(), anyLong())).thenReturn(handle);
Map<String, Object> result = service().abortTask(2309L, "限流中止");
assertEquals("failed", result.get("status"));
assertFalse((Boolean) result.get("resultGenerated"));
verify(brandCrawlTaskMapper, never()).update(any(), any());
}
private void assertPartialAssembly(String initialStatus) throws Exception {
File sourceFile = tempDir.resolve("source.xlsx").toFile();
Files.write(sourceFile.toPath(), new byte[]{1, 2, 3});
String fileUrl = sourceFile.getAbsolutePath();
when(brandCrawlTaskMapper.selectById(2309L)).thenReturn(task(initialStatus, fileUrl));
TaskDistributedLockService.LockHandle handle = mock(TaskDistributedLockService.LockHandle.class);
when(taskDistributedLockService.acquire(anyString(), any(), anyLong())).thenReturn(handle);
when(storageProperties.getLocalTempDir()).thenReturn(tempDir.toString());
BrandParsedFileCacheDto cached = new BrandParsedFileCacheDto();
cached.setFileUrl(fileUrl);
cached.setSheetName("Sheet1");
cached.setColumns(new ArrayList<>(List.of("品牌", "ASIN")));
cached.setRows(new ArrayList<>(List.of(
row("A"), row("B"), row("C"))));
when(brandTaskStorageService.getParsedPayload(2309L)).thenReturn(List.of(cached));
BrandFileAggregateCacheDto aggregate = new BrandFileAggregateCacheDto();
aggregate.setFileUrl(fileUrl);
aggregate.setChunkTotal(3);
aggregate.setReceivedChunkCount(1);
aggregate.setCompleted(false);
aggregate.setKeptBrands(new ArrayList<>(List.of("A")));
Map<String, BrandFileAggregateCacheDto> aggregates = new LinkedHashMap<>();
aggregates.put(fileUrl, aggregate);
when(brandTaskStorageService.getAllFileAggregates(2309L)).thenReturn(aggregates);
java.util.concurrent.atomic.AtomicReference<org.apache.poi.ss.usermodel.Workbook> capturedWorkbook =
new java.util.concurrent.atomic.AtomicReference<>();
when(ossStorageService.uploadResultFile(any(File.class), anyString())).thenAnswer(invocation -> {
File uploaded = invocation.getArgument(0);
if (uploaded.getName().endsWith(".xlsx") && capturedWorkbook.get() == null) {
capturedWorkbook.set(org.apache.poi.ss.usermodel.WorkbookFactory.create(uploaded));
}
return "result-key";
});
when(ossStorageService.generateFreshDownloadUrl(anyString())).thenReturn("https://oss.example/result-key");
when(brandCrawlTaskMapper.update(isNull(), any())).thenReturn(1);
lenient().when(brandTaskStorageService.countCompletedFiles(2309L)).thenReturn(0);
Map<String, Object> result = service().abortTask(2309L, "连续 8 次请求被 WIPO 限流");
assertEquals("failed", result.get("status"));
assertTrue((Boolean) result.get("resultGenerated"));
LambdaUpdateWrapper<BrandCrawlTaskEntity> wrapper = captureUpdateWrapper();
assertTrue(wrapper.getSqlSet().contains("result_paths"));
verify(ossStorageService, org.mockito.Mockito.atLeastOnce()).uploadResultFile(any(File.class), anyString());
org.apache.poi.ss.usermodel.Workbook workbook = capturedWorkbook.get();
assertNotNull(workbook, "应生成结果工作簿");
org.apache.poi.ss.usermodel.Sheet undetected = workbook.getSheet("未检测品牌");
assertNotNull(undetected, "部分结果应包含「未检测品牌」sheet");
assertEquals("B", undetected.getRow(1).getCell(0).getStringCellValue());
assertEquals("C", undetected.getRow(2).getCell(0).getStringCellValue());
assertEquals(1, workbook.getSheet("Sheet1").getLastRowNum(), "主 sheet 只剩表头 + 已检测品牌 A 一行");
}
@Test
void abortReportsRealErrorMessageInsteadOfStalePlaceholder() throws Exception {
when(brandCrawlTaskMapper.selectById(2309L)).thenReturn(task("running"));
TaskDistributedLockService.LockHandle handle = mock(TaskDistributedLockService.LockHandle.class);
when(taskDistributedLockService.acquire(anyString(), any(), anyLong())).thenReturn(handle);
when(brandTaskStorageService.getParsedPayload(2309L)).thenReturn(List.of());
when(brandTaskStorageService.getAllFileAggregates(2309L)).thenReturn(Map.of());
when(brandCrawlTaskMapper.update(isNull(), any())).thenReturn(1);
String realMessage = "连续 8 次请求被 WIPO 限流(返回 Forbidden),已中止任务;请检查代理配置或错峰重跑";
service().abortTask(2309L, realMessage);
LambdaUpdateWrapper<BrandCrawlTaskEntity> wrapper = captureUpdateWrapper();
assertTrue(wrapper.getParamNameValuePairs().containsValue(realMessage),
"error_message 应写入熔断的真实原因,而不是心跳超时兜底文案");
}
private Map<String, Object> row(String brand) {
Map<String, Object> row = new LinkedHashMap<>();
row.put("品牌", brand);
row.put("ASIN", "B0" + brand);
return row;
}
}