super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修复店铺任务数据覆盖bug
Build Backend JAR / build (push) Has been cancelled
Details

Browse Source
This commit is contained in:
黄自达
2026-08-29 00:16:47 +08:00
parent 79596a970c
commit 322905607a
5 changed files with 120 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlExcelAssemblyServiceTest.java
+17 -10
View File
@@ -73,28 +73,35 @@ class ShopDataCrawlExcelAssemblyServiceTest {
}
@Test
void appendsOnlyTheSelectedCountryAndPreservesExistingPictures() throws Exception {
ShopDataCrawlRowDto firstRow = row("2026-07-25", "B012345678");
firstRow.setCommodityImage("https://m.media-amazon.com/images/I/first.jpg");
ShopDataCrawlRowDto secondRow = row("2026-07-26", "B099999999");
secondRow.setCommodityImage(null);
void replacesOnlyTheSelectedCountryAndPreservesOtherSheets() throws Exception {
ShopDataCrawlRowDto ukRow = row("2026-07-25", "B012345678");
ukRow.setCommodityImage("https://m.media-amazon.com/images/I/first.jpg");
ShopDataCrawlRowDto oldDeRow = row("2026-07-26", "B099999998");
oldDeRow.setCommodityImage("https://m.media-amazon.com/images/I/de.jpg");
ShopDataCrawlRowDto newDeRow = row("2026-07-27", "B099999999");
SimilarAsinImageEmbedder imageEmbedder = mock(SimilarAsinImageEmbedder.class);
when(imageEmbedder.fetchAndResizeForCache(firstRow.getCommodityImage()))
when(imageEmbedder.fetchAndResizeForCache(ukRow.getCommodityImage()))
.thenReturn(new SimilarAsinImageEmbedder.ResizedImage(jpegBytes(), 2, 2));
when(imageEmbedder.fetchAndResizeForCache(oldDeRow.getCommodityImage()))
.thenReturn(new SimilarAsinImageEmbedder.ResizedImage(jpegBytes(), 2, 2));
ShopDataCrawlExcelAssemblyService service = new ShopDataCrawlExcelAssemblyService(imageEmbedder);
File base = tempDir.resolve("base.xlsx").toFile();
File output = tempDir.resolve("daily.xlsx").toFile();
service.writeWorkbook(base, List.of(item("UK", firstRow)));
service.appendWorkbook(base, output, List.of(item("DE", secondRow)));
service.writeWorkbook(base, List.of(item("UK", ukRow), item("DE", oldDeRow)));
int total = service.replaceCountriesWorkbook(base, output, List.of(item("DE", newDeRow)));
try (XSSFWorkbook workbook = new XSSFWorkbook(new FileInputStream(output))) {
assertEquals("B012345678", workbook.getSheetAt(0).getRow(1).getCell(1).getStringCellValue());
assertEquals("B099999999", workbook.getSheetAt(1).getRow(1).getCell(1).getStringCellValue());
assertEquals(1, workbook.getAllPictures().size());
assertEquals(1, workbook.getSheetAt(0).getDrawingPatriarch().getShapes().size());
assertEquals(1, workbook.getSheetAt(0).getLastRowNum());
assertEquals(1, workbook.getSheetAt(1).getLastRowNum());
assertEquals(0, workbook.getSheetAt(2).getLastRowNum());
assertEquals(1, workbook.getSheetAt(0).getDrawingPatriarch().getShapes().size());
assertEquals(0, workbook.getSheetAt(1).getDrawingPatriarch().getShapes().size());
}
assertEquals(2, total);
}
private ShopDataCrawlResultItemVo item(String countryCode, ShopDataCrawlRowDto row) {
backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskServiceChunkTest.java
+1 -1
View File
@@ -187,7 +187,7 @@ class ShopDataCrawlTaskServiceChunkTest {
"task:" + task.getId() + ":owner:instance-a");
assertNull(result.getResultFileUrl());
verify(excelAssemblyService, never()).writeWorkbook(any(), any());
verify(excelAssemblyService, never()).appendWorkbook(any(), any(), any());
verify(excelAssemblyService, never()).replaceCountriesWorkbook(any(), any(), any());
verify(ossStorageService, never()).uploadResultFile(any(), eq(MODULE_TYPE));
}
backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskServiceRetentionTest.java
+18 -8
View File
@@ -150,7 +150,7 @@ class ShopDataCrawlTaskServiceRetentionTest {
service.processResultFileJob(job);
verify(excelAssemblyService).writeWorkbook(any(), eq(List.of(snapshot)));
verify(excelAssemblyService, never()).appendWorkbook(any(), any(), any());
verify(excelAssemblyService, never()).replaceCountriesWorkbook(any(), any(), any());
verify(dailyFileService).addMember(301L, TASK_ID, RESULT_ID);
assertEquals("result/new.xlsx", currentRow.getResultFileUrl());
assertEquals(1, currentRow.getRowCount());
@@ -163,7 +163,7 @@ class ShopDataCrawlTaskServiceRetentionTest {
}
@Test
void sameDaySuccessAppendsAndReplacesPreviousPointer() {
void sameDaySuccessReplacesCountryRowsAndPointsToNewWorkbook() {
FileResultEntity previous = result(200L, 100L, "result/old.xlsx");
previous.setUserId(null);
FileTaskEntity previousTask = task();
@@ -175,10 +175,11 @@ class ShopDataCrawlTaskServiceRetentionTest {
when(dailyFileService.containsResult(301L, RESULT_ID)).thenReturn(false);
when(ossStorageService.readObjectBytes("result/old.xlsx")).thenReturn(new byte[]{1, 2, 3});
when(ossStorageService.uploadResultFile(any(), eq(MODULE_TYPE))).thenReturn("result/new.xlsx");
when(excelAssemblyService.replaceCountriesWorkbook(any(), any(), any())).thenReturn(3);
service.processResultFileJob(job);
verify(excelAssemblyService).appendWorkbook(any(), any(), eq(List.of(snapshot)));
verify(excelAssemblyService).replaceCountriesWorkbook(any(), any(), eq(List.of(snapshot)));
verify(dailyFileService).update(daily);
verify(ossStorageService).deleteObject("result/old.xlsx");
assertNull(previous.getResultFileUrl());
@@ -216,8 +217,8 @@ class ShopDataCrawlTaskServiceRetentionTest {
}).when(ossStorageService).readObjectBytes("result/old.xlsx");
doAnswer(invocation -> {
assertFalse(transactionActive.get(), "workbook assembly must run outside the database transaction");
return null;
}).when(excelAssemblyService).appendWorkbook(any(), any(), any());
return 3;
}).when(excelAssemblyService).replaceCountriesWorkbook(any(), any(), any());
doAnswer(invocation -> {
assertFalse(transactionActive.get(), "workbook upload must run outside the database transaction");
return "result/new.xlsx";
@@ -243,7 +244,7 @@ class ShopDataCrawlTaskServiceRetentionTest {
service.processResultFileJob(job);
verify(excelAssemblyService, never()).writeWorkbook(any(), any());
verify(excelAssemblyService, never()).appendWorkbook(any(), any(), any());
verify(excelAssemblyService, never()).replaceCountriesWorkbook(any(), any(), any());
verify(ossStorageService, never()).uploadResultFile(any(), anyString());
assertEquals("result/current.xlsx", currentRow.getResultFileUrl());
assertEquals(3, currentRow.getRowCount());
@@ -261,7 +262,7 @@ class ShopDataCrawlTaskServiceRetentionTest {
verify(ossStorageService, never()).readObjectBytes(anyString());
verify(excelAssemblyService, never()).writeWorkbook(any(), any());
verify(excelAssemblyService, never()).appendWorkbook(any(), any(), any());
verify(excelAssemblyService, never()).replaceCountriesWorkbook(any(), any(), any());
verify(ossStorageService, never()).uploadResultFile(any(), anyString());
verify(dailyFileService).addMember(301L, TASK_ID, RESULT_ID);
assertEquals("result/current.xlsx", currentRow.getResultFileUrl());
@@ -308,7 +309,7 @@ class ShopDataCrawlTaskServiceRetentionTest {
service.processResultFileJob(job);
verify(excelAssemblyService, never()).writeWorkbook(any(), any());
verify(excelAssemblyService, never()).appendWorkbook(any(), any(), any());
verify(excelAssemblyService, never()).replaceCountriesWorkbook(any(), any(), any());
verify(ossStorageService, never()).uploadResultFile(any(), anyString());
assertNull(currentRow.getResultFileUrl());
}
@@ -322,13 +323,18 @@ class ShopDataCrawlTaskServiceRetentionTest {
when(fileResultMapper.selectList(any())).thenReturn(List.of(currentRow), List.of(previous, currentRow));
when(dailyFileService.findForUpdate(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(null);
when(dailyFileService.findOlder(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(List.of(yesterday));
when(ossStorageService.readObjectBytes("result/yesterday.xlsx")).thenReturn(new byte[]{1, 2, 3});
when(ossStorageService.uploadResultFile(any(), eq(MODULE_TYPE))).thenReturn("result/today.xlsx");
when(excelAssemblyService.replaceCountriesWorkbook(any(), any(), any())).thenReturn(5);
service.processResultFileJob(job);
verify(excelAssemblyService, never()).writeWorkbook(any(), any());
verify(excelAssemblyService).replaceCountriesWorkbook(any(), any(), eq(List.of(snapshot)));
verify(dailyFileService).deleteDailyFile(300L);
verify(ossStorageService).deleteObject("result/yesterday.xlsx");
assertEquals("result/today.xlsx", currentRow.getResultFileUrl());
assertEquals(5, currentRow.getRowCount());
assertNull(previous.getResultFileUrl());
}
@@ -340,6 +346,8 @@ class ShopDataCrawlTaskServiceRetentionTest {
when(fileResultMapper.selectList(any())).thenReturn(List.of(currentRow));
when(dailyFileService.findForUpdate(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(null);
when(dailyFileService.findOlder(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(List.of(yesterday));
when(ossStorageService.readObjectBytes("result/yesterday.xlsx")).thenReturn(new byte[]{1, 2, 3});
when(excelAssemblyService.replaceCountriesWorkbook(any(), any(), any())).thenReturn(4);
doThrow(new IllegalStateException("upload failed"))
.when(ossStorageService).uploadResultFile(any(), eq(MODULE_TYPE));
@@ -357,6 +365,8 @@ class ShopDataCrawlTaskServiceRetentionTest {
when(fileResultMapper.selectList(any())).thenReturn(List.of(currentRow));
when(dailyFileService.findForUpdate(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(null);
when(dailyFileService.findOlder(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(List.of(yesterday));
when(ossStorageService.readObjectBytes("result/yesterday.xlsx")).thenReturn(new byte[]{1, 2, 3});
when(excelAssemblyService.replaceCountriesWorkbook(any(), any(), any())).thenReturn(4);
when(ossStorageService.uploadResultFile(any(), eq(MODULE_TYPE))).thenReturn("result/new.xlsx");
AtomicInteger commitCount = new AtomicInteger();
doAnswer(invocation -> {