修复店铺任务数据覆盖bug
Build Backend JAR / build (push) Has been cancelled

This commit is contained in:
2026-08-29 00:16:47 +08:00
parent 79596a970c
commit 322905607a
5 changed files with 120 additions and 88 deletions
@@ -14,7 +14,9 @@ import org.apache.poi.ss.usermodel.Drawing;
import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet; import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook; import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.usermodel.XSSFDrawing;
import org.apache.poi.xssf.usermodel.XSSFWorkbook; import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.openxmlformats.schemas.drawingml.x2006.spreadsheetDrawing.CTTwoCellAnchor;
import org.springframework.core.io.ClassPathResource; import org.springframework.core.io.ClassPathResource;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
@@ -66,10 +68,12 @@ public class ShopDataCrawlExcelAssemblyService {
} }
/** /**
* Appends the supplied task rows to an already assembled daily workbook. * Replaces the supplied countries' rows inside an already assembled daily
* Existing rows, styles and drawings are deliberately left untouched. * workbook. Sheets for countries that carry no new rows are left untouched
* (rows, styles and drawings preserved). Returns the total data-row count
* across all sheets after replacement.
*/ */
public void appendWorkbook(File baseXlsx, File outputXlsx, List<ShopDataCrawlResultItemVo> items) { public int replaceCountriesWorkbook(File baseXlsx, File outputXlsx, List<ShopDataCrawlResultItemVo> items) {
if (baseXlsx == null || !baseXlsx.isFile()) { if (baseXlsx == null || !baseXlsx.isFile()) {
throw new BusinessException("当天累计文件不存在"); throw new BusinessException("当天累计文件不存在");
} }
@@ -84,14 +88,17 @@ public class ShopDataCrawlExcelAssemblyService {
for (int i = 0; i < COUNTRIES.size(); i++) { for (int i = 0; i < COUNTRIES.size(); i++) {
List<ShopDataCrawlRowDto> rows = rowsByCountry.get(COUNTRIES.get(i)); List<ShopDataCrawlRowDto> rows = rowsByCountry.get(COUNTRIES.get(i));
if (rows != null && !rows.isEmpty()) { if (rows != null && !rows.isEmpty()) {
appendSheet(workbook, workbook.getSheetAt(i), rows, imageCache, pictureIndexes); Sheet sheet = workbook.getSheetAt(i);
clearSheetPictures(sheet, pictureIndexes);
writeSheet(workbook, sheet, rows, imageCache, pictureIndexes);
} }
} }
workbook.write(output); workbook.write(output);
return totalDataRows(workbook);
} catch (BusinessException ex) { } catch (BusinessException ex) {
throw ex; throw ex;
} catch (Exception ex) { } catch (Exception ex) {
throw new BusinessException("店铺数据抓取累计 Excel 追加失败: " + ex.getMessage()); throw new BusinessException("店铺数据抓取累计 Excel 按国家替换失败: " + ex.getMessage());
} }
} }
@@ -155,48 +162,15 @@ public class ShopDataCrawlExcelAssemblyService {
sheet.removeRow(row); sheet.removeRow(row);
} }
} }
clearSheetPictures(sheet, pictureIndexes);
int rowIndex = 1; int rowIndex = 1;
for (ShopDataCrawlRowDto value : rows == null ? List.<ShopDataCrawlRowDto>of() : rows) { for (ShopDataCrawlRowDto value : rows == null ? List.<ShopDataCrawlRowDto>of() : rows) {
Row row = sheet.createRow(rowIndex++); Row row = sheet.createRow(rowIndex++);
String[] values = {value.getDate(), value.getAsin(), "", value.getInventorySales(), value.getSalesRank(), writeDataRow(workbook, sheet, row, value, styles, imageCache, pictureIndexes);
value.getPageViews(), value.getUnitsSold(), value.getPrice(), value.getRecommendedOffer(), value.getBrand()};
for (int column = 0; column < values.length; column++) {
Cell cell = row.createCell(column);
if (styles[column] != null) cell.setCellStyle(styles[column]);
cell.setCellValue(values[column] == null ? "" : values[column]);
}
if (!blank(value.getCommodityImage())) {
row.setHeightInPoints(IMAGE_ROW_HEIGHT_POINTS);
embedImage(workbook, sheet, row, value.getCommodityImage(), imageCache, pictureIndexes);
}
}
}
private void appendSheet(XSSFWorkbook workbook,
Sheet sheet,
List<ShopDataCrawlRowDto> rows,
Map<String, SimilarAsinImageEmbedder.ResizedImage> imageCache,
Map<String, Integer> pictureIndexes) {
Row header = sheet.getRow(0);
Row styleRow = sheet.getRow(1);
boolean currentTemplate = header != null && "商品图片".equals(cellText(header, IMAGE_COLUMN));
boolean templateHasBrand = header != null && "品牌".equals(cellText(header, BRAND_COLUMN));
CellStyle[] styles = new CellStyle[HEADERS.size()];
for (int column = 0; column < styles.length; column++) {
int sourceColumn = templateColumnForOutput(column, currentTemplate, templateHasBrand);
Cell cell = styleRow == null ? null : styleRow.getCell(sourceColumn);
styles[column] = cell == null ? null : cell.getCellStyle();
}
writeHeaders(sheet, currentTemplate, templateHasBrand);
sheet.setColumnWidth(IMAGE_COLUMN, IMAGE_COLUMN_WIDTH);
int rowIndex = Math.max(1, sheet.getLastRowNum() + 1);
for (ShopDataCrawlRowDto value : rows) {
Row row = sheet.createRow(rowIndex++);
writeRow(workbook, sheet, row, value, styles, imageCache, pictureIndexes);
} }
} }
private void writeRow(XSSFWorkbook workbook, private void writeDataRow(XSSFWorkbook workbook,
Sheet sheet, Sheet sheet,
Row row, Row row,
ShopDataCrawlRowDto value, ShopDataCrawlRowDto value,
@@ -306,4 +280,25 @@ public class ShopDataCrawlExcelAssemblyService {
} }
return result; return result;
} }
private void clearSheetPictures(Sheet sheet, Map<String, Integer> pictureIndexes) {
Drawing<?> patriarch = sheet.getDrawingPatriarch();
if (!(patriarch instanceof XSSFDrawing drawing)) {
return;
}
List<CTTwoCellAnchor> anchors = drawing.getCTDrawing().getTwoCellAnchorList();
for (int i = anchors.size() - 1; i >= 0; i--) {
if (anchors.get(i).getPic() != null) {
drawing.getCTDrawing().removeTwoCellAnchor(i);
}
}
}
private int totalDataRows(XSSFWorkbook workbook) {
int total = 0;
for (int i = 0; i < workbook.getNumberOfSheets(); i++) {
total += Math.max(0, workbook.getSheetAt(i).getLastRowNum());
}
return total;
}
} }
@@ -1704,12 +1704,14 @@ public class ShopDataCrawlTaskService {
} }
int addedRowCount = excelAssemblyService.countRows(List.of(snapshot)); int addedRowCount = excelAssemblyService.countRows(List.of(snapshot));
ShopDataCrawlDailyFileEntity baseDailyFile = resolveBaseDailyFile(
preparation.dailyFile(), userId, shopKeyHash, businessDate);
DailyWorkbookArtifact artifact = assembleDailyWorkbook( DailyWorkbookArtifact artifact = assembleDailyWorkbook(
task, snapshot, preparation.dailyFile(), addedRowCount); task, snapshot, baseDailyFile, addedRowCount);
try { try {
persistedResult = executeShortTransaction(() -> persistDailyAggregation( persistedResult = executeShortTransaction(() -> persistDailyAggregation(
task, row, userId, shopKey, shopKeyHash, businessDate, task, row, userId, shopKey, shopKeyHash, businessDate,
preparation, artifact, addedRowCount)); preparation, artifact));
if (persistedResult.discardUploadedObject() && artifact.uploaded()) { if (persistedResult.discardUploadedObject() && artifact.uploaded()) {
deleteObjectQuietly(artifact.objectKey()); deleteObjectQuietly(artifact.objectKey());
} }
@@ -1743,23 +1745,35 @@ public class ShopDataCrawlTaskService {
return new DailyAggregationPreparation(dailyFile, false); return new DailyAggregationPreparation(dailyFile, false);
} }
private ShopDataCrawlDailyFileEntity resolveBaseDailyFile(ShopDataCrawlDailyFileEntity currentDailyFile,
Long userId,
String shopKeyHash,
LocalDate businessDate) {
if (currentDailyFile != null) {
return currentDailyFile;
}
List<ShopDataCrawlDailyFileEntity> older = dailyFileService.findOlder(userId, shopKeyHash, businessDate);
return older == null || older.isEmpty() ? null : older.get(0);
}
private DailyWorkbookArtifact assembleDailyWorkbook(FileTaskEntity task, private DailyWorkbookArtifact assembleDailyWorkbook(FileTaskEntity task,
ShopDataCrawlResultItemVo snapshot, ShopDataCrawlResultItemVo snapshot,
ShopDataCrawlDailyFileEntity dailyFile, ShopDataCrawlDailyFileEntity baseDailyFile,
int addedRowCount) { int addedRowCount) {
String filename = dailyFile != null && !blank(dailyFile.getResultFilename()) String filename = baseDailyFile != null && !blank(baseDailyFile.getResultFilename())
? dailyFile.getResultFilename() ? baseDailyFile.getResultFilename()
: buildTaskWorkbookFilename(task); : buildTaskWorkbookFilename(task);
String existingObjectKey = dailyFile == null ? null : dailyFile.getResultFileUrl(); String existingObjectKey = baseDailyFile == null ? null : baseDailyFile.getResultFileUrl();
// A result with no new rows only needs a new database membership. Reusing // A result with no new rows only needs a new database membership. Reusing
// the canonical daily object avoids both a local copy and an OSS round trip. // the canonical daily object avoids both a local copy and an OSS round trip.
if (dailyFile != null && addedRowCount == 0 && !blank(existingObjectKey)) { if (baseDailyFile != null && addedRowCount == 0 && !blank(existingObjectKey)) {
return new DailyWorkbookArtifact( return new DailyWorkbookArtifact(
existingObjectKey, existingObjectKey,
Math.max(0L, Objects.requireNonNullElse(dailyFile.getResultFileSize(), 0L)), Math.max(0L, Objects.requireNonNullElse(baseDailyFile.getResultFileSize(), 0L)),
false, false,
filename); filename,
Math.max(0, Objects.requireNonNullElse(baseDailyFile.getRowCount(), 0)));
} }
File workRoot = FileUtil.mkdir(FileUtil.file( File workRoot = FileUtil.mkdir(FileUtil.file(
@@ -1770,21 +1784,25 @@ public class ShopDataCrawlTaskService {
File baseXlsx = FileUtil.file(workRoot, "base.xlsx"); File baseXlsx = FileUtil.file(workRoot, "base.xlsx");
File outputXlsx = FileUtil.file(workRoot, filename); File outputXlsx = FileUtil.file(workRoot, filename);
try { try {
if (dailyFile != null && !blank(existingObjectKey)) { if (baseDailyFile != null && !blank(existingObjectKey)) {
try { try {
Files.write(baseXlsx.toPath(), ossStorageService.readObjectBytes(existingObjectKey)); Files.write(baseXlsx.toPath(), ossStorageService.readObjectBytes(existingObjectKey));
} catch (Exception ex) { } catch (Exception ex) {
throw new BusinessException("读取当天累计文件失败: " + safeMessage(ex)); throw new BusinessException("读取累计文件失败: " + safeMessage(ex));
}
excelAssemblyService.appendWorkbook(baseXlsx, outputXlsx, List.of(snapshot));
} else {
excelAssemblyService.writeWorkbook(outputXlsx, List.of(snapshot));
} }
int rowCount = excelAssemblyService.replaceCountriesWorkbook(baseXlsx, outputXlsx, List.of(snapshot));
String objectKey = ossStorageService.uploadResultFile(outputXlsx, MODULE_TYPE); String objectKey = ossStorageService.uploadResultFile(outputXlsx, MODULE_TYPE);
if (blank(objectKey)) { if (blank(objectKey)) {
throw new BusinessException("当天累计文件上传后未返回文件地址"); throw new BusinessException("累计文件上传后未返回文件地址");
} }
return new DailyWorkbookArtifact(objectKey, outputXlsx.length(), true, filename); return new DailyWorkbookArtifact(objectKey, outputXlsx.length(), true, filename, rowCount);
}
excelAssemblyService.writeWorkbook(outputXlsx, List.of(snapshot));
String objectKey = ossStorageService.uploadResultFile(outputXlsx, MODULE_TYPE);
if (blank(objectKey)) {
throw new BusinessException("累计文件上传后未返回文件地址");
}
return new DailyWorkbookArtifact(objectKey, outputXlsx.length(), true, filename, addedRowCount);
} finally { } finally {
FileUtil.del(baseXlsx); FileUtil.del(baseXlsx);
FileUtil.del(outputXlsx); FileUtil.del(outputXlsx);
@@ -1799,8 +1817,7 @@ public class ShopDataCrawlTaskService {
String shopKeyHash, String shopKeyHash,
LocalDate businessDate, LocalDate businessDate,
DailyAggregationPreparation preparation, DailyAggregationPreparation preparation,
DailyWorkbookArtifact artifact, DailyWorkbookArtifact artifact) {
int addedRowCount) {
ShopDataCrawlDailyFileEntity dailyFile = dailyFileService.findForUpdate( ShopDataCrawlDailyFileEntity dailyFile = dailyFileService.findForUpdate(
userId, shopKeyHash, businessDate); userId, shopKeyHash, businessDate);
if (handleExistingDailyMembership(row, dailyFile)) { if (handleExistingDailyMembership(row, dailyFile)) {
@@ -1826,9 +1843,7 @@ public class ShopDataCrawlTaskService {
row.setResultFileUrl(objectKey); row.setResultFileUrl(objectKey);
row.setResultFileSize(artifact.fileSize()); row.setResultFileSize(artifact.fileSize());
row.setResultContentType(CONTENT_TYPE_XLSX); row.setResultContentType(CONTENT_TYPE_XLSX);
row.setRowCount(dailyFile == null row.setRowCount(artifact.rowCount());
? addedRowCount
: Math.max(0, Objects.requireNonNullElse(dailyFile.getRowCount(), 0)) + addedRowCount);
fileResultMapper.updateById(row); fileResultMapper.updateById(row);
LocalDateTime now = dailyFileService.currentBusinessDateTime(); LocalDateTime now = dailyFileService.currentBusinessDateTime();
@@ -2140,7 +2155,12 @@ public class ShopDataCrawlTaskService {
private record DailyWorkbookArtifact(String objectKey, private record DailyWorkbookArtifact(String objectKey,
long fileSize, long fileSize,
boolean uploaded, boolean uploaded,
String filename) { String filename,
int rowCount) {
DailyWorkbookArtifact(String objectKey, long fileSize, boolean uploaded, String filename) {
this(objectKey, fileSize, uploaded, filename, 0);
}
} }
private record DailyAggregationResult(List<String> obsoleteObjectKeys, private record DailyAggregationResult(List<String> obsoleteObjectKeys,
@@ -73,28 +73,35 @@ class ShopDataCrawlExcelAssemblyServiceTest {
} }
@Test @Test
void appendsOnlyTheSelectedCountryAndPreservesExistingPictures() throws Exception { void replacesOnlyTheSelectedCountryAndPreservesOtherSheets() throws Exception {
ShopDataCrawlRowDto firstRow = row("2026-07-25", "B012345678"); ShopDataCrawlRowDto ukRow = row("2026-07-25", "B012345678");
firstRow.setCommodityImage("https://m.media-amazon.com/images/I/first.jpg"); ukRow.setCommodityImage("https://m.media-amazon.com/images/I/first.jpg");
ShopDataCrawlRowDto secondRow = row("2026-07-26", "B099999999"); ShopDataCrawlRowDto oldDeRow = row("2026-07-26", "B099999998");
secondRow.setCommodityImage(null); oldDeRow.setCommodityImage("https://m.media-amazon.com/images/I/de.jpg");
ShopDataCrawlRowDto newDeRow = row("2026-07-27", "B099999999");
SimilarAsinImageEmbedder imageEmbedder = mock(SimilarAsinImageEmbedder.class); SimilarAsinImageEmbedder imageEmbedder = mock(SimilarAsinImageEmbedder.class);
when(imageEmbedder.fetchAndResizeForCache(firstRow.getCommodityImage())) when(imageEmbedder.fetchAndResizeForCache(ukRow.getCommodityImage()))
.thenReturn(new SimilarAsinImageEmbedder.ResizedImage(jpegBytes(), 2, 2));
when(imageEmbedder.fetchAndResizeForCache(oldDeRow.getCommodityImage()))
.thenReturn(new SimilarAsinImageEmbedder.ResizedImage(jpegBytes(), 2, 2)); .thenReturn(new SimilarAsinImageEmbedder.ResizedImage(jpegBytes(), 2, 2));
ShopDataCrawlExcelAssemblyService service = new ShopDataCrawlExcelAssemblyService(imageEmbedder); ShopDataCrawlExcelAssemblyService service = new ShopDataCrawlExcelAssemblyService(imageEmbedder);
File base = tempDir.resolve("base.xlsx").toFile(); File base = tempDir.resolve("base.xlsx").toFile();
File output = tempDir.resolve("daily.xlsx").toFile(); File output = tempDir.resolve("daily.xlsx").toFile();
service.writeWorkbook(base, List.of(item("UK", firstRow))); service.writeWorkbook(base, List.of(item("UK", ukRow), item("DE", oldDeRow)));
service.appendWorkbook(base, output, List.of(item("DE", secondRow)));
int total = service.replaceCountriesWorkbook(base, output, List.of(item("DE", newDeRow)));
try (XSSFWorkbook workbook = new XSSFWorkbook(new FileInputStream(output))) { try (XSSFWorkbook workbook = new XSSFWorkbook(new FileInputStream(output))) {
assertEquals("B012345678", workbook.getSheetAt(0).getRow(1).getCell(1).getStringCellValue()); assertEquals("B012345678", workbook.getSheetAt(0).getRow(1).getCell(1).getStringCellValue());
assertEquals("B099999999", workbook.getSheetAt(1).getRow(1).getCell(1).getStringCellValue()); assertEquals("B099999999", workbook.getSheetAt(1).getRow(1).getCell(1).getStringCellValue());
assertEquals(1, workbook.getAllPictures().size()); assertEquals(1, workbook.getSheetAt(0).getLastRowNum());
assertEquals(1, workbook.getSheetAt(0).getDrawingPatriarch().getShapes().size()); assertEquals(1, workbook.getSheetAt(1).getLastRowNum());
assertEquals(0, workbook.getSheetAt(2).getLastRowNum()); assertEquals(0, workbook.getSheetAt(2).getLastRowNum());
assertEquals(1, workbook.getSheetAt(0).getDrawingPatriarch().getShapes().size());
assertEquals(0, workbook.getSheetAt(1).getDrawingPatriarch().getShapes().size());
} }
assertEquals(2, total);
} }
private ShopDataCrawlResultItemVo item(String countryCode, ShopDataCrawlRowDto row) { private ShopDataCrawlResultItemVo item(String countryCode, ShopDataCrawlRowDto row) {
@@ -187,7 +187,7 @@ class ShopDataCrawlTaskServiceChunkTest {
"task:" + task.getId() + ":owner:instance-a"); "task:" + task.getId() + ":owner:instance-a");
assertNull(result.getResultFileUrl()); assertNull(result.getResultFileUrl());
verify(excelAssemblyService, never()).writeWorkbook(any(), any()); verify(excelAssemblyService, never()).writeWorkbook(any(), any());
verify(excelAssemblyService, never()).appendWorkbook(any(), any(), any()); verify(excelAssemblyService, never()).replaceCountriesWorkbook(any(), any(), any());
verify(ossStorageService, never()).uploadResultFile(any(), eq(MODULE_TYPE)); verify(ossStorageService, never()).uploadResultFile(any(), eq(MODULE_TYPE));
} }
@@ -150,7 +150,7 @@ class ShopDataCrawlTaskServiceRetentionTest {
service.processResultFileJob(job); service.processResultFileJob(job);
verify(excelAssemblyService).writeWorkbook(any(), eq(List.of(snapshot))); verify(excelAssemblyService).writeWorkbook(any(), eq(List.of(snapshot)));
verify(excelAssemblyService, never()).appendWorkbook(any(), any(), any()); verify(excelAssemblyService, never()).replaceCountriesWorkbook(any(), any(), any());
verify(dailyFileService).addMember(301L, TASK_ID, RESULT_ID); verify(dailyFileService).addMember(301L, TASK_ID, RESULT_ID);
assertEquals("result/new.xlsx", currentRow.getResultFileUrl()); assertEquals("result/new.xlsx", currentRow.getResultFileUrl());
assertEquals(1, currentRow.getRowCount()); assertEquals(1, currentRow.getRowCount());
@@ -163,7 +163,7 @@ class ShopDataCrawlTaskServiceRetentionTest {
} }
@Test @Test
void sameDaySuccessAppendsAndReplacesPreviousPointer() { void sameDaySuccessReplacesCountryRowsAndPointsToNewWorkbook() {
FileResultEntity previous = result(200L, 100L, "result/old.xlsx"); FileResultEntity previous = result(200L, 100L, "result/old.xlsx");
previous.setUserId(null); previous.setUserId(null);
FileTaskEntity previousTask = task(); FileTaskEntity previousTask = task();
@@ -175,10 +175,11 @@ class ShopDataCrawlTaskServiceRetentionTest {
when(dailyFileService.containsResult(301L, RESULT_ID)).thenReturn(false); when(dailyFileService.containsResult(301L, RESULT_ID)).thenReturn(false);
when(ossStorageService.readObjectBytes("result/old.xlsx")).thenReturn(new byte[]{1, 2, 3}); when(ossStorageService.readObjectBytes("result/old.xlsx")).thenReturn(new byte[]{1, 2, 3});
when(ossStorageService.uploadResultFile(any(), eq(MODULE_TYPE))).thenReturn("result/new.xlsx"); when(ossStorageService.uploadResultFile(any(), eq(MODULE_TYPE))).thenReturn("result/new.xlsx");
when(excelAssemblyService.replaceCountriesWorkbook(any(), any(), any())).thenReturn(3);
service.processResultFileJob(job); service.processResultFileJob(job);
verify(excelAssemblyService).appendWorkbook(any(), any(), eq(List.of(snapshot))); verify(excelAssemblyService).replaceCountriesWorkbook(any(), any(), eq(List.of(snapshot)));
verify(dailyFileService).update(daily); verify(dailyFileService).update(daily);
verify(ossStorageService).deleteObject("result/old.xlsx"); verify(ossStorageService).deleteObject("result/old.xlsx");
assertNull(previous.getResultFileUrl()); assertNull(previous.getResultFileUrl());
@@ -216,8 +217,8 @@ class ShopDataCrawlTaskServiceRetentionTest {
}).when(ossStorageService).readObjectBytes("result/old.xlsx"); }).when(ossStorageService).readObjectBytes("result/old.xlsx");
doAnswer(invocation -> { doAnswer(invocation -> {
assertFalse(transactionActive.get(), "workbook assembly must run outside the database transaction"); assertFalse(transactionActive.get(), "workbook assembly must run outside the database transaction");
return null; return 3;
}).when(excelAssemblyService).appendWorkbook(any(), any(), any()); }).when(excelAssemblyService).replaceCountriesWorkbook(any(), any(), any());
doAnswer(invocation -> { doAnswer(invocation -> {
assertFalse(transactionActive.get(), "workbook upload must run outside the database transaction"); assertFalse(transactionActive.get(), "workbook upload must run outside the database transaction");
return "result/new.xlsx"; return "result/new.xlsx";
@@ -243,7 +244,7 @@ class ShopDataCrawlTaskServiceRetentionTest {
service.processResultFileJob(job); service.processResultFileJob(job);
verify(excelAssemblyService, never()).writeWorkbook(any(), any()); verify(excelAssemblyService, never()).writeWorkbook(any(), any());
verify(excelAssemblyService, never()).appendWorkbook(any(), any(), any()); verify(excelAssemblyService, never()).replaceCountriesWorkbook(any(), any(), any());
verify(ossStorageService, never()).uploadResultFile(any(), anyString()); verify(ossStorageService, never()).uploadResultFile(any(), anyString());
assertEquals("result/current.xlsx", currentRow.getResultFileUrl()); assertEquals("result/current.xlsx", currentRow.getResultFileUrl());
assertEquals(3, currentRow.getRowCount()); assertEquals(3, currentRow.getRowCount());
@@ -261,7 +262,7 @@ class ShopDataCrawlTaskServiceRetentionTest {
verify(ossStorageService, never()).readObjectBytes(anyString()); verify(ossStorageService, never()).readObjectBytes(anyString());
verify(excelAssemblyService, never()).writeWorkbook(any(), any()); verify(excelAssemblyService, never()).writeWorkbook(any(), any());
verify(excelAssemblyService, never()).appendWorkbook(any(), any(), any()); verify(excelAssemblyService, never()).replaceCountriesWorkbook(any(), any(), any());
verify(ossStorageService, never()).uploadResultFile(any(), anyString()); verify(ossStorageService, never()).uploadResultFile(any(), anyString());
verify(dailyFileService).addMember(301L, TASK_ID, RESULT_ID); verify(dailyFileService).addMember(301L, TASK_ID, RESULT_ID);
assertEquals("result/current.xlsx", currentRow.getResultFileUrl()); assertEquals("result/current.xlsx", currentRow.getResultFileUrl());
@@ -308,7 +309,7 @@ class ShopDataCrawlTaskServiceRetentionTest {
service.processResultFileJob(job); service.processResultFileJob(job);
verify(excelAssemblyService, never()).writeWorkbook(any(), any()); verify(excelAssemblyService, never()).writeWorkbook(any(), any());
verify(excelAssemblyService, never()).appendWorkbook(any(), any(), any()); verify(excelAssemblyService, never()).replaceCountriesWorkbook(any(), any(), any());
verify(ossStorageService, never()).uploadResultFile(any(), anyString()); verify(ossStorageService, never()).uploadResultFile(any(), anyString());
assertNull(currentRow.getResultFileUrl()); assertNull(currentRow.getResultFileUrl());
} }
@@ -322,13 +323,18 @@ class ShopDataCrawlTaskServiceRetentionTest {
when(fileResultMapper.selectList(any())).thenReturn(List.of(currentRow), List.of(previous, currentRow)); when(fileResultMapper.selectList(any())).thenReturn(List.of(currentRow), List.of(previous, currentRow));
when(dailyFileService.findForUpdate(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(null); when(dailyFileService.findForUpdate(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(null);
when(dailyFileService.findOlder(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(List.of(yesterday)); when(dailyFileService.findOlder(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(List.of(yesterday));
when(ossStorageService.readObjectBytes("result/yesterday.xlsx")).thenReturn(new byte[]{1, 2, 3});
when(ossStorageService.uploadResultFile(any(), eq(MODULE_TYPE))).thenReturn("result/today.xlsx"); when(ossStorageService.uploadResultFile(any(), eq(MODULE_TYPE))).thenReturn("result/today.xlsx");
when(excelAssemblyService.replaceCountriesWorkbook(any(), any(), any())).thenReturn(5);
service.processResultFileJob(job); service.processResultFileJob(job);
verify(excelAssemblyService, never()).writeWorkbook(any(), any());
verify(excelAssemblyService).replaceCountriesWorkbook(any(), any(), eq(List.of(snapshot)));
verify(dailyFileService).deleteDailyFile(300L); verify(dailyFileService).deleteDailyFile(300L);
verify(ossStorageService).deleteObject("result/yesterday.xlsx"); verify(ossStorageService).deleteObject("result/yesterday.xlsx");
assertEquals("result/today.xlsx", currentRow.getResultFileUrl()); assertEquals("result/today.xlsx", currentRow.getResultFileUrl());
assertEquals(5, currentRow.getRowCount());
assertNull(previous.getResultFileUrl()); assertNull(previous.getResultFileUrl());
} }
@@ -340,6 +346,8 @@ class ShopDataCrawlTaskServiceRetentionTest {
when(fileResultMapper.selectList(any())).thenReturn(List.of(currentRow)); when(fileResultMapper.selectList(any())).thenReturn(List.of(currentRow));
when(dailyFileService.findForUpdate(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(null); when(dailyFileService.findForUpdate(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(null);
when(dailyFileService.findOlder(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(List.of(yesterday)); when(dailyFileService.findOlder(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(List.of(yesterday));
when(ossStorageService.readObjectBytes("result/yesterday.xlsx")).thenReturn(new byte[]{1, 2, 3});
when(excelAssemblyService.replaceCountriesWorkbook(any(), any(), any())).thenReturn(4);
doThrow(new IllegalStateException("upload failed")) doThrow(new IllegalStateException("upload failed"))
.when(ossStorageService).uploadResultFile(any(), eq(MODULE_TYPE)); .when(ossStorageService).uploadResultFile(any(), eq(MODULE_TYPE));
@@ -357,6 +365,8 @@ class ShopDataCrawlTaskServiceRetentionTest {
when(fileResultMapper.selectList(any())).thenReturn(List.of(currentRow)); when(fileResultMapper.selectList(any())).thenReturn(List.of(currentRow));
when(dailyFileService.findForUpdate(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(null); when(dailyFileService.findForUpdate(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(null);
when(dailyFileService.findOlder(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(List.of(yesterday)); when(dailyFileService.findOlder(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(List.of(yesterday));
when(ossStorageService.readObjectBytes("result/yesterday.xlsx")).thenReturn(new byte[]{1, 2, 3});
when(excelAssemblyService.replaceCountriesWorkbook(any(), any(), any())).thenReturn(4);
when(ossStorageService.uploadResultFile(any(), eq(MODULE_TYPE))).thenReturn("result/new.xlsx"); when(ossStorageService.uploadResultFile(any(), eq(MODULE_TYPE))).thenReturn("result/new.xlsx");
AtomicInteger commitCount = new AtomicInteger(); AtomicInteger commitCount = new AtomicInteger();
doAnswer(invocation -> { doAnswer(invocation -> {