修复店铺任务数据覆盖bug
Build Backend JAR / build (push) Has been cancelled

This commit is contained in:
2026-08-29 00:16:47 +08:00
parent 79596a970c
commit 322905607a
5 changed files with 120 additions and 88 deletions
@@ -14,7 +14,9 @@ import org.apache.poi.ss.usermodel.Drawing;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.usermodel.XSSFDrawing;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.openxmlformats.schemas.drawingml.x2006.spreadsheetDrawing.CTTwoCellAnchor;
import org.springframework.core.io.ClassPathResource;
import org.springframework.stereotype.Service;
@@ -66,10 +68,12 @@ public class ShopDataCrawlExcelAssemblyService {
}
/**
* Appends the supplied task rows to an already assembled daily workbook.
* Existing rows, styles and drawings are deliberately left untouched.
* Replaces the supplied countries' rows inside an already assembled daily
* workbook. Sheets for countries that carry no new rows are left untouched
* (rows, styles and drawings preserved). Returns the total data-row count
* across all sheets after replacement.
*/
public void appendWorkbook(File baseXlsx, File outputXlsx, List<ShopDataCrawlResultItemVo> items) {
public int replaceCountriesWorkbook(File baseXlsx, File outputXlsx, List<ShopDataCrawlResultItemVo> items) {
if (baseXlsx == null || !baseXlsx.isFile()) {
throw new BusinessException("当天累计文件不存在");
}
@@ -84,14 +88,17 @@ public class ShopDataCrawlExcelAssemblyService {
for (int i = 0; i < COUNTRIES.size(); i++) {
List<ShopDataCrawlRowDto> rows = rowsByCountry.get(COUNTRIES.get(i));
if (rows != null && !rows.isEmpty()) {
appendSheet(workbook, workbook.getSheetAt(i), rows, imageCache, pictureIndexes);
Sheet sheet = workbook.getSheetAt(i);
clearSheetPictures(sheet, pictureIndexes);
writeSheet(workbook, sheet, rows, imageCache, pictureIndexes);
}
}
workbook.write(output);
return totalDataRows(workbook);
} catch (BusinessException ex) {
throw ex;
} catch (Exception ex) {
throw new BusinessException("店铺数据抓取累计 Excel 追加失败: " + ex.getMessage());
throw new BusinessException("店铺数据抓取累计 Excel 按国家替换失败: " + ex.getMessage());
}
}
@@ -155,54 +162,21 @@ public class ShopDataCrawlExcelAssemblyService {
sheet.removeRow(row);
}
}
clearSheetPictures(sheet, pictureIndexes);
int rowIndex = 1;
for (ShopDataCrawlRowDto value : rows == null ? List.<ShopDataCrawlRowDto>of() : rows) {
Row row = sheet.createRow(rowIndex++);
String[] values = {value.getDate(), value.getAsin(), "", value.getInventorySales(), value.getSalesRank(),
value.getPageViews(), value.getUnitsSold(), value.getPrice(), value.getRecommendedOffer(), value.getBrand()};
for (int column = 0; column < values.length; column++) {
Cell cell = row.createCell(column);
if (styles[column] != null) cell.setCellStyle(styles[column]);
cell.setCellValue(values[column] == null ? "" : values[column]);
}
if (!blank(value.getCommodityImage())) {
row.setHeightInPoints(IMAGE_ROW_HEIGHT_POINTS);
embedImage(workbook, sheet, row, value.getCommodityImage(), imageCache, pictureIndexes);
}
writeDataRow(workbook, sheet, row, value, styles, imageCache, pictureIndexes);
}
}
private void appendSheet(XSSFWorkbook workbook,
Sheet sheet,
List<ShopDataCrawlRowDto> rows,
Map<String, SimilarAsinImageEmbedder.ResizedImage> imageCache,
Map<String, Integer> pictureIndexes) {
Row header = sheet.getRow(0);
Row styleRow = sheet.getRow(1);
boolean currentTemplate = header != null && "商品图片".equals(cellText(header, IMAGE_COLUMN));
boolean templateHasBrand = header != null && "品牌".equals(cellText(header, BRAND_COLUMN));
CellStyle[] styles = new CellStyle[HEADERS.size()];
for (int column = 0; column < styles.length; column++) {
int sourceColumn = templateColumnForOutput(column, currentTemplate, templateHasBrand);
Cell cell = styleRow == null ? null : styleRow.getCell(sourceColumn);
styles[column] = cell == null ? null : cell.getCellStyle();
}
writeHeaders(sheet, currentTemplate, templateHasBrand);
sheet.setColumnWidth(IMAGE_COLUMN, IMAGE_COLUMN_WIDTH);
int rowIndex = Math.max(1, sheet.getLastRowNum() + 1);
for (ShopDataCrawlRowDto value : rows) {
Row row = sheet.createRow(rowIndex++);
writeRow(workbook, sheet, row, value, styles, imageCache, pictureIndexes);
}
}
private void writeRow(XSSFWorkbook workbook,
Sheet sheet,
Row row,
ShopDataCrawlRowDto value,
CellStyle[] styles,
Map<String, SimilarAsinImageEmbedder.ResizedImage> imageCache,
Map<String, Integer> pictureIndexes) {
private void writeDataRow(XSSFWorkbook workbook,
Sheet sheet,
Row row,
ShopDataCrawlRowDto value,
CellStyle[] styles,
Map<String, SimilarAsinImageEmbedder.ResizedImage> imageCache,
Map<String, Integer> pictureIndexes) {
String[] values = {value.getDate(), value.getAsin(), "", value.getInventorySales(), value.getSalesRank(),
value.getPageViews(), value.getUnitsSold(), value.getPrice(), value.getRecommendedOffer(), value.getBrand()};
for (int column = 0; column < values.length; column++) {
@@ -306,4 +280,25 @@ public class ShopDataCrawlExcelAssemblyService {
}
return result;
}
private void clearSheetPictures(Sheet sheet, Map<String, Integer> pictureIndexes) {
Drawing<?> patriarch = sheet.getDrawingPatriarch();
if (!(patriarch instanceof XSSFDrawing drawing)) {
return;
}
List<CTTwoCellAnchor> anchors = drawing.getCTDrawing().getTwoCellAnchorList();
for (int i = anchors.size() - 1; i >= 0; i--) {
if (anchors.get(i).getPic() != null) {
drawing.getCTDrawing().removeTwoCellAnchor(i);
}
}
}
private int totalDataRows(XSSFWorkbook workbook) {
int total = 0;
for (int i = 0; i < workbook.getNumberOfSheets(); i++) {
total += Math.max(0, workbook.getSheetAt(i).getLastRowNum());
}
return total;
}
}
@@ -1704,12 +1704,14 @@ public class ShopDataCrawlTaskService {
}
int addedRowCount = excelAssemblyService.countRows(List.of(snapshot));
ShopDataCrawlDailyFileEntity baseDailyFile = resolveBaseDailyFile(
preparation.dailyFile(), userId, shopKeyHash, businessDate);
DailyWorkbookArtifact artifact = assembleDailyWorkbook(
task, snapshot, preparation.dailyFile(), addedRowCount);
task, snapshot, baseDailyFile, addedRowCount);
try {
persistedResult = executeShortTransaction(() -> persistDailyAggregation(
task, row, userId, shopKey, shopKeyHash, businessDate,
preparation, artifact, addedRowCount));
preparation, artifact));
if (persistedResult.discardUploadedObject() && artifact.uploaded()) {
deleteObjectQuietly(artifact.objectKey());
}
@@ -1743,23 +1745,35 @@ public class ShopDataCrawlTaskService {
return new DailyAggregationPreparation(dailyFile, false);
}
private ShopDataCrawlDailyFileEntity resolveBaseDailyFile(ShopDataCrawlDailyFileEntity currentDailyFile,
Long userId,
String shopKeyHash,
LocalDate businessDate) {
if (currentDailyFile != null) {
return currentDailyFile;
}
List<ShopDataCrawlDailyFileEntity> older = dailyFileService.findOlder(userId, shopKeyHash, businessDate);
return older == null || older.isEmpty() ? null : older.get(0);
}
private DailyWorkbookArtifact assembleDailyWorkbook(FileTaskEntity task,
ShopDataCrawlResultItemVo snapshot,
ShopDataCrawlDailyFileEntity dailyFile,
ShopDataCrawlDailyFileEntity baseDailyFile,
int addedRowCount) {
String filename = dailyFile != null && !blank(dailyFile.getResultFilename())
? dailyFile.getResultFilename()
String filename = baseDailyFile != null && !blank(baseDailyFile.getResultFilename())
? baseDailyFile.getResultFilename()
: buildTaskWorkbookFilename(task);
String existingObjectKey = dailyFile == null ? null : dailyFile.getResultFileUrl();
String existingObjectKey = baseDailyFile == null ? null : baseDailyFile.getResultFileUrl();
// A result with no new rows only needs a new database membership. Reusing
// the canonical daily object avoids both a local copy and an OSS round trip.
if (dailyFile != null && addedRowCount == 0 && !blank(existingObjectKey)) {
if (baseDailyFile != null && addedRowCount == 0 && !blank(existingObjectKey)) {
return new DailyWorkbookArtifact(
existingObjectKey,
Math.max(0L, Objects.requireNonNullElse(dailyFile.getResultFileSize(), 0L)),
Math.max(0L, Objects.requireNonNullElse(baseDailyFile.getResultFileSize(), 0L)),
false,
filename);
filename,
Math.max(0, Objects.requireNonNullElse(baseDailyFile.getRowCount(), 0)));
}
File workRoot = FileUtil.mkdir(FileUtil.file(
@@ -1770,21 +1784,25 @@ public class ShopDataCrawlTaskService {
File baseXlsx = FileUtil.file(workRoot, "base.xlsx");
File outputXlsx = FileUtil.file(workRoot, filename);
try {
if (dailyFile != null && !blank(existingObjectKey)) {
if (baseDailyFile != null && !blank(existingObjectKey)) {
try {
Files.write(baseXlsx.toPath(), ossStorageService.readObjectBytes(existingObjectKey));
} catch (Exception ex) {
throw new BusinessException("读取当天累计文件失败: " + safeMessage(ex));
throw new BusinessException("读取累计文件失败: " + safeMessage(ex));
}
excelAssemblyService.appendWorkbook(baseXlsx, outputXlsx, List.of(snapshot));
} else {
excelAssemblyService.writeWorkbook(outputXlsx, List.of(snapshot));
int rowCount = excelAssemblyService.replaceCountriesWorkbook(baseXlsx, outputXlsx, List.of(snapshot));
String objectKey = ossStorageService.uploadResultFile(outputXlsx, MODULE_TYPE);
if (blank(objectKey)) {
throw new BusinessException("累计文件上传后未返回文件地址");
}
return new DailyWorkbookArtifact(objectKey, outputXlsx.length(), true, filename, rowCount);
}
excelAssemblyService.writeWorkbook(outputXlsx, List.of(snapshot));
String objectKey = ossStorageService.uploadResultFile(outputXlsx, MODULE_TYPE);
if (blank(objectKey)) {
throw new BusinessException("当天累计文件上传后未返回文件地址");
throw new BusinessException("累计文件上传后未返回文件地址");
}
return new DailyWorkbookArtifact(objectKey, outputXlsx.length(), true, filename);
return new DailyWorkbookArtifact(objectKey, outputXlsx.length(), true, filename, addedRowCount);
} finally {
FileUtil.del(baseXlsx);
FileUtil.del(outputXlsx);
@@ -1799,8 +1817,7 @@ public class ShopDataCrawlTaskService {
String shopKeyHash,
LocalDate businessDate,
DailyAggregationPreparation preparation,
DailyWorkbookArtifact artifact,
int addedRowCount) {
DailyWorkbookArtifact artifact) {
ShopDataCrawlDailyFileEntity dailyFile = dailyFileService.findForUpdate(
userId, shopKeyHash, businessDate);
if (handleExistingDailyMembership(row, dailyFile)) {
@@ -1826,9 +1843,7 @@ public class ShopDataCrawlTaskService {
row.setResultFileUrl(objectKey);
row.setResultFileSize(artifact.fileSize());
row.setResultContentType(CONTENT_TYPE_XLSX);
row.setRowCount(dailyFile == null
? addedRowCount
: Math.max(0, Objects.requireNonNullElse(dailyFile.getRowCount(), 0)) + addedRowCount);
row.setRowCount(artifact.rowCount());
fileResultMapper.updateById(row);
LocalDateTime now = dailyFileService.currentBusinessDateTime();
@@ -2140,7 +2155,12 @@ public class ShopDataCrawlTaskService {
private record DailyWorkbookArtifact(String objectKey,
long fileSize,
boolean uploaded,
String filename) {
String filename,
int rowCount) {
DailyWorkbookArtifact(String objectKey, long fileSize, boolean uploaded, String filename) {
this(objectKey, fileSize, uploaded, filename, 0);
}
}
private record DailyAggregationResult(List<String> obsoleteObjectKeys,
@@ -73,28 +73,35 @@ class ShopDataCrawlExcelAssemblyServiceTest {
}
@Test
void appendsOnlyTheSelectedCountryAndPreservesExistingPictures() throws Exception {
ShopDataCrawlRowDto firstRow = row("2026-07-25", "B012345678");
firstRow.setCommodityImage("https://m.media-amazon.com/images/I/first.jpg");
ShopDataCrawlRowDto secondRow = row("2026-07-26", "B099999999");
secondRow.setCommodityImage(null);
void replacesOnlyTheSelectedCountryAndPreservesOtherSheets() throws Exception {
ShopDataCrawlRowDto ukRow = row("2026-07-25", "B012345678");
ukRow.setCommodityImage("https://m.media-amazon.com/images/I/first.jpg");
ShopDataCrawlRowDto oldDeRow = row("2026-07-26", "B099999998");
oldDeRow.setCommodityImage("https://m.media-amazon.com/images/I/de.jpg");
ShopDataCrawlRowDto newDeRow = row("2026-07-27", "B099999999");
SimilarAsinImageEmbedder imageEmbedder = mock(SimilarAsinImageEmbedder.class);
when(imageEmbedder.fetchAndResizeForCache(firstRow.getCommodityImage()))
when(imageEmbedder.fetchAndResizeForCache(ukRow.getCommodityImage()))
.thenReturn(new SimilarAsinImageEmbedder.ResizedImage(jpegBytes(), 2, 2));
when(imageEmbedder.fetchAndResizeForCache(oldDeRow.getCommodityImage()))
.thenReturn(new SimilarAsinImageEmbedder.ResizedImage(jpegBytes(), 2, 2));
ShopDataCrawlExcelAssemblyService service = new ShopDataCrawlExcelAssemblyService(imageEmbedder);
File base = tempDir.resolve("base.xlsx").toFile();
File output = tempDir.resolve("daily.xlsx").toFile();
service.writeWorkbook(base, List.of(item("UK", firstRow)));
service.appendWorkbook(base, output, List.of(item("DE", secondRow)));
service.writeWorkbook(base, List.of(item("UK", ukRow), item("DE", oldDeRow)));
int total = service.replaceCountriesWorkbook(base, output, List.of(item("DE", newDeRow)));
try (XSSFWorkbook workbook = new XSSFWorkbook(new FileInputStream(output))) {
assertEquals("B012345678", workbook.getSheetAt(0).getRow(1).getCell(1).getStringCellValue());
assertEquals("B099999999", workbook.getSheetAt(1).getRow(1).getCell(1).getStringCellValue());
assertEquals(1, workbook.getAllPictures().size());
assertEquals(1, workbook.getSheetAt(0).getDrawingPatriarch().getShapes().size());
assertEquals(1, workbook.getSheetAt(0).getLastRowNum());
assertEquals(1, workbook.getSheetAt(1).getLastRowNum());
assertEquals(0, workbook.getSheetAt(2).getLastRowNum());
assertEquals(1, workbook.getSheetAt(0).getDrawingPatriarch().getShapes().size());
assertEquals(0, workbook.getSheetAt(1).getDrawingPatriarch().getShapes().size());
}
assertEquals(2, total);
}
private ShopDataCrawlResultItemVo item(String countryCode, ShopDataCrawlRowDto row) {
@@ -187,7 +187,7 @@ class ShopDataCrawlTaskServiceChunkTest {
"task:" + task.getId() + ":owner:instance-a");
assertNull(result.getResultFileUrl());
verify(excelAssemblyService, never()).writeWorkbook(any(), any());
verify(excelAssemblyService, never()).appendWorkbook(any(), any(), any());
verify(excelAssemblyService, never()).replaceCountriesWorkbook(any(), any(), any());
verify(ossStorageService, never()).uploadResultFile(any(), eq(MODULE_TYPE));
}
@@ -150,7 +150,7 @@ class ShopDataCrawlTaskServiceRetentionTest {
service.processResultFileJob(job);
verify(excelAssemblyService).writeWorkbook(any(), eq(List.of(snapshot)));
verify(excelAssemblyService, never()).appendWorkbook(any(), any(), any());
verify(excelAssemblyService, never()).replaceCountriesWorkbook(any(), any(), any());
verify(dailyFileService).addMember(301L, TASK_ID, RESULT_ID);
assertEquals("result/new.xlsx", currentRow.getResultFileUrl());
assertEquals(1, currentRow.getRowCount());
@@ -163,7 +163,7 @@ class ShopDataCrawlTaskServiceRetentionTest {
}
@Test
void sameDaySuccessAppendsAndReplacesPreviousPointer() {
void sameDaySuccessReplacesCountryRowsAndPointsToNewWorkbook() {
FileResultEntity previous = result(200L, 100L, "result/old.xlsx");
previous.setUserId(null);
FileTaskEntity previousTask = task();
@@ -175,10 +175,11 @@ class ShopDataCrawlTaskServiceRetentionTest {
when(dailyFileService.containsResult(301L, RESULT_ID)).thenReturn(false);
when(ossStorageService.readObjectBytes("result/old.xlsx")).thenReturn(new byte[]{1, 2, 3});
when(ossStorageService.uploadResultFile(any(), eq(MODULE_TYPE))).thenReturn("result/new.xlsx");
when(excelAssemblyService.replaceCountriesWorkbook(any(), any(), any())).thenReturn(3);
service.processResultFileJob(job);
verify(excelAssemblyService).appendWorkbook(any(), any(), eq(List.of(snapshot)));
verify(excelAssemblyService).replaceCountriesWorkbook(any(), any(), eq(List.of(snapshot)));
verify(dailyFileService).update(daily);
verify(ossStorageService).deleteObject("result/old.xlsx");
assertNull(previous.getResultFileUrl());
@@ -216,8 +217,8 @@ class ShopDataCrawlTaskServiceRetentionTest {
}).when(ossStorageService).readObjectBytes("result/old.xlsx");
doAnswer(invocation -> {
assertFalse(transactionActive.get(), "workbook assembly must run outside the database transaction");
return null;
}).when(excelAssemblyService).appendWorkbook(any(), any(), any());
return 3;
}).when(excelAssemblyService).replaceCountriesWorkbook(any(), any(), any());
doAnswer(invocation -> {
assertFalse(transactionActive.get(), "workbook upload must run outside the database transaction");
return "result/new.xlsx";
@@ -243,7 +244,7 @@ class ShopDataCrawlTaskServiceRetentionTest {
service.processResultFileJob(job);
verify(excelAssemblyService, never()).writeWorkbook(any(), any());
verify(excelAssemblyService, never()).appendWorkbook(any(), any(), any());
verify(excelAssemblyService, never()).replaceCountriesWorkbook(any(), any(), any());
verify(ossStorageService, never()).uploadResultFile(any(), anyString());
assertEquals("result/current.xlsx", currentRow.getResultFileUrl());
assertEquals(3, currentRow.getRowCount());
@@ -261,7 +262,7 @@ class ShopDataCrawlTaskServiceRetentionTest {
verify(ossStorageService, never()).readObjectBytes(anyString());
verify(excelAssemblyService, never()).writeWorkbook(any(), any());
verify(excelAssemblyService, never()).appendWorkbook(any(), any(), any());
verify(excelAssemblyService, never()).replaceCountriesWorkbook(any(), any(), any());
verify(ossStorageService, never()).uploadResultFile(any(), anyString());
verify(dailyFileService).addMember(301L, TASK_ID, RESULT_ID);
assertEquals("result/current.xlsx", currentRow.getResultFileUrl());
@@ -308,7 +309,7 @@ class ShopDataCrawlTaskServiceRetentionTest {
service.processResultFileJob(job);
verify(excelAssemblyService, never()).writeWorkbook(any(), any());
verify(excelAssemblyService, never()).appendWorkbook(any(), any(), any());
verify(excelAssemblyService, never()).replaceCountriesWorkbook(any(), any(), any());
verify(ossStorageService, never()).uploadResultFile(any(), anyString());
assertNull(currentRow.getResultFileUrl());
}
@@ -322,13 +323,18 @@ class ShopDataCrawlTaskServiceRetentionTest {
when(fileResultMapper.selectList(any())).thenReturn(List.of(currentRow), List.of(previous, currentRow));
when(dailyFileService.findForUpdate(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(null);
when(dailyFileService.findOlder(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(List.of(yesterday));
when(ossStorageService.readObjectBytes("result/yesterday.xlsx")).thenReturn(new byte[]{1, 2, 3});
when(ossStorageService.uploadResultFile(any(), eq(MODULE_TYPE))).thenReturn("result/today.xlsx");
when(excelAssemblyService.replaceCountriesWorkbook(any(), any(), any())).thenReturn(5);
service.processResultFileJob(job);
verify(excelAssemblyService, never()).writeWorkbook(any(), any());
verify(excelAssemblyService).replaceCountriesWorkbook(any(), any(), eq(List.of(snapshot)));
verify(dailyFileService).deleteDailyFile(300L);
verify(ossStorageService).deleteObject("result/yesterday.xlsx");
assertEquals("result/today.xlsx", currentRow.getResultFileUrl());
assertEquals(5, currentRow.getRowCount());
assertNull(previous.getResultFileUrl());
}
@@ -340,6 +346,8 @@ class ShopDataCrawlTaskServiceRetentionTest {
when(fileResultMapper.selectList(any())).thenReturn(List.of(currentRow));
when(dailyFileService.findForUpdate(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(null);
when(dailyFileService.findOlder(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(List.of(yesterday));
when(ossStorageService.readObjectBytes("result/yesterday.xlsx")).thenReturn(new byte[]{1, 2, 3});
when(excelAssemblyService.replaceCountriesWorkbook(any(), any(), any())).thenReturn(4);
doThrow(new IllegalStateException("upload failed"))
.when(ossStorageService).uploadResultFile(any(), eq(MODULE_TYPE));
@@ -357,6 +365,8 @@ class ShopDataCrawlTaskServiceRetentionTest {
when(fileResultMapper.selectList(any())).thenReturn(List.of(currentRow));
when(dailyFileService.findForUpdate(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(null);
when(dailyFileService.findOlder(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(List.of(yesterday));
when(ossStorageService.readObjectBytes("result/yesterday.xlsx")).thenReturn(new byte[]{1, 2, 3});
when(excelAssemblyService.replaceCountriesWorkbook(any(), any(), any())).thenReturn(4);
when(ossStorageService.uploadResultFile(any(), eq(MODULE_TYPE))).thenReturn("result/new.xlsx");
AtomicInteger commitCount = new AtomicInteger();
doAnswer(invocation -> {