+42
-47
@@ -14,7 +14,9 @@ import org.apache.poi.ss.usermodel.Drawing;
|
|||||||
import org.apache.poi.ss.usermodel.Row;
|
import org.apache.poi.ss.usermodel.Row;
|
||||||
import org.apache.poi.ss.usermodel.Sheet;
|
import org.apache.poi.ss.usermodel.Sheet;
|
||||||
import org.apache.poi.ss.usermodel.Workbook;
|
import org.apache.poi.ss.usermodel.Workbook;
|
||||||
|
import org.apache.poi.xssf.usermodel.XSSFDrawing;
|
||||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||||
|
import org.openxmlformats.schemas.drawingml.x2006.spreadsheetDrawing.CTTwoCellAnchor;
|
||||||
import org.springframework.core.io.ClassPathResource;
|
import org.springframework.core.io.ClassPathResource;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
@@ -66,10 +68,12 @@ public class ShopDataCrawlExcelAssemblyService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Appends the supplied task rows to an already assembled daily workbook.
|
* Replaces the supplied countries' rows inside an already assembled daily
|
||||||
* Existing rows, styles and drawings are deliberately left untouched.
|
* workbook. Sheets for countries that carry no new rows are left untouched
|
||||||
|
* (rows, styles and drawings preserved). Returns the total data-row count
|
||||||
|
* across all sheets after replacement.
|
||||||
*/
|
*/
|
||||||
public void appendWorkbook(File baseXlsx, File outputXlsx, List<ShopDataCrawlResultItemVo> items) {
|
public int replaceCountriesWorkbook(File baseXlsx, File outputXlsx, List<ShopDataCrawlResultItemVo> items) {
|
||||||
if (baseXlsx == null || !baseXlsx.isFile()) {
|
if (baseXlsx == null || !baseXlsx.isFile()) {
|
||||||
throw new BusinessException("当天累计文件不存在");
|
throw new BusinessException("当天累计文件不存在");
|
||||||
}
|
}
|
||||||
@@ -84,14 +88,17 @@ public class ShopDataCrawlExcelAssemblyService {
|
|||||||
for (int i = 0; i < COUNTRIES.size(); i++) {
|
for (int i = 0; i < COUNTRIES.size(); i++) {
|
||||||
List<ShopDataCrawlRowDto> rows = rowsByCountry.get(COUNTRIES.get(i));
|
List<ShopDataCrawlRowDto> rows = rowsByCountry.get(COUNTRIES.get(i));
|
||||||
if (rows != null && !rows.isEmpty()) {
|
if (rows != null && !rows.isEmpty()) {
|
||||||
appendSheet(workbook, workbook.getSheetAt(i), rows, imageCache, pictureIndexes);
|
Sheet sheet = workbook.getSheetAt(i);
|
||||||
|
clearSheetPictures(sheet, pictureIndexes);
|
||||||
|
writeSheet(workbook, sheet, rows, imageCache, pictureIndexes);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
workbook.write(output);
|
workbook.write(output);
|
||||||
|
return totalDataRows(workbook);
|
||||||
} catch (BusinessException ex) {
|
} catch (BusinessException ex) {
|
||||||
throw ex;
|
throw ex;
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
throw new BusinessException("店铺数据抓取累计 Excel 追加失败: " + ex.getMessage());
|
throw new BusinessException("店铺数据抓取累计 Excel 按国家替换失败: " + ex.getMessage());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -155,54 +162,21 @@ public class ShopDataCrawlExcelAssemblyService {
|
|||||||
sheet.removeRow(row);
|
sheet.removeRow(row);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
clearSheetPictures(sheet, pictureIndexes);
|
||||||
int rowIndex = 1;
|
int rowIndex = 1;
|
||||||
for (ShopDataCrawlRowDto value : rows == null ? List.<ShopDataCrawlRowDto>of() : rows) {
|
for (ShopDataCrawlRowDto value : rows == null ? List.<ShopDataCrawlRowDto>of() : rows) {
|
||||||
Row row = sheet.createRow(rowIndex++);
|
Row row = sheet.createRow(rowIndex++);
|
||||||
String[] values = {value.getDate(), value.getAsin(), "", value.getInventorySales(), value.getSalesRank(),
|
writeDataRow(workbook, sheet, row, value, styles, imageCache, pictureIndexes);
|
||||||
value.getPageViews(), value.getUnitsSold(), value.getPrice(), value.getRecommendedOffer(), value.getBrand()};
|
|
||||||
for (int column = 0; column < values.length; column++) {
|
|
||||||
Cell cell = row.createCell(column);
|
|
||||||
if (styles[column] != null) cell.setCellStyle(styles[column]);
|
|
||||||
cell.setCellValue(values[column] == null ? "" : values[column]);
|
|
||||||
}
|
|
||||||
if (!blank(value.getCommodityImage())) {
|
|
||||||
row.setHeightInPoints(IMAGE_ROW_HEIGHT_POINTS);
|
|
||||||
embedImage(workbook, sheet, row, value.getCommodityImage(), imageCache, pictureIndexes);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void appendSheet(XSSFWorkbook workbook,
|
private void writeDataRow(XSSFWorkbook workbook,
|
||||||
Sheet sheet,
|
Sheet sheet,
|
||||||
List<ShopDataCrawlRowDto> rows,
|
Row row,
|
||||||
Map<String, SimilarAsinImageEmbedder.ResizedImage> imageCache,
|
ShopDataCrawlRowDto value,
|
||||||
Map<String, Integer> pictureIndexes) {
|
CellStyle[] styles,
|
||||||
Row header = sheet.getRow(0);
|
Map<String, SimilarAsinImageEmbedder.ResizedImage> imageCache,
|
||||||
Row styleRow = sheet.getRow(1);
|
Map<String, Integer> pictureIndexes) {
|
||||||
boolean currentTemplate = header != null && "商品图片".equals(cellText(header, IMAGE_COLUMN));
|
|
||||||
boolean templateHasBrand = header != null && "品牌".equals(cellText(header, BRAND_COLUMN));
|
|
||||||
CellStyle[] styles = new CellStyle[HEADERS.size()];
|
|
||||||
for (int column = 0; column < styles.length; column++) {
|
|
||||||
int sourceColumn = templateColumnForOutput(column, currentTemplate, templateHasBrand);
|
|
||||||
Cell cell = styleRow == null ? null : styleRow.getCell(sourceColumn);
|
|
||||||
styles[column] = cell == null ? null : cell.getCellStyle();
|
|
||||||
}
|
|
||||||
writeHeaders(sheet, currentTemplate, templateHasBrand);
|
|
||||||
sheet.setColumnWidth(IMAGE_COLUMN, IMAGE_COLUMN_WIDTH);
|
|
||||||
int rowIndex = Math.max(1, sheet.getLastRowNum() + 1);
|
|
||||||
for (ShopDataCrawlRowDto value : rows) {
|
|
||||||
Row row = sheet.createRow(rowIndex++);
|
|
||||||
writeRow(workbook, sheet, row, value, styles, imageCache, pictureIndexes);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void writeRow(XSSFWorkbook workbook,
|
|
||||||
Sheet sheet,
|
|
||||||
Row row,
|
|
||||||
ShopDataCrawlRowDto value,
|
|
||||||
CellStyle[] styles,
|
|
||||||
Map<String, SimilarAsinImageEmbedder.ResizedImage> imageCache,
|
|
||||||
Map<String, Integer> pictureIndexes) {
|
|
||||||
String[] values = {value.getDate(), value.getAsin(), "", value.getInventorySales(), value.getSalesRank(),
|
String[] values = {value.getDate(), value.getAsin(), "", value.getInventorySales(), value.getSalesRank(),
|
||||||
value.getPageViews(), value.getUnitsSold(), value.getPrice(), value.getRecommendedOffer(), value.getBrand()};
|
value.getPageViews(), value.getUnitsSold(), value.getPrice(), value.getRecommendedOffer(), value.getBrand()};
|
||||||
for (int column = 0; column < values.length; column++) {
|
for (int column = 0; column < values.length; column++) {
|
||||||
@@ -306,4 +280,25 @@ public class ShopDataCrawlExcelAssemblyService {
|
|||||||
}
|
}
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void clearSheetPictures(Sheet sheet, Map<String, Integer> pictureIndexes) {
|
||||||
|
Drawing<?> patriarch = sheet.getDrawingPatriarch();
|
||||||
|
if (!(patriarch instanceof XSSFDrawing drawing)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
List<CTTwoCellAnchor> anchors = drawing.getCTDrawing().getTwoCellAnchorList();
|
||||||
|
for (int i = anchors.size() - 1; i >= 0; i--) {
|
||||||
|
if (anchors.get(i).getPic() != null) {
|
||||||
|
drawing.getCTDrawing().removeTwoCellAnchor(i);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private int totalDataRows(XSSFWorkbook workbook) {
|
||||||
|
int total = 0;
|
||||||
|
for (int i = 0; i < workbook.getNumberOfSheets(); i++) {
|
||||||
|
total += Math.max(0, workbook.getSheetAt(i).getLastRowNum());
|
||||||
|
}
|
||||||
|
return total;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+42
-22
@@ -1704,12 +1704,14 @@ public class ShopDataCrawlTaskService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
int addedRowCount = excelAssemblyService.countRows(List.of(snapshot));
|
int addedRowCount = excelAssemblyService.countRows(List.of(snapshot));
|
||||||
|
ShopDataCrawlDailyFileEntity baseDailyFile = resolveBaseDailyFile(
|
||||||
|
preparation.dailyFile(), userId, shopKeyHash, businessDate);
|
||||||
DailyWorkbookArtifact artifact = assembleDailyWorkbook(
|
DailyWorkbookArtifact artifact = assembleDailyWorkbook(
|
||||||
task, snapshot, preparation.dailyFile(), addedRowCount);
|
task, snapshot, baseDailyFile, addedRowCount);
|
||||||
try {
|
try {
|
||||||
persistedResult = executeShortTransaction(() -> persistDailyAggregation(
|
persistedResult = executeShortTransaction(() -> persistDailyAggregation(
|
||||||
task, row, userId, shopKey, shopKeyHash, businessDate,
|
task, row, userId, shopKey, shopKeyHash, businessDate,
|
||||||
preparation, artifact, addedRowCount));
|
preparation, artifact));
|
||||||
if (persistedResult.discardUploadedObject() && artifact.uploaded()) {
|
if (persistedResult.discardUploadedObject() && artifact.uploaded()) {
|
||||||
deleteObjectQuietly(artifact.objectKey());
|
deleteObjectQuietly(artifact.objectKey());
|
||||||
}
|
}
|
||||||
@@ -1743,23 +1745,35 @@ public class ShopDataCrawlTaskService {
|
|||||||
return new DailyAggregationPreparation(dailyFile, false);
|
return new DailyAggregationPreparation(dailyFile, false);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private ShopDataCrawlDailyFileEntity resolveBaseDailyFile(ShopDataCrawlDailyFileEntity currentDailyFile,
|
||||||
|
Long userId,
|
||||||
|
String shopKeyHash,
|
||||||
|
LocalDate businessDate) {
|
||||||
|
if (currentDailyFile != null) {
|
||||||
|
return currentDailyFile;
|
||||||
|
}
|
||||||
|
List<ShopDataCrawlDailyFileEntity> older = dailyFileService.findOlder(userId, shopKeyHash, businessDate);
|
||||||
|
return older == null || older.isEmpty() ? null : older.get(0);
|
||||||
|
}
|
||||||
|
|
||||||
private DailyWorkbookArtifact assembleDailyWorkbook(FileTaskEntity task,
|
private DailyWorkbookArtifact assembleDailyWorkbook(FileTaskEntity task,
|
||||||
ShopDataCrawlResultItemVo snapshot,
|
ShopDataCrawlResultItemVo snapshot,
|
||||||
ShopDataCrawlDailyFileEntity dailyFile,
|
ShopDataCrawlDailyFileEntity baseDailyFile,
|
||||||
int addedRowCount) {
|
int addedRowCount) {
|
||||||
String filename = dailyFile != null && !blank(dailyFile.getResultFilename())
|
String filename = baseDailyFile != null && !blank(baseDailyFile.getResultFilename())
|
||||||
? dailyFile.getResultFilename()
|
? baseDailyFile.getResultFilename()
|
||||||
: buildTaskWorkbookFilename(task);
|
: buildTaskWorkbookFilename(task);
|
||||||
String existingObjectKey = dailyFile == null ? null : dailyFile.getResultFileUrl();
|
String existingObjectKey = baseDailyFile == null ? null : baseDailyFile.getResultFileUrl();
|
||||||
|
|
||||||
// A result with no new rows only needs a new database membership. Reusing
|
// A result with no new rows only needs a new database membership. Reusing
|
||||||
// the canonical daily object avoids both a local copy and an OSS round trip.
|
// the canonical daily object avoids both a local copy and an OSS round trip.
|
||||||
if (dailyFile != null && addedRowCount == 0 && !blank(existingObjectKey)) {
|
if (baseDailyFile != null && addedRowCount == 0 && !blank(existingObjectKey)) {
|
||||||
return new DailyWorkbookArtifact(
|
return new DailyWorkbookArtifact(
|
||||||
existingObjectKey,
|
existingObjectKey,
|
||||||
Math.max(0L, Objects.requireNonNullElse(dailyFile.getResultFileSize(), 0L)),
|
Math.max(0L, Objects.requireNonNullElse(baseDailyFile.getResultFileSize(), 0L)),
|
||||||
false,
|
false,
|
||||||
filename);
|
filename,
|
||||||
|
Math.max(0, Objects.requireNonNullElse(baseDailyFile.getRowCount(), 0)));
|
||||||
}
|
}
|
||||||
|
|
||||||
File workRoot = FileUtil.mkdir(FileUtil.file(
|
File workRoot = FileUtil.mkdir(FileUtil.file(
|
||||||
@@ -1770,21 +1784,25 @@ public class ShopDataCrawlTaskService {
|
|||||||
File baseXlsx = FileUtil.file(workRoot, "base.xlsx");
|
File baseXlsx = FileUtil.file(workRoot, "base.xlsx");
|
||||||
File outputXlsx = FileUtil.file(workRoot, filename);
|
File outputXlsx = FileUtil.file(workRoot, filename);
|
||||||
try {
|
try {
|
||||||
if (dailyFile != null && !blank(existingObjectKey)) {
|
if (baseDailyFile != null && !blank(existingObjectKey)) {
|
||||||
try {
|
try {
|
||||||
Files.write(baseXlsx.toPath(), ossStorageService.readObjectBytes(existingObjectKey));
|
Files.write(baseXlsx.toPath(), ossStorageService.readObjectBytes(existingObjectKey));
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
throw new BusinessException("读取当天累计文件失败: " + safeMessage(ex));
|
throw new BusinessException("读取累计文件失败: " + safeMessage(ex));
|
||||||
}
|
}
|
||||||
excelAssemblyService.appendWorkbook(baseXlsx, outputXlsx, List.of(snapshot));
|
int rowCount = excelAssemblyService.replaceCountriesWorkbook(baseXlsx, outputXlsx, List.of(snapshot));
|
||||||
} else {
|
String objectKey = ossStorageService.uploadResultFile(outputXlsx, MODULE_TYPE);
|
||||||
excelAssemblyService.writeWorkbook(outputXlsx, List.of(snapshot));
|
if (blank(objectKey)) {
|
||||||
|
throw new BusinessException("累计文件上传后未返回文件地址");
|
||||||
|
}
|
||||||
|
return new DailyWorkbookArtifact(objectKey, outputXlsx.length(), true, filename, rowCount);
|
||||||
}
|
}
|
||||||
|
excelAssemblyService.writeWorkbook(outputXlsx, List.of(snapshot));
|
||||||
String objectKey = ossStorageService.uploadResultFile(outputXlsx, MODULE_TYPE);
|
String objectKey = ossStorageService.uploadResultFile(outputXlsx, MODULE_TYPE);
|
||||||
if (blank(objectKey)) {
|
if (blank(objectKey)) {
|
||||||
throw new BusinessException("当天累计文件上传后未返回文件地址");
|
throw new BusinessException("累计文件上传后未返回文件地址");
|
||||||
}
|
}
|
||||||
return new DailyWorkbookArtifact(objectKey, outputXlsx.length(), true, filename);
|
return new DailyWorkbookArtifact(objectKey, outputXlsx.length(), true, filename, addedRowCount);
|
||||||
} finally {
|
} finally {
|
||||||
FileUtil.del(baseXlsx);
|
FileUtil.del(baseXlsx);
|
||||||
FileUtil.del(outputXlsx);
|
FileUtil.del(outputXlsx);
|
||||||
@@ -1799,8 +1817,7 @@ public class ShopDataCrawlTaskService {
|
|||||||
String shopKeyHash,
|
String shopKeyHash,
|
||||||
LocalDate businessDate,
|
LocalDate businessDate,
|
||||||
DailyAggregationPreparation preparation,
|
DailyAggregationPreparation preparation,
|
||||||
DailyWorkbookArtifact artifact,
|
DailyWorkbookArtifact artifact) {
|
||||||
int addedRowCount) {
|
|
||||||
ShopDataCrawlDailyFileEntity dailyFile = dailyFileService.findForUpdate(
|
ShopDataCrawlDailyFileEntity dailyFile = dailyFileService.findForUpdate(
|
||||||
userId, shopKeyHash, businessDate);
|
userId, shopKeyHash, businessDate);
|
||||||
if (handleExistingDailyMembership(row, dailyFile)) {
|
if (handleExistingDailyMembership(row, dailyFile)) {
|
||||||
@@ -1826,9 +1843,7 @@ public class ShopDataCrawlTaskService {
|
|||||||
row.setResultFileUrl(objectKey);
|
row.setResultFileUrl(objectKey);
|
||||||
row.setResultFileSize(artifact.fileSize());
|
row.setResultFileSize(artifact.fileSize());
|
||||||
row.setResultContentType(CONTENT_TYPE_XLSX);
|
row.setResultContentType(CONTENT_TYPE_XLSX);
|
||||||
row.setRowCount(dailyFile == null
|
row.setRowCount(artifact.rowCount());
|
||||||
? addedRowCount
|
|
||||||
: Math.max(0, Objects.requireNonNullElse(dailyFile.getRowCount(), 0)) + addedRowCount);
|
|
||||||
fileResultMapper.updateById(row);
|
fileResultMapper.updateById(row);
|
||||||
|
|
||||||
LocalDateTime now = dailyFileService.currentBusinessDateTime();
|
LocalDateTime now = dailyFileService.currentBusinessDateTime();
|
||||||
@@ -2140,7 +2155,12 @@ public class ShopDataCrawlTaskService {
|
|||||||
private record DailyWorkbookArtifact(String objectKey,
|
private record DailyWorkbookArtifact(String objectKey,
|
||||||
long fileSize,
|
long fileSize,
|
||||||
boolean uploaded,
|
boolean uploaded,
|
||||||
String filename) {
|
String filename,
|
||||||
|
int rowCount) {
|
||||||
|
|
||||||
|
DailyWorkbookArtifact(String objectKey, long fileSize, boolean uploaded, String filename) {
|
||||||
|
this(objectKey, fileSize, uploaded, filename, 0);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private record DailyAggregationResult(List<String> obsoleteObjectKeys,
|
private record DailyAggregationResult(List<String> obsoleteObjectKeys,
|
||||||
|
|||||||
+17
-10
@@ -73,28 +73,35 @@ class ShopDataCrawlExcelAssemblyServiceTest {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void appendsOnlyTheSelectedCountryAndPreservesExistingPictures() throws Exception {
|
void replacesOnlyTheSelectedCountryAndPreservesOtherSheets() throws Exception {
|
||||||
ShopDataCrawlRowDto firstRow = row("2026-07-25", "B012345678");
|
ShopDataCrawlRowDto ukRow = row("2026-07-25", "B012345678");
|
||||||
firstRow.setCommodityImage("https://m.media-amazon.com/images/I/first.jpg");
|
ukRow.setCommodityImage("https://m.media-amazon.com/images/I/first.jpg");
|
||||||
ShopDataCrawlRowDto secondRow = row("2026-07-26", "B099999999");
|
ShopDataCrawlRowDto oldDeRow = row("2026-07-26", "B099999998");
|
||||||
secondRow.setCommodityImage(null);
|
oldDeRow.setCommodityImage("https://m.media-amazon.com/images/I/de.jpg");
|
||||||
|
ShopDataCrawlRowDto newDeRow = row("2026-07-27", "B099999999");
|
||||||
|
|
||||||
SimilarAsinImageEmbedder imageEmbedder = mock(SimilarAsinImageEmbedder.class);
|
SimilarAsinImageEmbedder imageEmbedder = mock(SimilarAsinImageEmbedder.class);
|
||||||
when(imageEmbedder.fetchAndResizeForCache(firstRow.getCommodityImage()))
|
when(imageEmbedder.fetchAndResizeForCache(ukRow.getCommodityImage()))
|
||||||
|
.thenReturn(new SimilarAsinImageEmbedder.ResizedImage(jpegBytes(), 2, 2));
|
||||||
|
when(imageEmbedder.fetchAndResizeForCache(oldDeRow.getCommodityImage()))
|
||||||
.thenReturn(new SimilarAsinImageEmbedder.ResizedImage(jpegBytes(), 2, 2));
|
.thenReturn(new SimilarAsinImageEmbedder.ResizedImage(jpegBytes(), 2, 2));
|
||||||
ShopDataCrawlExcelAssemblyService service = new ShopDataCrawlExcelAssemblyService(imageEmbedder);
|
ShopDataCrawlExcelAssemblyService service = new ShopDataCrawlExcelAssemblyService(imageEmbedder);
|
||||||
File base = tempDir.resolve("base.xlsx").toFile();
|
File base = tempDir.resolve("base.xlsx").toFile();
|
||||||
File output = tempDir.resolve("daily.xlsx").toFile();
|
File output = tempDir.resolve("daily.xlsx").toFile();
|
||||||
service.writeWorkbook(base, List.of(item("UK", firstRow)));
|
service.writeWorkbook(base, List.of(item("UK", ukRow), item("DE", oldDeRow)));
|
||||||
service.appendWorkbook(base, output, List.of(item("DE", secondRow)));
|
|
||||||
|
int total = service.replaceCountriesWorkbook(base, output, List.of(item("DE", newDeRow)));
|
||||||
|
|
||||||
try (XSSFWorkbook workbook = new XSSFWorkbook(new FileInputStream(output))) {
|
try (XSSFWorkbook workbook = new XSSFWorkbook(new FileInputStream(output))) {
|
||||||
assertEquals("B012345678", workbook.getSheetAt(0).getRow(1).getCell(1).getStringCellValue());
|
assertEquals("B012345678", workbook.getSheetAt(0).getRow(1).getCell(1).getStringCellValue());
|
||||||
assertEquals("B099999999", workbook.getSheetAt(1).getRow(1).getCell(1).getStringCellValue());
|
assertEquals("B099999999", workbook.getSheetAt(1).getRow(1).getCell(1).getStringCellValue());
|
||||||
assertEquals(1, workbook.getAllPictures().size());
|
assertEquals(1, workbook.getSheetAt(0).getLastRowNum());
|
||||||
assertEquals(1, workbook.getSheetAt(0).getDrawingPatriarch().getShapes().size());
|
assertEquals(1, workbook.getSheetAt(1).getLastRowNum());
|
||||||
assertEquals(0, workbook.getSheetAt(2).getLastRowNum());
|
assertEquals(0, workbook.getSheetAt(2).getLastRowNum());
|
||||||
|
assertEquals(1, workbook.getSheetAt(0).getDrawingPatriarch().getShapes().size());
|
||||||
|
assertEquals(0, workbook.getSheetAt(1).getDrawingPatriarch().getShapes().size());
|
||||||
}
|
}
|
||||||
|
assertEquals(2, total);
|
||||||
}
|
}
|
||||||
|
|
||||||
private ShopDataCrawlResultItemVo item(String countryCode, ShopDataCrawlRowDto row) {
|
private ShopDataCrawlResultItemVo item(String countryCode, ShopDataCrawlRowDto row) {
|
||||||
|
|||||||
+1
-1
@@ -187,7 +187,7 @@ class ShopDataCrawlTaskServiceChunkTest {
|
|||||||
"task:" + task.getId() + ":owner:instance-a");
|
"task:" + task.getId() + ":owner:instance-a");
|
||||||
assertNull(result.getResultFileUrl());
|
assertNull(result.getResultFileUrl());
|
||||||
verify(excelAssemblyService, never()).writeWorkbook(any(), any());
|
verify(excelAssemblyService, never()).writeWorkbook(any(), any());
|
||||||
verify(excelAssemblyService, never()).appendWorkbook(any(), any(), any());
|
verify(excelAssemblyService, never()).replaceCountriesWorkbook(any(), any(), any());
|
||||||
verify(ossStorageService, never()).uploadResultFile(any(), eq(MODULE_TYPE));
|
verify(ossStorageService, never()).uploadResultFile(any(), eq(MODULE_TYPE));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+18
-8
@@ -150,7 +150,7 @@ class ShopDataCrawlTaskServiceRetentionTest {
|
|||||||
service.processResultFileJob(job);
|
service.processResultFileJob(job);
|
||||||
|
|
||||||
verify(excelAssemblyService).writeWorkbook(any(), eq(List.of(snapshot)));
|
verify(excelAssemblyService).writeWorkbook(any(), eq(List.of(snapshot)));
|
||||||
verify(excelAssemblyService, never()).appendWorkbook(any(), any(), any());
|
verify(excelAssemblyService, never()).replaceCountriesWorkbook(any(), any(), any());
|
||||||
verify(dailyFileService).addMember(301L, TASK_ID, RESULT_ID);
|
verify(dailyFileService).addMember(301L, TASK_ID, RESULT_ID);
|
||||||
assertEquals("result/new.xlsx", currentRow.getResultFileUrl());
|
assertEquals("result/new.xlsx", currentRow.getResultFileUrl());
|
||||||
assertEquals(1, currentRow.getRowCount());
|
assertEquals(1, currentRow.getRowCount());
|
||||||
@@ -163,7 +163,7 @@ class ShopDataCrawlTaskServiceRetentionTest {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void sameDaySuccessAppendsAndReplacesPreviousPointer() {
|
void sameDaySuccessReplacesCountryRowsAndPointsToNewWorkbook() {
|
||||||
FileResultEntity previous = result(200L, 100L, "result/old.xlsx");
|
FileResultEntity previous = result(200L, 100L, "result/old.xlsx");
|
||||||
previous.setUserId(null);
|
previous.setUserId(null);
|
||||||
FileTaskEntity previousTask = task();
|
FileTaskEntity previousTask = task();
|
||||||
@@ -175,10 +175,11 @@ class ShopDataCrawlTaskServiceRetentionTest {
|
|||||||
when(dailyFileService.containsResult(301L, RESULT_ID)).thenReturn(false);
|
when(dailyFileService.containsResult(301L, RESULT_ID)).thenReturn(false);
|
||||||
when(ossStorageService.readObjectBytes("result/old.xlsx")).thenReturn(new byte[]{1, 2, 3});
|
when(ossStorageService.readObjectBytes("result/old.xlsx")).thenReturn(new byte[]{1, 2, 3});
|
||||||
when(ossStorageService.uploadResultFile(any(), eq(MODULE_TYPE))).thenReturn("result/new.xlsx");
|
when(ossStorageService.uploadResultFile(any(), eq(MODULE_TYPE))).thenReturn("result/new.xlsx");
|
||||||
|
when(excelAssemblyService.replaceCountriesWorkbook(any(), any(), any())).thenReturn(3);
|
||||||
|
|
||||||
service.processResultFileJob(job);
|
service.processResultFileJob(job);
|
||||||
|
|
||||||
verify(excelAssemblyService).appendWorkbook(any(), any(), eq(List.of(snapshot)));
|
verify(excelAssemblyService).replaceCountriesWorkbook(any(), any(), eq(List.of(snapshot)));
|
||||||
verify(dailyFileService).update(daily);
|
verify(dailyFileService).update(daily);
|
||||||
verify(ossStorageService).deleteObject("result/old.xlsx");
|
verify(ossStorageService).deleteObject("result/old.xlsx");
|
||||||
assertNull(previous.getResultFileUrl());
|
assertNull(previous.getResultFileUrl());
|
||||||
@@ -216,8 +217,8 @@ class ShopDataCrawlTaskServiceRetentionTest {
|
|||||||
}).when(ossStorageService).readObjectBytes("result/old.xlsx");
|
}).when(ossStorageService).readObjectBytes("result/old.xlsx");
|
||||||
doAnswer(invocation -> {
|
doAnswer(invocation -> {
|
||||||
assertFalse(transactionActive.get(), "workbook assembly must run outside the database transaction");
|
assertFalse(transactionActive.get(), "workbook assembly must run outside the database transaction");
|
||||||
return null;
|
return 3;
|
||||||
}).when(excelAssemblyService).appendWorkbook(any(), any(), any());
|
}).when(excelAssemblyService).replaceCountriesWorkbook(any(), any(), any());
|
||||||
doAnswer(invocation -> {
|
doAnswer(invocation -> {
|
||||||
assertFalse(transactionActive.get(), "workbook upload must run outside the database transaction");
|
assertFalse(transactionActive.get(), "workbook upload must run outside the database transaction");
|
||||||
return "result/new.xlsx";
|
return "result/new.xlsx";
|
||||||
@@ -243,7 +244,7 @@ class ShopDataCrawlTaskServiceRetentionTest {
|
|||||||
service.processResultFileJob(job);
|
service.processResultFileJob(job);
|
||||||
|
|
||||||
verify(excelAssemblyService, never()).writeWorkbook(any(), any());
|
verify(excelAssemblyService, never()).writeWorkbook(any(), any());
|
||||||
verify(excelAssemblyService, never()).appendWorkbook(any(), any(), any());
|
verify(excelAssemblyService, never()).replaceCountriesWorkbook(any(), any(), any());
|
||||||
verify(ossStorageService, never()).uploadResultFile(any(), anyString());
|
verify(ossStorageService, never()).uploadResultFile(any(), anyString());
|
||||||
assertEquals("result/current.xlsx", currentRow.getResultFileUrl());
|
assertEquals("result/current.xlsx", currentRow.getResultFileUrl());
|
||||||
assertEquals(3, currentRow.getRowCount());
|
assertEquals(3, currentRow.getRowCount());
|
||||||
@@ -261,7 +262,7 @@ class ShopDataCrawlTaskServiceRetentionTest {
|
|||||||
|
|
||||||
verify(ossStorageService, never()).readObjectBytes(anyString());
|
verify(ossStorageService, never()).readObjectBytes(anyString());
|
||||||
verify(excelAssemblyService, never()).writeWorkbook(any(), any());
|
verify(excelAssemblyService, never()).writeWorkbook(any(), any());
|
||||||
verify(excelAssemblyService, never()).appendWorkbook(any(), any(), any());
|
verify(excelAssemblyService, never()).replaceCountriesWorkbook(any(), any(), any());
|
||||||
verify(ossStorageService, never()).uploadResultFile(any(), anyString());
|
verify(ossStorageService, never()).uploadResultFile(any(), anyString());
|
||||||
verify(dailyFileService).addMember(301L, TASK_ID, RESULT_ID);
|
verify(dailyFileService).addMember(301L, TASK_ID, RESULT_ID);
|
||||||
assertEquals("result/current.xlsx", currentRow.getResultFileUrl());
|
assertEquals("result/current.xlsx", currentRow.getResultFileUrl());
|
||||||
@@ -308,7 +309,7 @@ class ShopDataCrawlTaskServiceRetentionTest {
|
|||||||
service.processResultFileJob(job);
|
service.processResultFileJob(job);
|
||||||
|
|
||||||
verify(excelAssemblyService, never()).writeWorkbook(any(), any());
|
verify(excelAssemblyService, never()).writeWorkbook(any(), any());
|
||||||
verify(excelAssemblyService, never()).appendWorkbook(any(), any(), any());
|
verify(excelAssemblyService, never()).replaceCountriesWorkbook(any(), any(), any());
|
||||||
verify(ossStorageService, never()).uploadResultFile(any(), anyString());
|
verify(ossStorageService, never()).uploadResultFile(any(), anyString());
|
||||||
assertNull(currentRow.getResultFileUrl());
|
assertNull(currentRow.getResultFileUrl());
|
||||||
}
|
}
|
||||||
@@ -322,13 +323,18 @@ class ShopDataCrawlTaskServiceRetentionTest {
|
|||||||
when(fileResultMapper.selectList(any())).thenReturn(List.of(currentRow), List.of(previous, currentRow));
|
when(fileResultMapper.selectList(any())).thenReturn(List.of(currentRow), List.of(previous, currentRow));
|
||||||
when(dailyFileService.findForUpdate(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(null);
|
when(dailyFileService.findForUpdate(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(null);
|
||||||
when(dailyFileService.findOlder(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(List.of(yesterday));
|
when(dailyFileService.findOlder(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(List.of(yesterday));
|
||||||
|
when(ossStorageService.readObjectBytes("result/yesterday.xlsx")).thenReturn(new byte[]{1, 2, 3});
|
||||||
when(ossStorageService.uploadResultFile(any(), eq(MODULE_TYPE))).thenReturn("result/today.xlsx");
|
when(ossStorageService.uploadResultFile(any(), eq(MODULE_TYPE))).thenReturn("result/today.xlsx");
|
||||||
|
when(excelAssemblyService.replaceCountriesWorkbook(any(), any(), any())).thenReturn(5);
|
||||||
|
|
||||||
service.processResultFileJob(job);
|
service.processResultFileJob(job);
|
||||||
|
|
||||||
|
verify(excelAssemblyService, never()).writeWorkbook(any(), any());
|
||||||
|
verify(excelAssemblyService).replaceCountriesWorkbook(any(), any(), eq(List.of(snapshot)));
|
||||||
verify(dailyFileService).deleteDailyFile(300L);
|
verify(dailyFileService).deleteDailyFile(300L);
|
||||||
verify(ossStorageService).deleteObject("result/yesterday.xlsx");
|
verify(ossStorageService).deleteObject("result/yesterday.xlsx");
|
||||||
assertEquals("result/today.xlsx", currentRow.getResultFileUrl());
|
assertEquals("result/today.xlsx", currentRow.getResultFileUrl());
|
||||||
|
assertEquals(5, currentRow.getRowCount());
|
||||||
assertNull(previous.getResultFileUrl());
|
assertNull(previous.getResultFileUrl());
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -340,6 +346,8 @@ class ShopDataCrawlTaskServiceRetentionTest {
|
|||||||
when(fileResultMapper.selectList(any())).thenReturn(List.of(currentRow));
|
when(fileResultMapper.selectList(any())).thenReturn(List.of(currentRow));
|
||||||
when(dailyFileService.findForUpdate(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(null);
|
when(dailyFileService.findForUpdate(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(null);
|
||||||
when(dailyFileService.findOlder(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(List.of(yesterday));
|
when(dailyFileService.findOlder(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(List.of(yesterday));
|
||||||
|
when(ossStorageService.readObjectBytes("result/yesterday.xlsx")).thenReturn(new byte[]{1, 2, 3});
|
||||||
|
when(excelAssemblyService.replaceCountriesWorkbook(any(), any(), any())).thenReturn(4);
|
||||||
doThrow(new IllegalStateException("upload failed"))
|
doThrow(new IllegalStateException("upload failed"))
|
||||||
.when(ossStorageService).uploadResultFile(any(), eq(MODULE_TYPE));
|
.when(ossStorageService).uploadResultFile(any(), eq(MODULE_TYPE));
|
||||||
|
|
||||||
@@ -357,6 +365,8 @@ class ShopDataCrawlTaskServiceRetentionTest {
|
|||||||
when(fileResultMapper.selectList(any())).thenReturn(List.of(currentRow));
|
when(fileResultMapper.selectList(any())).thenReturn(List.of(currentRow));
|
||||||
when(dailyFileService.findForUpdate(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(null);
|
when(dailyFileService.findForUpdate(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(null);
|
||||||
when(dailyFileService.findOlder(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(List.of(yesterday));
|
when(dailyFileService.findOlder(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(List.of(yesterday));
|
||||||
|
when(ossStorageService.readObjectBytes("result/yesterday.xlsx")).thenReturn(new byte[]{1, 2, 3});
|
||||||
|
when(excelAssemblyService.replaceCountriesWorkbook(any(), any(), any())).thenReturn(4);
|
||||||
when(ossStorageService.uploadResultFile(any(), eq(MODULE_TYPE))).thenReturn("result/new.xlsx");
|
when(ossStorageService.uploadResultFile(any(), eq(MODULE_TYPE))).thenReturn("result/new.xlsx");
|
||||||
AtomicInteger commitCount = new AtomicInteger();
|
AtomicInteger commitCount = new AtomicInteger();
|
||||||
doAnswer(invocation -> {
|
doAnswer(invocation -> {
|
||||||
|
|||||||
Reference in New Issue
Block a user