super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

后台去重数据新增国家

Browse Source
This commit is contained in:
黄自达
2026-08-23 22:50:58 +08:00
parent 5a750be066
commit 8712df2645
12 changed files with 294 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/controller/DedupeTotalDataController.java
+5 -3
View File
@@ -87,10 +87,11 @@ public class DedupeTotalDataController {
@Parameter(description = "结束日期（包含）")
@RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE) LocalDate endDate,
@Parameter(description = "分组 ID") @RequestParam(required = false) Long groupId,
@Parameter(description = "国家代码（如 DE、UK）") @RequestParam(required = false) String country,
HttpServletRequest request) {
RequestOperator operator = requireDedupeTotalDataAccess(request);
return ApiResponse.success(dedupeTotalDataService.page(
page, pageSize, keyword, username, startDate, endDate, groupId, operator.id()));
page, pageSize, keyword, username, startDate, endDate, groupId, country, operator.id()));
}
@GetMapping("/export")
@@ -102,6 +103,7 @@ public class DedupeTotalDataController {
@Parameter(description = "结束日期（包含）")
@RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE) LocalDate endDate,
@Parameter(description = "分组 ID") @RequestParam(required = false) Long groupId,
@Parameter(description = "国家代码（如 DE、UK）") @RequestParam(required = false) String country,
HttpServletRequest request) {
if (startDate != null && endDate != null && startDate.isAfter(endDate)) {
throw new BusinessException(400, "invalid export date range");
@@ -112,9 +114,9 @@ public class DedupeTotalDataController {
+ (monthlyZip ? ".zip" : ".xlsx");
StreamingResponseBody body = monthlyZip
? outputStream -> dedupeTotalDataService.writeMonthlyZipExport(
outputStream, username, startDate, endDate, groupId, operator.id())
outputStream, username, startDate, endDate, groupId, country, operator.id())
: outputStream -> dedupeTotalDataService.writeExport(
outputStream, username, startDate, endDate, groupId, operator.id());
outputStream, username, startDate, endDate, groupId, country, operator.id());
return ResponseEntity.ok()
.header(HttpHeaders.CONTENT_DISPOSITION, DownloadHeaderUtil.contentDisposition(filename))
.contentType(MediaType.parseMediaType(monthlyZip ? ZIP_CONTENT_TYPE : XLSX_CONTENT_TYPE))
backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/mapper/DedupeTotalDataMapper.java
+2 -2
View File
@@ -28,10 +28,10 @@ public interface DedupeTotalDataMapper extends BaseMapper<DedupeTotalDataEntity>
@Insert("""
<script>
INSERT IGNORE INTO biz_dedupe_total_data
(data_value, group_id, uploader_user_id, uploader_username)
(data_value, country, group_id, uploader_user_id, uploader_username)
VALUES
<foreach collection="rows" item="row" separator=",">
(#{row.dataValue}, #{row.groupId}, #{row.uploaderUserId}, #{row.uploaderUsername})
(#{row.dataValue}, #{row.country}, #{row.groupId}, #{row.uploaderUserId}, #{row.uploaderUsername})
</foreach>
</script>
""")
backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/entity/DedupeTotalDataEntity.java
+1
View File
@@ -14,6 +14,7 @@ public class DedupeTotalDataEntity {
@TableId(type = IdType.AUTO)
private Long id;
private String dataValue;
private String country;
private Long groupId;
private Long uploaderUserId;
private String uploaderUsername;
backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/model/vo/DedupeTotalDataItemVo.java
+3
View File
@@ -15,6 +15,9 @@ public class DedupeTotalDataItemVo {
@Schema(description = "总数据值")
private String dataValue;
@Schema(description = "国家代码，逗号分隔，如 DE,UK")
private String country;
@Schema(description = "分组 ID")
private Long groupId;
backend-java/src/main/java/com/nanri/aiimage/modules/dedupe/service/DedupeTotalDataService.java
+115 -17
View File
@@ -94,6 +94,11 @@ public class DedupeTotalDataService {
public DedupeTotalDataPageVo page(long page, long pageSize, String keyword, String username,
LocalDate startDate, LocalDate endDate, Long groupId, Long operatorId) {
return page(page, pageSize, keyword, username, startDate, endDate, groupId, null, operatorId);
}
public DedupeTotalDataPageVo page(long page, long pageSize, String keyword, String username,
LocalDate startDate, LocalDate endDate, Long groupId, String country, Long operatorId) {
if (startDate != null && endDate != null && startDate.isAfter(endDate)) {
throw new BusinessException("开始日期不能晚于结束日期");
}
@@ -101,10 +106,14 @@ public class DedupeTotalDataService {
long safePageSize = Math.min(Math.max(pageSize, 1), 100);
String safeKeyword = keyword == null ? "" : keyword.trim();
String safeUsername = username == null ? "" : username.trim();
String safeCountry = country == null ? "" : country.trim();
AccessScope scope = resolveAccessScope(operatorId);
LambdaQueryWrapper<DedupeTotalDataEntity> query = new LambdaQueryWrapper<DedupeTotalDataEntity>()
.like(!safeKeyword.isEmpty(), DedupeTotalDataEntity::getDataValue, safeKeyword)
.like(!safeUsername.isEmpty(), DedupeTotalDataEntity::getUploaderUsername, safeUsername)
.apply(!safeCountry.isEmpty(),
"FIND_IN_SET({0}, IFNULL(country, '')) > 0",
safeCountry)
.ge(startDate != null, DedupeTotalDataEntity::getCreatedAt,
startDate == null ? null : startDate.atStartOfDay())
.lt(endDate != null, DedupeTotalDataEntity::getCreatedAt,
@@ -134,13 +143,18 @@ public class DedupeTotalDataService {
public byte[] export(String username, LocalDate startDate, LocalDate endDate,
Long groupId, Long operatorId) {
return export(username, startDate, endDate, groupId, null, operatorId);
}
public byte[] export(String username, LocalDate startDate, LocalDate endDate,
Long groupId, String country, Long operatorId) {
if (startDate != null && endDate != null && startDate.isAfter(endDate)) {
throw new BusinessException("开始日期不能晚于结束日期");
}
String safeUsername = username == null ? "" : username.trim();
AccessScope scope = resolveAccessScope(operatorId);
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
buildExportWorkbook(outputStream, safeUsername, startDate, endDate, groupId, scope);
buildExportWorkbook(outputStream, safeUsername, startDate, endDate, groupId, country, scope);
return outputStream.toByteArray();
}
@@ -150,6 +164,16 @@ public class DedupeTotalDataService {
LocalDate endDate,
Long groupId,
Long operatorId) {
writeExport(outputStream, username, startDate, endDate, groupId, null, operatorId);
}
public void writeExport(OutputStream outputStream,
String username,
LocalDate startDate,
LocalDate endDate,
Long groupId,
String country,
Long operatorId) {
if (outputStream == null) {
throw new BusinessException("导出输出流不能为空");
}
@@ -158,7 +182,7 @@ public class DedupeTotalDataService {
}
String safeUsername = username == null ? "" : username.trim();
AccessScope scope = resolveAccessScope(operatorId);
buildExportWorkbook(outputStream, safeUsername, startDate, endDate, groupId, scope);
buildExportWorkbook(outputStream, safeUsername, startDate, endDate, groupId, country, scope);
}
public void writeMonthlyZipExport(OutputStream outputStream,
@@ -167,17 +191,28 @@ public class DedupeTotalDataService {
LocalDate endDate,
Long groupId,
Long operatorId) {
writeMonthlyZipExport(outputStream, username, startDate, endDate, groupId, null, operatorId);
}
public void writeMonthlyZipExport(OutputStream outputStream,
String username,
LocalDate startDate,
LocalDate endDate,
Long groupId,
String country,
Long operatorId) {
if (outputStream == null) {
throw new BusinessException("export output stream cannot be null");
}
if (startDate == null || endDate == null || YearMonth.from(startDate).equals(YearMonth.from(endDate))) {
writeExport(outputStream, username, startDate, endDate, groupId, operatorId);
writeExport(outputStream, username, startDate, endDate, groupId, country, operatorId);
return;
}
if (startDate.isAfter(endDate)) {
throw new BusinessException("invalid export date range");
}
String safeUsername = username == null ? "" : username.trim();
String safeCountry = country == null ? "" : country.trim();
AccessScope scope = resolveAccessScope(operatorId);
boolean groupPrevalidated = groupId != null && groupId > 0;
if (groupPrevalidated) {
@@ -194,7 +229,7 @@ public class DedupeTotalDataService {
LocalDate entryEndDate = endDate.isBefore(monthEnd) ? endDate : monthEnd;
zipOutputStream.putNextEntry(new ZipEntry(monthlyExportEntryName(currentMonth)));
buildExportWorkbook(zipOutputStream, safeUsername, entryStartDate, entryEndDate,
groupId, scope, groupPrevalidated);
groupId, safeCountry, scope, groupPrevalidated);
zipOutputStream.closeEntry();
currentMonth = currentMonth.plusMonths(1);
}
@@ -211,8 +246,9 @@ public class DedupeTotalDataService {
LocalDate startDate,
LocalDate endDate,
Long groupId,
String country,
AccessScope scope) {
buildExportWorkbook(outputStream, username, startDate, endDate, groupId, scope, false);
buildExportWorkbook(outputStream, username, startDate, endDate, groupId, country, scope, false);
}
private void buildExportWorkbook(OutputStream outputStream,
@@ -220,6 +256,7 @@ public class DedupeTotalDataService {
LocalDate startDate,
LocalDate endDate,
Long groupId,
String country,
AccessScope scope,
boolean groupPrevalidated) {
try (SXSSFWorkbook workbook = new SXSSFWorkbook(100)) {
@@ -227,14 +264,15 @@ public class DedupeTotalDataService {
Row header = sheet.createRow(0);
header.createCell(0).setCellValue("ID");
header.createCell(1).setCellValue("ASIN值");
header.createCell(2).setCellValue("用户名");
header.createCell(3).setCellValue("分组");
header.createCell(4).setCellValue("创建时间");
header.createCell(2).setCellValue("国家");
header.createCell(3).setCellValue("用户名");
header.createCell(4).setCellValue("分组");
header.createCell(5).setCellValue("创建时间");
Long lastId = null;
int rowIndex = 1;
while (true) {
LambdaQueryWrapper<DedupeTotalDataEntity> pageQuery = buildExportQuery(
username, startDate, endDate, groupId, scope, groupPrevalidated);
username, startDate, endDate, groupId, country, scope, groupPrevalidated);
if (lastId != null) {
pageQuery.lt(DedupeTotalDataEntity::getId, lastId);
}
@@ -248,11 +286,12 @@ public class DedupeTotalDataService {
Row row = sheet.createRow(rowIndex++);
row.createCell(0).setCellValue(entity.getId() == null ? "" : String.valueOf(entity.getId()));
row.createCell(1).setCellValue(entity.getDataValue() == null ? "" : entity.getDataValue());
row.createCell(2).setCellValue(entity.getUploaderUsername() == null ? "" : entity.getUploaderUsername());
row.createCell(3).setCellValue(entity.getGroupId() == null
row.createCell(2).setCellValue(entity.getCountry() == null ? "" : entity.getCountry());
row.createCell(3).setCellValue(entity.getUploaderUsername() == null ? "" : entity.getUploaderUsername());
row.createCell(4).setCellValue(entity.getGroupId() == null
? ""
: groupNames.getOrDefault(entity.getGroupId(), ""));
row.createCell(4).setCellValue(formatExportTime(entity.getCreatedAt()));
row.createCell(5).setCellValue(formatExportTime(entity.getCreatedAt()));
}
DedupeTotalDataEntity lastRow = rows.getLast();
lastId = lastRow == null ? null : lastRow.getId();
@@ -265,9 +304,10 @@ public class DedupeTotalDataService {
}
sheet.setColumnWidth(0, 3600);
sheet.setColumnWidth(1, 5200);
sheet.setColumnWidth(2, 5200);
sheet.setColumnWidth(2, 3200);
sheet.setColumnWidth(3, 5200);
sheet.setColumnWidth(4, 5600);
sheet.setColumnWidth(4, 5200);
sheet.setColumnWidth(5, 5600);
workbook.write(outputStream);
workbook.dispose();
} catch (Exception ex) {
@@ -280,18 +320,23 @@ public class DedupeTotalDataService {
LocalDate endDate,
Long groupId,
AccessScope scope) {
return buildExportQuery(username, startDate, endDate, groupId, scope, false);
return buildExportQuery(username, startDate, endDate, groupId, null, scope, false);
}
private LambdaQueryWrapper<DedupeTotalDataEntity> buildExportQuery(String username,
LocalDate startDate,
LocalDate endDate,
Long groupId,
String country,
AccessScope scope,
boolean groupPrevalidated) {
String safeUsername = username == null ? "" : username.trim();
String safeCountry = country == null ? "" : country.trim();
LambdaQueryWrapper<DedupeTotalDataEntity> query = new LambdaQueryWrapper<DedupeTotalDataEntity>()
.like(!safeUsername.isEmpty(), DedupeTotalDataEntity::getUploaderUsername, safeUsername)
.apply(!safeCountry.isEmpty(),
"FIND_IN_SET({0}, IFNULL(country, '')) > 0",
safeCountry)
.ge(startDate != null, DedupeTotalDataEntity::getCreatedAt,
startDate == null ? null : startDate.atStartOfDay())
.lt(endDate != null, DedupeTotalDataEntity::getCreatedAt,
@@ -608,9 +653,11 @@ public class DedupeTotalDataService {
if (asinIndex == null) {
throw new BusinessException("缺少 ASIN 列");
}
Integer countryIndex = headerMap.get("国家");
Set<String> seenInFile = new HashSet<>();
List<String> pendingValues = new ArrayList<>(IMPORT_BATCH_SIZE);
Map<String, String> pendingCountries = new HashMap<>();
int totalRows = Math.max(sheet.getLastRowNum(), 0);
int asinCount = 0;
int insertedCount = 0;
@@ -644,21 +691,26 @@ public class DedupeTotalDataService {
continue;
}
pendingValues.add(asin);
if (countryIndex != null) {
pendingCountries.put(asin, parseCountries(formatter.formatCellValue(row.getCell(countryIndex))));
}
if (pendingValues.size() >= IMPORT_BATCH_SIZE) {
ImportBatchResult batch = insertImportBatch(
pendingValues, groupId, uploaderUserId, uploaderUsername);
pendingValues, pendingCountries, groupId, uploaderUserId, uploaderUsername);
insertedCount += batch.insertedCount();
skippedCount += batch.skippedCount();
pendingValues.clear();
pendingCountries.clear();
}
updateImportProgress(progress, rowNum, asinCount, insertedCount, skippedCount);
}
if (!pendingValues.isEmpty()) {
ImportBatchResult batch = insertImportBatch(
pendingValues, groupId, uploaderUserId, uploaderUsername);
pendingValues, pendingCountries, groupId, uploaderUserId, uploaderUsername);
insertedCount += batch.insertedCount();
skippedCount += batch.skippedCount();
pendingValues.clear();
pendingCountries.clear();
}
updateImportProgress(progress, totalRows, asinCount, insertedCount, skippedCount);
@@ -676,6 +728,7 @@ public class DedupeTotalDataService {
}
private ImportBatchResult insertImportBatch(List<String> values,
Map<String, String> countriesByValue,
Long groupId,
Long uploaderUserId,
String uploaderUsername) {
@@ -690,6 +743,7 @@ public class DedupeTotalDataService {
}
DedupeTotalDataEntity entity = new DedupeTotalDataEntity();
entity.setDataValue(value);
entity.setCountry(countriesByValue == null ? null : countriesByValue.get(value));
entity.setGroupId(groupId);
entity.setUploaderUserId(uploaderUserId);
entity.setUploaderUsername(uploaderUsername);
@@ -1076,6 +1130,49 @@ public class DedupeTotalDataService {
.replaceAll("\\s+", " ");
}
private static final List<String> COUNTRY_CODE_MAP = List.of(
"DE", "UK", "FR", "IT", "ES", "US",
"美国", "德国", "英国", "法国", "意大利", "西班牙");
private String normalizeCountryCode(String raw) {
String upper = normalizeExcelText(raw).toUpperCase(Locale.ROOT);
return upper
.replace("GERMANY", "DE")
.replace("UNITED KINGDOM", "UK")
.replace("UNITEDKINGDOM", "UK")
.replace("ENGLAND", "UK")
.replace("FRANCE", "FR")
.replace("ITALY", "IT")
.replace("SPAIN", "ES")
.replace("USA", "US")
.replace("UNITED STATES", "US")
.replace("UNITEDSTATES", "US")
.replace("AMERICA", "US")
.replace("德国", "DE")
.replace("英国", "UK")
.replace("法国", "FR")
.replace("意大利", "IT")
.replace("西班牙", "ES")
.replace("美国", "US");
}
/** 将国家列单元格内容解析为标准国家代码（去重后按出现顺序），无有效国家返回 null。 */
private String parseCountries(String raw) {
if (raw == null || raw.isBlank()) {
return null;
}
// 先对整个单元格做别名归一（含多词国家名如 "United Kingdom"），再按分隔符/空格拆分。
String normalized = normalizeCountryCode(raw);
LinkedHashSet<String> codes = new LinkedHashSet<>();
String[] tokens = normalized.split("[,，;；、/\\\\|\\s]+");
for (String token : tokens) {
if (COUNTRY_CODE_MAP.contains(token)) {
codes.add(token);
}
}
return codes.isEmpty() ? null : String.join(",", codes);
}
private Map<Long, String> loadGroupNames(List<DedupeTotalDataEntity> rows) {
List<Long> groupIds = rows.stream()
.map(DedupeTotalDataEntity::getGroupId)
@@ -1098,6 +1195,7 @@ public class DedupeTotalDataService {
DedupeTotalDataItemVo vo = new DedupeTotalDataItemVo();
vo.setId(entity.getId());
vo.setDataValue(entity.getDataValue());
vo.setCountry(entity.getCountry());
vo.setGroupId(entity.getGroupId());
vo.setGroupName(groupName == null ? "" : groupName);
vo.setUploaderUserId(entity.getUploaderUserId());