Compare commits

..

5 Commits

Author SHA1 Message Date
huangzd1997 bc8870f957 移除
Build Backend JAR / build (push) Has been cancelled
2026-08-24 00:08:13 +08:00
huangzd1997 a659132469 去重总数据国家限定五国并优化筛选布局
- 国家仅保留德国/英国/法国/意大利/西班牙,移除美国及US映射
- 保留服务器已有 V90 迁移,新增 V91 添加 country 列及索引
- ASIN列表筛选区改为两行布局:查询条件+按钮在上,日期+导出在下
2026-08-24 00:03:28 +08:00
huangzd1997 8712df2645 后台去重数据新增国家 2026-08-23 22:50:58 +08:00
huangzd1997 5a750be066 Merge branch 'master' of https://git.aishufu.top/super/crawler-plugin 2026-08-23 22:06:11 +08:00
huangzd1997 b00ea71c1b 换设备重新绑定 2026-08-22 16:06:05 +08:00
14 changed files with 337 additions and 47 deletions
@@ -56,20 +56,19 @@ public class AuthService {
stored = deviceId;
log.info("[auth] first-login bind userId={} device={}", user.getId(), deviceId);
} else if (!stored.equals(deviceId)) {
if (isAdmin) {
log.warn("[auth] device mismatch but admin bypass userId={} stored={} current={}",
user.getId(), stored, deviceId);
} else {
log.warn("[auth] device mismatch reject userId={} stored={} current={}",
user.getId(), stored, deviceId);
throw new BusinessException("当前设备与首次登录设备不一致,请在原设备上登录");
}
// 设备指纹变化(换电脑/重装/清理注册表)会锁死账号,密码已验证通过,
// 直接重新绑定到当前设备,避免账号被锁、数据因重建账号而丢失。
loginUserMapper.update(null, new LambdaUpdateWrapper<LoginUserEntity>()
.eq(LoginUserEntity::getId, user.getId())
.set(LoginUserEntity::getMachine, deviceId));
log.warn("[auth] device rebound on login userId={} old={} new={}",
user.getId(), stored, deviceId);
} else {
log.info("[auth] device match userId={} isAdmin={} device={}",
user.getId(), isAdmin, deviceId);
}
return buildResult(user, stored, isAdmin);
return buildResult(user, deviceId, isAdmin);
}
public LoginResultVo checkLogin(String token, String currentDeviceId) {
@@ -87,10 +87,11 @@ public class DedupeTotalDataController {
@Parameter(description = "结束日期(包含)")
@RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE) LocalDate endDate,
@Parameter(description = "分组 ID") @RequestParam(required = false) Long groupId,
@Parameter(description = "国家代码(如 DE、UK") @RequestParam(required = false) String country,
HttpServletRequest request) {
RequestOperator operator = requireDedupeTotalDataAccess(request);
return ApiResponse.success(dedupeTotalDataService.page(
page, pageSize, keyword, username, startDate, endDate, groupId, operator.id()));
page, pageSize, keyword, username, startDate, endDate, groupId, country, operator.id()));
}
@GetMapping("/export")
@@ -102,6 +103,7 @@ public class DedupeTotalDataController {
@Parameter(description = "结束日期(包含)")
@RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE) LocalDate endDate,
@Parameter(description = "分组 ID") @RequestParam(required = false) Long groupId,
@Parameter(description = "国家代码(如 DE、UK") @RequestParam(required = false) String country,
HttpServletRequest request) {
if (startDate != null && endDate != null && startDate.isAfter(endDate)) {
throw new BusinessException(400, "invalid export date range");
@@ -112,9 +114,9 @@ public class DedupeTotalDataController {
+ (monthlyZip ? ".zip" : ".xlsx");
StreamingResponseBody body = monthlyZip
? outputStream -> dedupeTotalDataService.writeMonthlyZipExport(
outputStream, username, startDate, endDate, groupId, operator.id())
outputStream, username, startDate, endDate, groupId, country, operator.id())
: outputStream -> dedupeTotalDataService.writeExport(
outputStream, username, startDate, endDate, groupId, operator.id());
outputStream, username, startDate, endDate, groupId, country, operator.id());
return ResponseEntity.ok()
.header(HttpHeaders.CONTENT_DISPOSITION, DownloadHeaderUtil.contentDisposition(filename))
.contentType(MediaType.parseMediaType(monthlyZip ? ZIP_CONTENT_TYPE : XLSX_CONTENT_TYPE))
@@ -28,10 +28,10 @@ public interface DedupeTotalDataMapper extends BaseMapper<DedupeTotalDataEntity>
@Insert("""
<script>
INSERT IGNORE INTO biz_dedupe_total_data
(data_value, group_id, uploader_user_id, uploader_username)
(data_value, country, group_id, uploader_user_id, uploader_username)
VALUES
<foreach collection="rows" item="row" separator=",">
(#{row.dataValue}, #{row.groupId}, #{row.uploaderUserId}, #{row.uploaderUsername})
(#{row.dataValue}, #{row.country}, #{row.groupId}, #{row.uploaderUserId}, #{row.uploaderUsername})
</foreach>
</script>
""")
@@ -14,6 +14,7 @@ public class DedupeTotalDataEntity {
@TableId(type = IdType.AUTO)
private Long id;
private String dataValue;
private String country;
private Long groupId;
private Long uploaderUserId;
private String uploaderUsername;
@@ -15,6 +15,9 @@ public class DedupeTotalDataItemVo {
@Schema(description = "总数据值")
private String dataValue;
@Schema(description = "国家代码,逗号分隔,如 DE,UK")
private String country;
@Schema(description = "分组 ID")
private Long groupId;
@@ -94,6 +94,11 @@ public class DedupeTotalDataService {
public DedupeTotalDataPageVo page(long page, long pageSize, String keyword, String username,
LocalDate startDate, LocalDate endDate, Long groupId, Long operatorId) {
return page(page, pageSize, keyword, username, startDate, endDate, groupId, null, operatorId);
}
public DedupeTotalDataPageVo page(long page, long pageSize, String keyword, String username,
LocalDate startDate, LocalDate endDate, Long groupId, String country, Long operatorId) {
if (startDate != null && endDate != null && startDate.isAfter(endDate)) {
throw new BusinessException("开始日期不能晚于结束日期");
}
@@ -101,10 +106,14 @@ public class DedupeTotalDataService {
long safePageSize = Math.min(Math.max(pageSize, 1), 100);
String safeKeyword = keyword == null ? "" : keyword.trim();
String safeUsername = username == null ? "" : username.trim();
String safeCountry = country == null ? "" : country.trim();
AccessScope scope = resolveAccessScope(operatorId);
LambdaQueryWrapper<DedupeTotalDataEntity> query = new LambdaQueryWrapper<DedupeTotalDataEntity>()
.like(!safeKeyword.isEmpty(), DedupeTotalDataEntity::getDataValue, safeKeyword)
.like(!safeUsername.isEmpty(), DedupeTotalDataEntity::getUploaderUsername, safeUsername)
.apply(!safeCountry.isEmpty(),
"FIND_IN_SET({0}, IFNULL(country, '')) > 0",
safeCountry)
.ge(startDate != null, DedupeTotalDataEntity::getCreatedAt,
startDate == null ? null : startDate.atStartOfDay())
.lt(endDate != null, DedupeTotalDataEntity::getCreatedAt,
@@ -134,13 +143,18 @@ public class DedupeTotalDataService {
public byte[] export(String username, LocalDate startDate, LocalDate endDate,
Long groupId, Long operatorId) {
return export(username, startDate, endDate, groupId, null, operatorId);
}
public byte[] export(String username, LocalDate startDate, LocalDate endDate,
Long groupId, String country, Long operatorId) {
if (startDate != null && endDate != null && startDate.isAfter(endDate)) {
throw new BusinessException("开始日期不能晚于结束日期");
}
String safeUsername = username == null ? "" : username.trim();
AccessScope scope = resolveAccessScope(operatorId);
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
buildExportWorkbook(outputStream, safeUsername, startDate, endDate, groupId, scope);
buildExportWorkbook(outputStream, safeUsername, startDate, endDate, groupId, country, scope);
return outputStream.toByteArray();
}
@@ -150,6 +164,16 @@ public class DedupeTotalDataService {
LocalDate endDate,
Long groupId,
Long operatorId) {
writeExport(outputStream, username, startDate, endDate, groupId, null, operatorId);
}
public void writeExport(OutputStream outputStream,
String username,
LocalDate startDate,
LocalDate endDate,
Long groupId,
String country,
Long operatorId) {
if (outputStream == null) {
throw new BusinessException("导出输出流不能为空");
}
@@ -158,7 +182,7 @@ public class DedupeTotalDataService {
}
String safeUsername = username == null ? "" : username.trim();
AccessScope scope = resolveAccessScope(operatorId);
buildExportWorkbook(outputStream, safeUsername, startDate, endDate, groupId, scope);
buildExportWorkbook(outputStream, safeUsername, startDate, endDate, groupId, country, scope);
}
public void writeMonthlyZipExport(OutputStream outputStream,
@@ -167,17 +191,28 @@ public class DedupeTotalDataService {
LocalDate endDate,
Long groupId,
Long operatorId) {
writeMonthlyZipExport(outputStream, username, startDate, endDate, groupId, null, operatorId);
}
public void writeMonthlyZipExport(OutputStream outputStream,
String username,
LocalDate startDate,
LocalDate endDate,
Long groupId,
String country,
Long operatorId) {
if (outputStream == null) {
throw new BusinessException("export output stream cannot be null");
}
if (startDate == null || endDate == null || YearMonth.from(startDate).equals(YearMonth.from(endDate))) {
writeExport(outputStream, username, startDate, endDate, groupId, operatorId);
writeExport(outputStream, username, startDate, endDate, groupId, country, operatorId);
return;
}
if (startDate.isAfter(endDate)) {
throw new BusinessException("invalid export date range");
}
String safeUsername = username == null ? "" : username.trim();
String safeCountry = country == null ? "" : country.trim();
AccessScope scope = resolveAccessScope(operatorId);
boolean groupPrevalidated = groupId != null && groupId > 0;
if (groupPrevalidated) {
@@ -194,7 +229,7 @@ public class DedupeTotalDataService {
LocalDate entryEndDate = endDate.isBefore(monthEnd) ? endDate : monthEnd;
zipOutputStream.putNextEntry(new ZipEntry(monthlyExportEntryName(currentMonth)));
buildExportWorkbook(zipOutputStream, safeUsername, entryStartDate, entryEndDate,
groupId, scope, groupPrevalidated);
groupId, safeCountry, scope, groupPrevalidated);
zipOutputStream.closeEntry();
currentMonth = currentMonth.plusMonths(1);
}
@@ -211,8 +246,9 @@ public class DedupeTotalDataService {
LocalDate startDate,
LocalDate endDate,
Long groupId,
String country,
AccessScope scope) {
buildExportWorkbook(outputStream, username, startDate, endDate, groupId, scope, false);
buildExportWorkbook(outputStream, username, startDate, endDate, groupId, country, scope, false);
}
private void buildExportWorkbook(OutputStream outputStream,
@@ -220,6 +256,7 @@ public class DedupeTotalDataService {
LocalDate startDate,
LocalDate endDate,
Long groupId,
String country,
AccessScope scope,
boolean groupPrevalidated) {
try (SXSSFWorkbook workbook = new SXSSFWorkbook(100)) {
@@ -227,14 +264,15 @@ public class DedupeTotalDataService {
Row header = sheet.createRow(0);
header.createCell(0).setCellValue("ID");
header.createCell(1).setCellValue("ASIN值");
header.createCell(2).setCellValue("用户名");
header.createCell(3).setCellValue("分组");
header.createCell(4).setCellValue("创建时间");
header.createCell(2).setCellValue("国家");
header.createCell(3).setCellValue("用户名");
header.createCell(4).setCellValue("分组");
header.createCell(5).setCellValue("创建时间");
Long lastId = null;
int rowIndex = 1;
while (true) {
LambdaQueryWrapper<DedupeTotalDataEntity> pageQuery = buildExportQuery(
username, startDate, endDate, groupId, scope, groupPrevalidated);
username, startDate, endDate, groupId, country, scope, groupPrevalidated);
if (lastId != null) {
pageQuery.lt(DedupeTotalDataEntity::getId, lastId);
}
@@ -248,11 +286,12 @@ public class DedupeTotalDataService {
Row row = sheet.createRow(rowIndex++);
row.createCell(0).setCellValue(entity.getId() == null ? "" : String.valueOf(entity.getId()));
row.createCell(1).setCellValue(entity.getDataValue() == null ? "" : entity.getDataValue());
row.createCell(2).setCellValue(entity.getUploaderUsername() == null ? "" : entity.getUploaderUsername());
row.createCell(3).setCellValue(entity.getGroupId() == null
row.createCell(2).setCellValue(entity.getCountry() == null ? "" : entity.getCountry());
row.createCell(3).setCellValue(entity.getUploaderUsername() == null ? "" : entity.getUploaderUsername());
row.createCell(4).setCellValue(entity.getGroupId() == null
? ""
: groupNames.getOrDefault(entity.getGroupId(), ""));
row.createCell(4).setCellValue(formatExportTime(entity.getCreatedAt()));
row.createCell(5).setCellValue(formatExportTime(entity.getCreatedAt()));
}
DedupeTotalDataEntity lastRow = rows.getLast();
lastId = lastRow == null ? null : lastRow.getId();
@@ -265,9 +304,10 @@ public class DedupeTotalDataService {
}
sheet.setColumnWidth(0, 3600);
sheet.setColumnWidth(1, 5200);
sheet.setColumnWidth(2, 5200);
sheet.setColumnWidth(2, 3200);
sheet.setColumnWidth(3, 5200);
sheet.setColumnWidth(4, 5600);
sheet.setColumnWidth(4, 5200);
sheet.setColumnWidth(5, 5600);
workbook.write(outputStream);
workbook.dispose();
} catch (Exception ex) {
@@ -280,18 +320,23 @@ public class DedupeTotalDataService {
LocalDate endDate,
Long groupId,
AccessScope scope) {
return buildExportQuery(username, startDate, endDate, groupId, scope, false);
return buildExportQuery(username, startDate, endDate, groupId, null, scope, false);
}
private LambdaQueryWrapper<DedupeTotalDataEntity> buildExportQuery(String username,
LocalDate startDate,
LocalDate endDate,
Long groupId,
String country,
AccessScope scope,
boolean groupPrevalidated) {
String safeUsername = username == null ? "" : username.trim();
String safeCountry = country == null ? "" : country.trim();
LambdaQueryWrapper<DedupeTotalDataEntity> query = new LambdaQueryWrapper<DedupeTotalDataEntity>()
.like(!safeUsername.isEmpty(), DedupeTotalDataEntity::getUploaderUsername, safeUsername)
.apply(!safeCountry.isEmpty(),
"FIND_IN_SET({0}, IFNULL(country, '')) > 0",
safeCountry)
.ge(startDate != null, DedupeTotalDataEntity::getCreatedAt,
startDate == null ? null : startDate.atStartOfDay())
.lt(endDate != null, DedupeTotalDataEntity::getCreatedAt,
@@ -608,9 +653,11 @@ public class DedupeTotalDataService {
if (asinIndex == null) {
throw new BusinessException("缺少 ASIN 列");
}
Integer countryIndex = headerMap.get("国家");
Set<String> seenInFile = new HashSet<>();
List<String> pendingValues = new ArrayList<>(IMPORT_BATCH_SIZE);
Map<String, String> pendingCountries = new HashMap<>();
int totalRows = Math.max(sheet.getLastRowNum(), 0);
int asinCount = 0;
int insertedCount = 0;
@@ -644,21 +691,26 @@ public class DedupeTotalDataService {
continue;
}
pendingValues.add(asin);
if (countryIndex != null) {
pendingCountries.put(asin, parseCountries(formatter.formatCellValue(row.getCell(countryIndex))));
}
if (pendingValues.size() >= IMPORT_BATCH_SIZE) {
ImportBatchResult batch = insertImportBatch(
pendingValues, groupId, uploaderUserId, uploaderUsername);
pendingValues, pendingCountries, groupId, uploaderUserId, uploaderUsername);
insertedCount += batch.insertedCount();
skippedCount += batch.skippedCount();
pendingValues.clear();
pendingCountries.clear();
}
updateImportProgress(progress, rowNum, asinCount, insertedCount, skippedCount);
}
if (!pendingValues.isEmpty()) {
ImportBatchResult batch = insertImportBatch(
pendingValues, groupId, uploaderUserId, uploaderUsername);
pendingValues, pendingCountries, groupId, uploaderUserId, uploaderUsername);
insertedCount += batch.insertedCount();
skippedCount += batch.skippedCount();
pendingValues.clear();
pendingCountries.clear();
}
updateImportProgress(progress, totalRows, asinCount, insertedCount, skippedCount);
@@ -676,6 +728,7 @@ public class DedupeTotalDataService {
}
private ImportBatchResult insertImportBatch(List<String> values,
Map<String, String> countriesByValue,
Long groupId,
Long uploaderUserId,
String uploaderUsername) {
@@ -690,6 +743,7 @@ public class DedupeTotalDataService {
}
DedupeTotalDataEntity entity = new DedupeTotalDataEntity();
entity.setDataValue(value);
entity.setCountry(countriesByValue == null ? null : countriesByValue.get(value));
entity.setGroupId(groupId);
entity.setUploaderUserId(uploaderUserId);
entity.setUploaderUsername(uploaderUsername);
@@ -1076,6 +1130,44 @@ public class DedupeTotalDataService {
.replaceAll("\\s+", " ");
}
private static final List<String> COUNTRY_CODE_MAP = List.of(
"DE", "UK", "FR", "IT", "ES",
"德国", "英国", "法国", "意大利", "西班牙");
private String normalizeCountryCode(String raw) {
String upper = normalizeExcelText(raw).toUpperCase(Locale.ROOT);
return upper
.replace("GERMANY", "DE")
.replace("UNITED KINGDOM", "UK")
.replace("UNITEDKINGDOM", "UK")
.replace("ENGLAND", "UK")
.replace("FRANCE", "FR")
.replace("ITALY", "IT")
.replace("SPAIN", "ES")
.replace("德国", "DE")
.replace("英国", "UK")
.replace("法国", "FR")
.replace("意大利", "IT")
.replace("西班牙", "ES");
}
/** 将国家列单元格内容解析为标准国家代码(去重后按出现顺序),无有效国家返回 null。 */
private String parseCountries(String raw) {
if (raw == null || raw.isBlank()) {
return null;
}
// 先对整个单元格做别名归一(含多词国家名如 "United Kingdom"),再按分隔符/空格拆分。
String normalized = normalizeCountryCode(raw);
LinkedHashSet<String> codes = new LinkedHashSet<>();
String[] tokens = normalized.split("[,;;、/\\\\|\\s]+");
for (String token : tokens) {
if (COUNTRY_CODE_MAP.contains(token)) {
codes.add(token);
}
}
return codes.isEmpty() ? null : String.join(",", codes);
}
private Map<Long, String> loadGroupNames(List<DedupeTotalDataEntity> rows) {
List<Long> groupIds = rows.stream()
.map(DedupeTotalDataEntity::getGroupId)
@@ -1098,6 +1190,7 @@ public class DedupeTotalDataService {
DedupeTotalDataItemVo vo = new DedupeTotalDataItemVo();
vo.setId(entity.getId());
vo.setDataValue(entity.getDataValue());
vo.setCountry(entity.getCountry());
vo.setGroupId(entity.getGroupId());
vo.setGroupName(groupName == null ? "" : groupName);
vo.setUploaderUserId(entity.getUploaderUserId());
@@ -0,0 +1,17 @@
-- Register the existing legacy static page in the APP permission menu.
-- The route_path is the menu slug; the page URL is /web_source/variant-collection.html.
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`, `parent_id`)
SELECT 'ASIN变体采集', 'variant-collection', 'app', 'variant-collection', 118,
(SELECT id FROM `columns` WHERE `column_key` = 'brand_front_tools' LIMIT 1)
WHERE NOT EXISTS (
SELECT 1 FROM `columns` WHERE `column_key` = 'variant-collection'
);
UPDATE `columns` AS child
JOIN `columns` AS parent ON parent.`column_key` = 'brand_front_tools'
SET child.`name` = 'ASIN变体采集',
child.`menu_type` = 'app',
child.`route_path` = 'variant-collection',
child.`sort_order` = 118,
child.`parent_id` = parent.`id`
WHERE child.`column_key` = 'variant-collection';
@@ -0,0 +1,33 @@
SET @country_col_exists := (
SELECT COUNT(*)
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'biz_dedupe_total_data'
AND COLUMN_NAME = 'country'
);
SET @sql_add_country := IF(
@country_col_exists = 0,
'ALTER TABLE biz_dedupe_total_data ADD COLUMN country VARCHAR(64) DEFAULT NULL COMMENT ''国家代码,逗号分隔,如 DE,UK'' AFTER data_value',
'SELECT 1'
);
PREPARE stmt_add_country FROM @sql_add_country;
EXECUTE stmt_add_country;
DEALLOCATE PREPARE stmt_add_country;
SET @country_idx_exists := (
SELECT COUNT(*)
FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'biz_dedupe_total_data'
AND INDEX_NAME = 'idx_country'
);
SET @sql_add_country_idx := IF(
@country_idx_exists = 0,
'ALTER TABLE biz_dedupe_total_data ADD INDEX idx_country (country)',
'SELECT 1'
);
PREPARE stmt_add_country_idx FROM @sql_add_country_idx;
EXECUTE stmt_add_country_idx;
DEALLOCATE PREPARE stmt_add_country_idx;
@@ -56,6 +56,7 @@ class DedupeTotalDataControllerTest {
eq(LocalDate.of(2026, 7, 1)),
eq(LocalDate.of(2026, 7, 31)),
eq(3L),
isNull(),
eq(8L));
ResponseEntity<StreamingResponseBody> response = controller.export(
@@ -63,6 +64,7 @@ class DedupeTotalDataControllerTest {
LocalDate.of(2026, 7, 1),
LocalDate.of(2026, 7, 31),
3L,
null,
request);
assertNotNull(response.getBody());
@@ -84,9 +86,10 @@ class DedupeTotalDataControllerTest {
eq(LocalDate.of(2026, 7, 1)),
eq(LocalDate.of(2026, 7, 31)),
eq(3L),
isNull(),
eq(8L));
verify(service, never()).writeMonthlyZipExport(
any(OutputStream.class), any(), any(), any(), any(), any());
any(OutputStream.class), any(), any(), any(), any(), any(), any());
verifyNoInteractions(permissionMenuService);
}
@@ -109,6 +112,7 @@ class DedupeTotalDataControllerTest {
null,
LocalDate.of(2026, 8, 2),
3L,
null,
request);
assertEquals(MediaType.parseMediaType(
@@ -125,9 +129,10 @@ class DedupeTotalDataControllerTest {
isNull(),
eq(LocalDate.of(2026, 8, 2)),
eq(3L),
isNull(),
eq(8L));
verify(service, never()).writeMonthlyZipExport(
any(OutputStream.class), any(), any(), any(), any(), any());
any(OutputStream.class), any(), any(), any(), any(), any(), any());
}
@Test
@@ -153,6 +158,7 @@ class DedupeTotalDataControllerTest {
eq(LocalDate.of(2026, 7, 15)),
eq(LocalDate.of(2026, 8, 2)),
eq(3L),
isNull(),
eq(8L));
ResponseEntity<StreamingResponseBody> response = controller.export(
@@ -160,6 +166,7 @@ class DedupeTotalDataControllerTest {
LocalDate.of(2026, 7, 15),
LocalDate.of(2026, 8, 2),
3L,
null,
request);
assertEquals(MediaType.parseMediaType("application/zip"), response.getHeaders().getContentType());
@@ -176,8 +183,9 @@ class DedupeTotalDataControllerTest {
eq(LocalDate.of(2026, 7, 15)),
eq(LocalDate.of(2026, 8, 2)),
eq(3L),
isNull(),
eq(8L));
verify(service, never()).writeExport(any(OutputStream.class), any(), any(), any(), any(), any());
verify(service, never()).writeExport(any(OutputStream.class), any(), any(), any(), any(), any(), any());
}
@Test
@@ -194,6 +202,7 @@ class DedupeTotalDataControllerTest {
LocalDate.of(2026, 8, 2),
LocalDate.of(2026, 7, 15),
3L,
null,
request));
assertEquals(400, exception.getCode());
@@ -21,6 +21,7 @@ import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.apache.ibatis.builder.MapperBuilderAssistant;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.test.util.ReflectionTestUtils;
@@ -188,6 +189,59 @@ class DedupeTotalDataServiceTest {
verify(dedupeTotalDataMapper, never()).insert(any(DedupeTotalDataEntity.class));
}
@Test
void importReadsAndNormalizesCountryColumn() throws Exception {
when(adminUserMapper.selectById(23L)).thenReturn(user(23L, "normal", "member-a"));
stubWritableGroup(23L, 7L);
when(dedupeTotalDataMapper.selectExistingDataValues(List.of("B012345678"))).thenReturn(List.of());
when(dedupeTotalDataMapper.insertBatchIgnore(any())).thenReturn(1);
MockMultipartFile file = asinCountryWorkbook("B012345678", "德国, US,United Kingdom");
var result = service.importFromExcel(file, 7L, 23L);
assertEquals(1, result.getInsertedCount());
ArgumentCaptor<List<DedupeTotalDataEntity>> captor = ArgumentCaptor.forClass(List.class);
verify(dedupeTotalDataMapper).insertBatchIgnore(captor.capture());
// US 不在五国范围内,应被过滤掉
assertEquals("DE,UK", captor.getValue().getFirst().getCountry());
}
@Test
void importKeepsCountryBlankWhenHeaderMissing() throws Exception {
when(adminUserMapper.selectById(23L)).thenReturn(user(23L, "normal", "member-a"));
stubWritableGroup(23L, 7L);
when(dedupeTotalDataMapper.selectExistingDataValues(List.of("B012345678"))).thenReturn(List.of());
when(dedupeTotalDataMapper.insertBatchIgnore(any())).thenReturn(1);
var result = service.importFromExcel(asinWorkbook("B012345678"), 7L, 23L);
assertEquals(1, result.getInsertedCount());
ArgumentCaptor<List<DedupeTotalDataEntity>> captor = ArgumentCaptor.forClass(List.class);
verify(dedupeTotalDataMapper).insertBatchIgnore(captor.capture());
assertEquals(null, captor.getValue().getFirst().getCountry());
}
@Test
@SuppressWarnings({"rawtypes", "unchecked"})
void pageFiltersByCountryWithFindInSet() {
TableInfoHelper.initTableInfo(
new MapperBuilderAssistant(new MybatisConfiguration(), ""),
DedupeTotalDataEntity.class);
when(adminUserMapper.selectById(1L)).thenReturn(user(1L, "super_admin", "root"));
when(dedupeTotalDataMapper.selectCount(any())).thenReturn(0L);
when(dedupeTotalDataMapper.selectList(any())).thenReturn(List.of());
service.page(1, 15, "", "", null, null, null, "UK", 1L);
ArgumentCaptor<LambdaQueryWrapper<DedupeTotalDataEntity>> queryCaptor =
ArgumentCaptor.forClass((Class) LambdaQueryWrapper.class);
verify(dedupeTotalDataMapper).selectCount(queryCaptor.capture());
LambdaQueryWrapper<DedupeTotalDataEntity> query = queryCaptor.getValue();
query.getSqlSegment();
assertTrue(query.getParamNameValuePairs().containsValue("UK"));
assertTrue(query.getCustomSqlSegment().contains("FIND_IN_SET"));
}
@Test
void concurrentDuplicateDuringImportIsSkipped() throws Exception {
when(adminUserMapper.selectById(23L)).thenReturn(user(23L, "normal", "member-a"));
@@ -294,6 +348,7 @@ class DedupeTotalDataServiceTest {
when(shopManageGroupMapper.selectManagedMemberUserIds(10L)).thenReturn(List.of(23L));
DedupeTotalDataEntity entity = data(91L, 23L);
entity.setUploaderUsername("member-a");
entity.setCountry("DE,UK");
entity.setCreatedAt(LocalDateTime.of(2026, 7, 20, 12, 30));
when(dedupeTotalDataMapper.selectList(any())).thenReturn(List.of(entity));
@@ -305,11 +360,13 @@ class DedupeTotalDataServiceTest {
try (XSSFWorkbook workbook = new XSSFWorkbook(new ByteArrayInputStream(bytes))) {
var sheet = workbook.getSheetAt(0);
assertEquals("国家", sheet.getRow(0).getCell(2).getStringCellValue());
assertEquals("ASIN值", sheet.getRow(0).getCell(1).getStringCellValue());
assertEquals("B012345678", sheet.getRow(1).getCell(1).getStringCellValue());
assertEquals("member-a", sheet.getRow(1).getCell(2).getStringCellValue());
assertEquals("", sheet.getRow(1).getCell(3).getStringCellValue());
assertEquals("2026-07-20 12:30:00", sheet.getRow(1).getCell(4).getStringCellValue());
assertEquals("DE,UK", sheet.getRow(1).getCell(2).getStringCellValue());
assertEquals("member-a", sheet.getRow(1).getCell(3).getStringCellValue());
assertEquals("", sheet.getRow(1).getCell(4).getStringCellValue());
assertEquals("2026-07-20 12:30:00", sheet.getRow(1).getCell(5).getStringCellValue());
}
verify(shopManageGroupMapper).selectManagedMemberUserIds(10L);
@@ -548,4 +605,23 @@ class DedupeTotalDataServiceTest {
output.toByteArray());
}
}
private MockMultipartFile asinCountryWorkbook(String asin, String country) throws Exception {
try (XSSFWorkbook workbook = new XSSFWorkbook();
ByteArrayOutputStream output = new ByteArrayOutputStream()) {
var sheet = workbook.createSheet("data");
Row header = sheet.createRow(0);
header.createCell(0).setCellValue("ASIN");
header.createCell(1).setCellValue("国家");
Row row = sheet.createRow(1);
row.createCell(0).setCellValue(asin);
row.createCell(1).setCellValue(country);
workbook.write(output);
return new MockMultipartFile(
"file",
"data.xlsx",
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
output.toByteArray());
}
}
}
+7
View File
@@ -2929,6 +2929,7 @@ def _format_dedupe_total_data_item(item):
return {
'id': item.get('id'),
'data_value': item.get('dataValue') or '',
'country': item.get('country') or '',
'group_id': item.get('groupId'),
'group_name': item.get('groupName') or '',
'uploader_user_id': item.get('uploaderUserId'),
@@ -2948,6 +2949,7 @@ def list_dedupe_total_data():
keyword = (request.args.get('keyword') or '').strip()
username = (request.args.get('username') or '').strip()
group_id = _parse_positive_group_id(request.args.get('group_id') or request.args.get('groupId'))
country = (request.args.get('country') or '').strip()
start_date = (request.args.get('start_date') or request.args.get('startDate') or '').strip()
end_date = (request.args.get('end_date') or request.args.get('endDate') or '').strip()
params = {
@@ -2957,6 +2959,8 @@ def list_dedupe_total_data():
'username': username,
'operatorId': current_row.get('id'),
}
if country:
params['country'] = country
if start_date:
params['startDate'] = start_date
if end_date:
@@ -2993,10 +2997,13 @@ def export_dedupe_total_data():
return denied
params = {'operatorId': current_row.get('id')}
username = (request.args.get('username') or '').strip()
country = (request.args.get('country') or '').strip()
start_date = (request.args.get('start_date') or request.args.get('startDate') or '').strip()
end_date = (request.args.get('end_date') or request.args.get('endDate') or '').strip()
if username:
params['username'] = username
if country:
params['country'] = country
if start_date:
params['startDate'] = start_date
if end_date:
+20 -4
View File
@@ -2180,14 +2180,28 @@
var keyword = (document.getElementById('searchDedupeTotalData').value || '').trim();
var username = (document.getElementById('searchDedupeTotalDataUsername').value || '').trim();
var groupId = (document.getElementById('dedupeTotalDataGroupFilterId').value || '').trim();
var country = (document.getElementById('dedupeTotalDataCountryFilter').value || '').trim();
var dateRange = getDedupeTotalDataDateRange();
if (keyword) q += '&keyword=' + encodeURIComponent(keyword);
if (username) q += '&username=' + encodeURIComponent(username);
if (groupId) q += '&group_id=' + encodeURIComponent(groupId);
if (country) q += '&country=' + encodeURIComponent(country);
if (dateRange.startDate) q += '&start_date=' + encodeURIComponent(dateRange.startDate);
if (dateRange.endDate) q += '&end_date=' + encodeURIComponent(dateRange.endDate);
return q;
}
function getDedupeTotalDataCountryLabel(countryCodes) {
var countryLabels = {
DE: '德国', UK: '英国', FR: '法国', IT: '意大利', ES: '西班牙'
};
if (Array.isArray(countryCodes)) {
return countryCodes.map(function (code) { return countryLabels[code] || code; }).join('、');
}
return String(countryCodes || '').split(/[,]/).map(function (code) {
code = code.trim();
return countryLabels[code] || code;
}).filter(Boolean).join('、');
}
function loadDedupeTotalData(page) {
if (!validateDedupeTotalDataDateRange()) return;
dedupeTotalDataPage = page || 1;
@@ -2196,15 +2210,15 @@
.then(function (res) {
var tbody = document.getElementById('dedupeTotalDataListBody');
if (!res.success) {
tbody.innerHTML = '<tr><td colspan="6" class="empty-tip">加载失败: ' + escapeHtml(res.error || '') + '</td></tr>';
tbody.innerHTML = '<tr><td colspan="7" class="empty-tip">加载失败: ' + escapeHtml(res.error || '') + '</td></tr>';
return;
}
var items = res.items || [];
if (items.length === 0) {
tbody.innerHTML = '<tr><td colspan="6" class="empty-tip">暂无总数据</td></tr>';
tbody.innerHTML = '<tr><td colspan="7" class="empty-tip">暂无总数据</td></tr>';
} else {
tbody.innerHTML = items.map(function (item) {
return '<tr><td>' + escapeHtml(item.id) + '</td><td>' + escapeHtml(item.data_value || '') + '</td><td>' + escapeHtml(item.username || '') + '</td><td>' + escapeHtml(item.group_name || '未分组') + '</td><td>' + escapeHtml(item.created_at || '') + '</td><td>' +
return '<tr><td>' + escapeHtml(item.id) + '</td><td>' + escapeHtml(item.data_value || '') + '</td><td>' + escapeHtml(getDedupeTotalDataCountryLabel(item.country)) + '</td><td>' + escapeHtml(item.username || '') + '</td><td>' + escapeHtml(item.group_name || '未分组') + '</td><td>' + escapeHtml(item.created_at || '') + '</td><td>' +
'<button class="btn btn-sm" data-dedupe-total-edit="' + escapeHtml(item.id) + '" data-value="' + escapeHtml(item.data_value || '') + '" data-group-id="' + escapeHtml(item.group_id || '') + '">编辑</button> ' +
'<button class="btn btn-sm btn-danger" data-dedupe-total-delete="' + escapeHtml(item.id) + '" data-value="' + escapeHtml(item.data_value || '') + '">删除</button>' +
'</td></tr>';
@@ -2214,7 +2228,7 @@
bindDedupeTotalDataActions();
})
.catch(function () {
document.getElementById('dedupeTotalDataListBody').innerHTML = '<tr><td colspan="6" class="empty-tip">请求失败</td></tr>';
document.getElementById('dedupeTotalDataListBody').innerHTML = '<tr><td colspan="7" class="empty-tip">请求失败</td></tr>';
});
}
function bindDedupeTotalDataActions() {
@@ -2268,11 +2282,13 @@
if (dedupeTotalDataExportButton.disabled) return;
var username = (document.getElementById('searchDedupeTotalDataUsername').value || '').trim();
var groupId = (document.getElementById('dedupeTotalDataGroupFilterId').value || '').trim();
var country = (document.getElementById('dedupeTotalDataCountryFilter').value || '').trim();
var dateRange = getDedupeTotalDataDateRange();
if (!validateDedupeTotalDataDateRange()) return;
var params = [];
if (username) params.push('username=' + encodeURIComponent(username));
if (groupId) params.push('group_id=' + encodeURIComponent(groupId));
if (country) params.push('country=' + encodeURIComponent(country));
if (dateRange.startDate) params.push('start_date=' + encodeURIComponent(dateRange.startDate));
if (dateRange.endDate) params.push('end_date=' + encodeURIComponent(dateRange.endDate));
var originalButtonText = dedupeTotalDataExportButton.textContent;
+38 -4
View File
@@ -4,7 +4,7 @@
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>管理后台 - 南日AI</title>
<title>管理后台 - 数富AI</title>
<style>
* {
margin: 0;
@@ -182,6 +182,24 @@
white-space: nowrap;
}
.dedupe-filter-date-row {
display: flex;
gap: 12px;
align-items: flex-end;
flex-wrap: wrap;
margin-bottom: 16px;
}
.dedupe-filter-date-row .form-group {
flex: 0 0 200px;
min-width: 0;
margin-bottom: 0;
}
.dedupe-filter-date-row .dedupe-filter-actions {
margin-left: auto;
}
.btn {
padding: 10px 20px;
background: #667eea;
@@ -2030,7 +2048,7 @@
<h3 style="margin-bottom:16px;font-size:15px;">新增ASIN</h3>
<div class="form-row">
<div class="form-group" style="min-width:320px;">
<label>上传 Excel 文件(读取 ASIN 列)</label>
<label>上传 Excel 文件(读取 ASIN 列,可选 国家 列</label>
<input type="file" id="dedupeTotalDataFile" accept=".xlsx,.xls">
</div>
<div class="form-group" style="min-width:220px;">
@@ -2091,6 +2109,22 @@
<option value="">全部分组</option>
</select>
</div>
<div class="form-group">
<label>国家</label>
<select id="dedupeTotalDataCountryFilter">
<option value="">全部国家</option>
<option value="DE">德国</option>
<option value="UK">英国</option>
<option value="FR">法国</option>
<option value="IT">意大利</option>
<option value="ES">西班牙</option>
</select>
</div>
<div class="dedupe-filter-actions">
<button class="btn" id="btnSearchDedupeTotalData">查询</button>
</div>
</div>
<div class="dedupe-filter-date-row">
<div class="form-group">
<label>开始日期</label>
<input type="date" id="exportDedupeTotalDataStartDate">
@@ -2100,7 +2134,6 @@
<input type="date" id="exportDedupeTotalDataEndDate">
</div>
<div class="dedupe-filter-actions">
<button class="btn" id="btnSearchDedupeTotalData">查询</button>
<button class="btn btn-secondary" id="btnExportDedupeTotalData" type="button">导出 XLSX</button>
</div>
</div>
@@ -2109,6 +2142,7 @@
<tr>
<th>ID</th>
<th>ASIN值</th>
<th>国家</th>
<th>用户名</th>
<th>分组</th>
<th>创建时间</th>
@@ -3175,7 +3209,7 @@
</div>
</div>
</div>
<script src="/static/admin.js?v=dedupe-export-wait-1"></script>
<script src="/static/admin.js?v=dedupe-country-2"></script>
</body>
</html>
+2 -2
View File
@@ -3,7 +3,7 @@
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>登录 - 南日AI</title>
<title>登录 - 数富AI</title>
<style>
* { margin: 0; padding: 0; box-sizing: border-box; }
body {
@@ -96,7 +96,7 @@
</head>
<body>
<header class="header">
<span class="header-title">南日AI</span>
<span class="header-title">数富AI</span>
</header>
<div class="login-box">