feat(keyword-share): 新增客户端批量同步端点,保住真实使用人/时间与四级类目
工作台「关键词使用记录」与后台管理对不上,根因不是链路不通(后台现存词 426 与桌面 426 逐字相等),而是「谁、何时」两个字段在推拉两个方向上各丢一次: - 推:markInternal 把 user_name/used_time 硬写成 token 账号与服务器此刻, 历史使用记录推上去一律变成「同步账号 + 同步那一刻」; - 拉:selectRows/selectRandomUnused 投影 u.user_name,而 VO 属性叫 usedUser, 驼峰映射只会认成不存在的 userName ⇒ GET /keywords 的 usedUser 恒为 null; - 词本体:add() 从不写四级类目,且 keyword_share 侧只有 INSERT IGNORE、 没有任何 UPSERT,重复词静默 no-op。 处置: - 修 used_user 别名(两处投影); - 新增 KeywordShareMapper#upsertBatchFromSync,类目/chinese 按 COALESCE(NULLIF(VALUES(x),''),x) 非空才覆盖,更新分支不碰 user_name/add_time (那是历史归属,不由同步改写); - 新增 POST /api/keyword-share/sync/import:每行携带 usedUser/usedTime, 绕开 markInternal 复用现成 KeywordUsageMapper#upsertBatch(本就是 =VALUES(...)), 不改表结构、不加 Flyway 迁移; - used 取三态:null=不改动使用状态。按 false 处理会让「只同步词库」的一次 调用把后台已有使用记录整批删掉; - 共享 API 账号下的归属失真用服务端名单收口:命中 aiimage.keyword-share-sync .used-users 才按姓名落库且落名单里那份写法(全/半角括号归一,否则同一人被 后管拆成两个使用者);strict 默认 false,名单未填时放行并计 usedUserUnknown, 避免首次上线把全部记录静默改写成账号名。 顺带:CatchSilenceRatchetTest 要求新增 catch 必须同批净减,而宽松时间解析需要 try/catch ⇒ 把 ShopDataCrawlAdminTasksService 已有的同形状实现抽成 common/support/LooseLocalDateTimeParser 两处共用,净新增 catch 为 0,未放宽基线。 尺子:XML_STATEMENT_TOTAL 140→141,NOT_MIGRATED 点名新 insert(非 select, XML_SELECT_TOTAL 不动)。 验证:mvn -o test-compile rc=0;后端整面 mvn -o test 4892 passed/0 failed/18 skipped。 本机无 MySQL,端到端未跑,真链路待客户端联调(ops BACKLOG B44)。
This commit is contained in:
+51
@@ -0,0 +1,51 @@
|
||||
package com.nanri.aiimage.common.support;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.time.format.DateTimeParseException;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 宽松本地日期时间解析:兼容 {@code T}/空格分隔、可选秒与小时、尾随 {@code Z} 与小数秒。
|
||||
*
|
||||
* <p>存在的理由是「调用方给的从来不是同一个时间形状」:Flask 侧的管理接口会传
|
||||
* {@code 2026-10-25 22:59:26},Python 客户端 {@code isoformat()} 会传 {@code 2026-10-25T22:59:26},
|
||||
* 带毫秒和 UTC 后缀的也见过。收在一处,是为了让"试几种格式"这件事只有一份实现、只有一处 try/catch。</p>
|
||||
*
|
||||
* <p>解析不出来返回 {@code null},不抛异常——要不要把 null 变成 400、还是降级成服务器此刻,
|
||||
* 是调用方的业务判断,不该由解析器替它决定。</p>
|
||||
*/
|
||||
public final class LooseLocalDateTimeParser {
|
||||
|
||||
/** 按优先级逐个试;顺序即语义(先信息全的后退化到只到日)。 */
|
||||
private static final List<DateTimeFormatter> FORMATS = List.of(
|
||||
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"),
|
||||
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm"),
|
||||
DateTimeFormatter.ofPattern("yyyy-MM-dd HH"),
|
||||
DateTimeFormatter.ofPattern("yyyy-MM-dd"));
|
||||
|
||||
private LooseLocalDateTimeParser() {
|
||||
}
|
||||
|
||||
/**
|
||||
* @return 解析结果;{@code raw} 为空或任何格式都对不上时返回 {@code null}
|
||||
*/
|
||||
public static LocalDateTime parseOrNull(String raw) {
|
||||
String value = raw == null ? "" : raw.trim();
|
||||
if (value.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
String normalized = value.replace('T', ' ')
|
||||
.replaceAll("[Zz]$", "")
|
||||
.replaceAll("\\.\\d+$", "")
|
||||
.trim();
|
||||
for (DateTimeFormatter formatter : FORMATS) {
|
||||
try {
|
||||
return LocalDateTime.parse(normalized, formatter);
|
||||
} catch (DateTimeParseException ignored) {
|
||||
// 尝试下一种格式
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package com.nanri.aiimage.config;
|
||||
|
||||
import lombok.Data;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 五国关键词客户端同步({@code /api/keyword-share/sync/import})的服务端参数。
|
||||
*
|
||||
* <p>存在的主要理由:工作台按对接文档口径使用**共享专用 API 账号**,而 {@code keyword_usage.user_name}
|
||||
* 是「谁用了哪个词」的唯一出处。若一律取 token 账号名,多人使用会被记成同一个人,后台按使用者筛选
|
||||
* 就失去意义;若无条件信任客户端自报姓名,又会把「阿武(扶金凡)/阿武(吴棋龙)」这类同音异写
|
||||
* 直接灌进留痕表。名单校验是这两者之间的收口:命中名单才按自报姓名落库。</p>
|
||||
*/
|
||||
@Data
|
||||
@ConfigurationProperties(prefix = "aiimage.keyword-share-sync")
|
||||
public class KeywordShareSyncProperties {
|
||||
|
||||
/**
|
||||
* 可信操作人姓名名单(真人显示名,与 {@code keyword_usage.user_name} 同形)。
|
||||
* 比对时两侧都做 trim + 全角括号归一,所以名单里写「阿武(吴棋龙)」也能匹配半角写法。
|
||||
*/
|
||||
private List<String> usedUsers = new ArrayList<>();
|
||||
|
||||
/**
|
||||
* 名单外姓名的处置。
|
||||
* <ul>
|
||||
* <li>{@code false}(默认):放行自报姓名,只在响应里计数回报——首次上线时名单还没填,
|
||||
* 直接回退会把全部记录静默改写成账号名,比现状更糟;</li>
|
||||
* <li>{@code true}:名单外一律回退为本次调用的账号名,并计入 {@code usedUserFallback}。</li>
|
||||
* </ul>
|
||||
*/
|
||||
private boolean strict = false;
|
||||
}
|
||||
@@ -4,6 +4,6 @@ import org.springframework.boot.context.properties.EnableConfigurationProperties
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
@Configuration
|
||||
@EnableConfigurationProperties({OssProperties.class, TransientStorageProperties.class, StorageProperties.class, BrandProgressProperties.class, DeleteBrandProgressProperties.class, BrandCheckProperties.class, ZiniaoProperties.class, ModuleCleanupProperties.class, TaskPressureProperties.class, TaskImageCacheCleanupProperties.class, AppearancePatentProperties.class, SimilarAsinProperties.class, ImageVideoProperties.class, InstanceRoutingProperties.class, CapacityPlanProperties.class, UserSecretProperties.class, NotificationProperties.class, DeviceLogOssProperties.class, CollectDataBrandRecheckProperties.class, ClientCredentialProperties.class, CostGuardProperties.class})
|
||||
@EnableConfigurationProperties({OssProperties.class, TransientStorageProperties.class, StorageProperties.class, BrandProgressProperties.class, DeleteBrandProgressProperties.class, BrandCheckProperties.class, ZiniaoProperties.class, ModuleCleanupProperties.class, TaskPressureProperties.class, TaskImageCacheCleanupProperties.class, AppearancePatentProperties.class, SimilarAsinProperties.class, ImageVideoProperties.class, InstanceRoutingProperties.class, CapacityPlanProperties.class, UserSecretProperties.class, NotificationProperties.class, DeviceLogOssProperties.class, CollectDataBrandRecheckProperties.class, ClientCredentialProperties.class, CostGuardProperties.class, KeywordShareSyncProperties.class})
|
||||
public class PropertiesConfig {
|
||||
}
|
||||
|
||||
+20
@@ -5,6 +5,7 @@ import com.nanri.aiimage.common.model.entity.AdminUserEntity;
|
||||
import com.nanri.aiimage.common.security.AdminAuthSupport;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.dto.KeywordShareAddRequest;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.dto.KeywordShareDeleteRequest;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.dto.KeywordShareSyncImportRequest;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.dto.KeywordUsageBatchRequest;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.vo.KeywordShareAddResultVo;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.vo.KeywordShareDeleteResultVo;
|
||||
@@ -14,6 +15,7 @@ import com.nanri.aiimage.modules.keywordshare.model.vo.KeywordShareMarkResultVo;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.vo.KeywordSharePageVo;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.vo.KeywordShareRowVo;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.vo.KeywordShareStatsGroupedVo;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.vo.KeywordShareSyncResultVo;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.vo.KeywordShareUnmarkResultVo;
|
||||
import com.nanri.aiimage.modules.keywordshare.service.KeywordShareService;
|
||||
import com.nanri.aiimage.modules.permission.service.PermissionMenuService;
|
||||
@@ -154,6 +156,24 @@ public class KeywordShareController {
|
||||
country, category, subCategory, categoryL3, categoryL4, search, count));
|
||||
}
|
||||
|
||||
/**
|
||||
* 客户端全量同步导入。与 {@code POST /keywords} + {@code /usage/mark} 的分工:那两条是网页侧
|
||||
* 交互式单条操作(使用人/时间取服务端此刻),这条专供工作台将本地历史批量搬入,
|
||||
* 必须保住客户端传来的四级类目与真实使用人/使用时间。
|
||||
*/
|
||||
@PostMapping("/sync/import")
|
||||
@Operation(summary = "客户端批量同步导入",
|
||||
description = "按 (company_id, country, keyword) UPSERT 词本体并同步使用状态,"
|
||||
+ "保留客户端传入的 usedTime/usedUser;used=false 删除对应使用记录。"
|
||||
+ "幂等,单请求不超过 2000 行,失败只需重传该批。")
|
||||
public ApiResponse<KeywordShareSyncResultVo> syncImport(
|
||||
HttpServletRequest request,
|
||||
@Valid @RequestBody KeywordShareSyncImportRequest body) {
|
||||
Operator operator = requireKeywordShareAccess(request);
|
||||
return ApiResponse.success("同步完成",
|
||||
keywordShareService.syncImport(body, operator.companyId(), operator.userName()));
|
||||
}
|
||||
|
||||
/**
|
||||
* 逐端点把门:登录态 + 后台菜单权限;公司 id 与用户名都从登录人实体取,前端传参不参与归属判定。
|
||||
*/
|
||||
|
||||
+6
@@ -59,4 +59,10 @@ public interface KeywordShareMapper extends BaseMapper<KeywordShareEntity> {
|
||||
|
||||
/** 批量 INSERT IGNORE:命中唯一键 uk_comp_cw 的重复行静默跳过,兜底与预检双保险。 */
|
||||
int insertBatchIgnore(@Param("rows") List<KeywordShareEntity> rows);
|
||||
|
||||
/**
|
||||
* 客户端全量同步专用 UPSERT:新词整行插入,已存在的词只把非空的四级类目/chinese/ktype/source
|
||||
* 覆盖进去,添加人与添加时间不动。与 {@link #insertBatchIgnore} 的区别正是这条会更新已有行。
|
||||
*/
|
||||
int upsertBatchFromSync(@Param("rows") List<KeywordShareEntity> rows);
|
||||
}
|
||||
|
||||
+80
@@ -0,0 +1,80 @@
|
||||
package com.nanri.aiimage.modules.keywordshare.model.dto;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import jakarta.validation.Valid;
|
||||
import jakarta.validation.constraints.NotEmpty;
|
||||
import jakarta.validation.constraints.Size;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 客户端全量同步导入请求。与 {@link KeywordShareAddRequest} 的分工:那条只补新词、由服务端
|
||||
* 定添加人与时间;这条要把客户端本地那份四级类目与真实使用历史原样搬进主库。
|
||||
*/
|
||||
@Data
|
||||
@Schema(description = "五国关键词客户端批量同步导入请求")
|
||||
public class KeywordShareSyncImportRequest {
|
||||
|
||||
@Schema(description = "只统计不写库,用于导入前对账", example = "false")
|
||||
private Boolean dryRun;
|
||||
|
||||
@Schema(description = "是否为新词写 keyword_add_log 留痕;几十万级全量导入默认不写,避免刷屏",
|
||||
example = "false")
|
||||
private Boolean writeAddLogs;
|
||||
|
||||
@Valid
|
||||
@NotEmpty(message = "items 不能为空")
|
||||
@Size(max = KeywordShareSyncImportRequest.MAX_ITEMS, message = "items 单次不超过 2000 条")
|
||||
@Schema(description = "待同步的词行列表", requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
private List<Item> items;
|
||||
|
||||
/** 单请求上限:2000 行 × 12 列约 0.6–1MB,留出网关与请求体日志的余量。 */
|
||||
public static final int MAX_ITEMS = 2000;
|
||||
|
||||
@Data
|
||||
@Schema(description = "单个待同步词行")
|
||||
public static class Item {
|
||||
|
||||
@Schema(description = "国家", requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
private String country;
|
||||
|
||||
@Schema(description = "关键词", requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
private String keyword;
|
||||
|
||||
@Schema(description = "中文释义;传空不会清空主库已有值")
|
||||
private String chinese;
|
||||
|
||||
@Schema(description = "关键词类型;传空不会清空主库已有值")
|
||||
private String ktype;
|
||||
|
||||
@Schema(description = "来源;传空不会清空主库已有值")
|
||||
private String source;
|
||||
|
||||
@Schema(description = "一级类目;传空不会清空主库已有值")
|
||||
private String category;
|
||||
|
||||
@Schema(description = "二级类目;传空不会清空主库已有值")
|
||||
private String subCategory;
|
||||
|
||||
@Schema(description = "三级类目;传空不会清空主库已有值")
|
||||
private String categoryL3;
|
||||
|
||||
@Schema(description = "四级类目;传空不会清空主库已有值")
|
||||
private String categoryL4;
|
||||
|
||||
@Schema(description = "使用状态三态:true 写/刷 keyword_usage,false 删该行,"
|
||||
+ "**不传(null)=完全不改动该词的使用状态**。只同步词库时必须整批省略此字段,"
|
||||
+ "否则会把后台已有使用记录删掉",
|
||||
requiredMode = Schema.RequiredMode.NOT_REQUIRED)
|
||||
private Boolean used;
|
||||
|
||||
@Schema(description = "真实使用人;命中服务端名单才落库,否则回退为本次调用的账号名",
|
||||
example = "阿武(吴棋龙)")
|
||||
private String usedUser;
|
||||
|
||||
@Schema(description = "真实使用时间,ISO-8601 本地日期时间;服务端按此保留历史时刻而非同步时刻",
|
||||
example = "2026-10-25T22:59:26")
|
||||
private String usedTime;
|
||||
}
|
||||
}
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
package com.nanri.aiimage.modules.keywordshare.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
/**
|
||||
* 客户端批量同步导入结果。计数是观测值而非事务保证:预检与写入之间有并发窗口,
|
||||
* 同一批被两个客户端同时推时 added/updated 会漂,但落库内容幂等。
|
||||
*/
|
||||
@Data
|
||||
@Schema(description = "五国关键词客户端批量同步导入结果")
|
||||
public class KeywordShareSyncResultVo {
|
||||
|
||||
@Schema(description = "新插入的词行数")
|
||||
private int added;
|
||||
|
||||
@Schema(description = "命中唯一键、被刷了类目/释义的既有词行数")
|
||||
private int updated;
|
||||
|
||||
@Schema(description = "整行丢弃数(国家或关键词为空、批内重复、字段超长)")
|
||||
private int skipped;
|
||||
|
||||
@Schema(description = "本批请求写入或刷新 keyword_usage 的行数(dryRun 下为预计数)")
|
||||
private int usageUpserted;
|
||||
|
||||
@Schema(description = "本批请求按 used=false 取消使用的行数(dryRun 下为预计数;"
|
||||
+ "实际删除条数不会多于它,主库可能本就没有该 usage 行)")
|
||||
private int usageDeleted;
|
||||
|
||||
@Schema(description = "词已入库但使用状态被丢弃的行数(usedTime 解析失败或早于下限)")
|
||||
private int usageSkipped;
|
||||
|
||||
@Schema(description = "usedUser 未命中服务端名单、回退为调用账号名的行数(仅 strict=true 时会产生)")
|
||||
private int usedUserFallback;
|
||||
|
||||
@Schema(description = "usedUser 自报姓名不在名单内的行数;非 strict 下照原样落库但仍计数,"
|
||||
+ "供运维据此判断名单是否该补齐、能否切 strict")
|
||||
private int usedUserUnknown;
|
||||
|
||||
@Schema(description = "usedTime 缺失、回退为服务端当前时间的行数")
|
||||
private int usedTimeFallback;
|
||||
|
||||
@Schema(description = "usedTime 落在未来、被钳到当前时间的行数")
|
||||
private int usedTimeClamped;
|
||||
|
||||
@Schema(description = "写入 keyword_add_log 留痕的行数(writeAddLogs=true 时才可能非 0)")
|
||||
private int addLogged;
|
||||
|
||||
@Schema(description = "本次是否为 dryRun")
|
||||
private boolean dryRun;
|
||||
}
|
||||
+278
@@ -2,6 +2,8 @@ package com.nanri.aiimage.modules.keywordshare.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.common.support.LooseLocalDateTimeParser;
|
||||
import com.nanri.aiimage.config.KeywordShareSyncProperties;
|
||||
import com.nanri.aiimage.modules.keywordshare.mapper.KeywordAddLogMapper;
|
||||
import com.nanri.aiimage.modules.keywordshare.mapper.KeywordDeleteLogMapper;
|
||||
import com.nanri.aiimage.modules.keywordshare.mapper.KeywordShareMapper;
|
||||
@@ -14,6 +16,7 @@ import com.nanri.aiimage.modules.keywordshare.model.dto.KeywordShareAddRequest;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.dto.KeywordShareDeleteRequest;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.dto.KeywordShareFilterQuery;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.dto.KeywordShareKey;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.dto.KeywordShareSyncImportRequest;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.dto.KeywordUsageBatchRequest;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.entity.KeywordAddLogEntity;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.entity.KeywordDeleteLogEntity;
|
||||
@@ -29,6 +32,7 @@ import com.nanri.aiimage.modules.keywordshare.model.vo.KeywordSharePageVo;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.vo.KeywordShareRowVo;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.vo.KeywordShareStatsGroupedVo;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.vo.KeywordShareStatsVo;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.vo.KeywordShareSyncResultVo;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.vo.KeywordShareUnmarkResultVo;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.stereotype.Service;
|
||||
@@ -75,10 +79,33 @@ public class KeywordShareService {
|
||||
private static final String KTYPE_MANUAL = "主关键词";
|
||||
private static final String SOURCE_MANUAL = "手动添加";
|
||||
|
||||
/**
|
||||
* 同步写入的分片行数:与仓内批量写惯例一致,控制单条 SQL 的占位符总量
|
||||
* (12 列 × 500 行 = 6000 个占位符)。
|
||||
*/
|
||||
private static final int SYNC_SHARD_SIZE = 500;
|
||||
/**
|
||||
* 客户端自报使用时间的可信下限。早于此的一律当作脏值丢掉使用状态(1970/0000 这类
|
||||
* SQLite 零值日期),但词本体照收——不该为一个坏时间戳丢掉一个词。
|
||||
*/
|
||||
private static final LocalDateTime MIN_SYNC_USED_TIME = LocalDateTime.of(2000, 1, 1, 0, 0);
|
||||
/**
|
||||
* 各列字符上限,对齐 V177 建表口径(线上真表若更窄,由部署前的 SHOW CREATE TABLE 预检兜住)。
|
||||
* 超限一律整行拒绝而不是截断入库:静默截断会让客户端以为同步成功,而词已经被改坏了。
|
||||
*/
|
||||
private static final int WIDTH_COUNTRY = 32;
|
||||
private static final int WIDTH_KEYWORD = 255;
|
||||
private static final int WIDTH_CHINESE = 255;
|
||||
private static final int WIDTH_KTYPE = 64;
|
||||
private static final int WIDTH_SOURCE = 64;
|
||||
private static final int WIDTH_CATEGORY = 128;
|
||||
private static final int WIDTH_USER_NAME = 128;
|
||||
|
||||
private final KeywordShareMapper keywordShareMapper;
|
||||
private final KeywordUsageMapper keywordUsageMapper;
|
||||
private final KeywordAddLogMapper keywordAddLogMapper;
|
||||
private final KeywordDeleteLogMapper keywordDeleteLogMapper;
|
||||
private final KeywordShareSyncProperties syncProperties;
|
||||
|
||||
// ---------------------------------------------------------------- 列表与统计
|
||||
|
||||
@@ -300,6 +327,135 @@ public class KeywordShareService {
|
||||
return vo;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- 客户端全量同步
|
||||
|
||||
/**
|
||||
* 客户端全量同步导入:把工作台本地那份词库+使用状态搬进主库,并且**保住客户端传来的
|
||||
* 真实使用人与真实使用时间**——这正是 {@link #add} 与 {@link #mark} 给不了的两件事:
|
||||
* add() 从不写四级类目且会自动标记已使用,mark() 把 user_name/used_time 硬写成 token 账号与此刻。
|
||||
*
|
||||
* <p>幂等:词按 {@code uk_comp_cw} UPSERT、使用状态按同键 UPSERT/删除,同一批重复推送结果一致。
|
||||
* 一个请求=一个事务=一批(items ≤ 2000),整批要么全落要么全不回;客户端只需重传失败的那一批,
|
||||
* 不做逐行重放——事务内一次 SQLException 已把整笔标记为 rollback-only,重放不出第二结果。</p>
|
||||
*
|
||||
* @param companyId 一律由服务端从 token 取,与其余写侧同口径
|
||||
* @param tokenUser 本次调用的账号名,用作自报姓名不可信时的落点
|
||||
*/
|
||||
@Transactional
|
||||
public KeywordShareSyncResultVo syncImport(KeywordShareSyncImportRequest request,
|
||||
Long companyId, String tokenUser) {
|
||||
KeywordShareSyncResultVo vo = new KeywordShareSyncResultVo();
|
||||
boolean dryRun = Boolean.TRUE.equals(request.getDryRun());
|
||||
boolean writeAddLogs = Boolean.TRUE.equals(request.getWriteAddLogs());
|
||||
vo.setDryRun(dryRun);
|
||||
|
||||
Map<String, String> roster = canonicalRoster(syncProperties.getUsedUsers());
|
||||
boolean strict = syncProperties.isStrict();
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
|
||||
List<KeywordShareEntity> words = new ArrayList<>();
|
||||
List<KeywordUsageEntity> usageUpserts = new ArrayList<>();
|
||||
List<KeywordShareKey> usageDeletes = new ArrayList<>();
|
||||
Set<String> seen = new LinkedHashSet<>();
|
||||
int usedUserFallback = 0;
|
||||
int usedUserUnknown = 0;
|
||||
int usedTimeFallback = 0;
|
||||
int usedTimeClamped = 0;
|
||||
int usageSkipped = 0;
|
||||
|
||||
for (KeywordShareSyncImportRequest.Item item : request.getItems()) {
|
||||
String country = normalize(item == null ? null : item.getCountry());
|
||||
String keyword = normalize(item == null ? null : item.getKeyword());
|
||||
if (country.isEmpty() || keyword.isEmpty()
|
||||
|| !seen.add(country + "\n" + keyword)
|
||||
|| !withinSyncColumnWidths(country, keyword, item)) {
|
||||
vo.setSkipped(vo.getSkipped() + 1);
|
||||
continue;
|
||||
}
|
||||
words.add(syncWordRow(companyId, tokenUser, now, country, keyword, item));
|
||||
|
||||
// used 三态:null=客户端没表达使用状态,一律不改动 keyword_usage。
|
||||
// 不能把缺省当 false——那等于「只同步词库」的一次调用会把后台已有使用记录整批删掉。
|
||||
Boolean used = item.getUsed();
|
||||
if (used == null) {
|
||||
continue;
|
||||
}
|
||||
if (!used) {
|
||||
usageDeletes.add(KeywordShareKey.of(country, keyword));
|
||||
continue;
|
||||
}
|
||||
UsedUserResult usedUser = resolveUsedUser(item.getUsedUser(), tokenUser, roster, strict);
|
||||
if (usedUser.fellBack()) {
|
||||
usedUserFallback++;
|
||||
}
|
||||
if (usedUser.unknown()) {
|
||||
usedUserUnknown++;
|
||||
}
|
||||
UsedTimeResult usedTime = resolveUsedTime(item.getUsedTime(), now);
|
||||
switch (usedTime.outcome()) {
|
||||
case MISSING, UNPARSEABLE -> usedTimeFallback++;
|
||||
case FUTURE -> usedTimeClamped++;
|
||||
case TOO_OLD -> usageSkipped++;
|
||||
case EXACT -> { }
|
||||
}
|
||||
if (usedTime.value() == null) {
|
||||
continue;
|
||||
}
|
||||
KeywordUsageEntity usage = new KeywordUsageEntity();
|
||||
usage.setCompanyId(companyId);
|
||||
usage.setCountry(country);
|
||||
usage.setKeyword(keyword);
|
||||
usage.setUserName(usedUser.value());
|
||||
usage.setUsedTime(usedTime.value());
|
||||
usageUpserts.add(usage);
|
||||
}
|
||||
vo.setUsedUserFallback(usedUserFallback);
|
||||
vo.setUsedUserUnknown(usedUserUnknown);
|
||||
vo.setUsedTimeFallback(usedTimeFallback);
|
||||
vo.setUsedTimeClamped(usedTimeClamped);
|
||||
vo.setUsageSkipped(usageSkipped);
|
||||
|
||||
// added/updated 走预检分组:UPSERT 的 affected-rows 是 1/2/0(插入/值变/值未变),
|
||||
// 求和后不可逆,拿它计数会把「重复推同一批」报成 0 新增或 2 倍新增。
|
||||
Set<String> existing = existingWordKeys(companyId, words);
|
||||
List<KeywordShareEntity> freshWords = words.stream()
|
||||
.filter(row -> !existing.contains(row.getCountry() + "\n" + row.getKeyword()))
|
||||
.toList();
|
||||
vo.setAdded(freshWords.size());
|
||||
vo.setUpdated(words.size() - freshWords.size());
|
||||
vo.setUsageUpserted(usageUpserts.size());
|
||||
vo.setUsageDeleted(usageDeletes.size());
|
||||
if (dryRun) {
|
||||
return vo;
|
||||
}
|
||||
|
||||
for (List<KeywordShareEntity> shard : shard(words, SYNC_SHARD_SIZE)) {
|
||||
keywordShareMapper.upsertBatchFromSync(shard);
|
||||
}
|
||||
if (writeAddLogs && !freshWords.isEmpty()) {
|
||||
List<KeywordAddLogEntity> logs = freshWords.stream().map(row -> {
|
||||
KeywordAddLogEntity log = new KeywordAddLogEntity();
|
||||
log.setCompanyId(companyId);
|
||||
log.setCountry(row.getCountry());
|
||||
log.setKeyword(row.getKeyword());
|
||||
log.setUserName(tokenUser);
|
||||
log.setAddTime(row.getAddTime());
|
||||
return log;
|
||||
}).toList();
|
||||
for (List<KeywordAddLogEntity> shard : shard(logs, SYNC_SHARD_SIZE)) {
|
||||
keywordAddLogMapper.insertBatch(shard);
|
||||
}
|
||||
vo.setAddLogged(logs.size());
|
||||
}
|
||||
for (List<KeywordUsageEntity> shard : shard(usageUpserts, SYNC_SHARD_SIZE)) {
|
||||
keywordUsageMapper.upsertBatch(shard);
|
||||
}
|
||||
for (List<KeywordShareKey> shard : shard(usageDeletes, SYNC_SHARD_SIZE)) {
|
||||
keywordUsageMapper.deleteBatch(companyId, shard);
|
||||
}
|
||||
return vo;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- 共享记录(只读)
|
||||
|
||||
public KeywordShareLogPageVo logs(Long companyId, String type, String country, Long page, Long size) {
|
||||
@@ -465,6 +621,119 @@ public class KeywordShareService {
|
||||
return size == null ? DEFAULT_PAGE_SIZE : Math.min(Math.max(size, 1), MAX_PAGE_SIZE);
|
||||
}
|
||||
|
||||
private KeywordShareEntity syncWordRow(Long companyId, String tokenUser, LocalDateTime now,
|
||||
String country, String keyword,
|
||||
KeywordShareSyncImportRequest.Item item) {
|
||||
KeywordShareEntity row = new KeywordShareEntity();
|
||||
row.setCompanyId(companyId);
|
||||
row.setCountry(country);
|
||||
row.setKeyword(keyword);
|
||||
row.setChinese(normalize(item.getChinese()));
|
||||
row.setKtype(normalize(item.getKtype()));
|
||||
row.setSource(normalize(item.getSource()));
|
||||
row.setCategory(normalize(item.getCategory()));
|
||||
row.setSubCategory(normalize(item.getSubCategory()));
|
||||
row.setCategoryL3(normalize(item.getCategoryL3()));
|
||||
row.setCategoryL4(normalize(item.getCategoryL4()));
|
||||
// 词本体的 user_name 是「添加人」,与 usedUser(使用人)不是一回事:客户端没有自报添加人的通道,
|
||||
// 新词的添加人就是本次同步账号。UPSERT 的更新分支不碰这两列,既有归属不会被同步改写。
|
||||
row.setUserName(tokenUser);
|
||||
row.setAddTime(now);
|
||||
return row;
|
||||
}
|
||||
|
||||
/**
|
||||
* 逐列比宽。用 {@code String.length()} 数的是 UTF-16 码元,MySQL 数的是字符,
|
||||
* 差异只在增补平面字符(emoji)上出现且这里偏大 ⇒ 只会多拒不会漏拒。
|
||||
*/
|
||||
private boolean withinSyncColumnWidths(String country, String keyword,
|
||||
KeywordShareSyncImportRequest.Item item) {
|
||||
return country.length() <= WIDTH_COUNTRY
|
||||
&& keyword.length() <= WIDTH_KEYWORD
|
||||
&& normalize(item.getChinese()).length() <= WIDTH_CHINESE
|
||||
&& normalize(item.getKtype()).length() <= WIDTH_KTYPE
|
||||
&& normalize(item.getSource()).length() <= WIDTH_SOURCE
|
||||
&& normalize(item.getCategory()).length() <= WIDTH_CATEGORY
|
||||
&& normalize(item.getSubCategory()).length() <= WIDTH_CATEGORY
|
||||
&& normalize(item.getCategoryL3()).length() <= WIDTH_CATEGORY
|
||||
&& normalize(item.getCategoryL4()).length() <= WIDTH_CATEGORY;
|
||||
}
|
||||
|
||||
/** 预检哪些 (country, keyword) 已在主库:分片查,避免 IN 列表一次塞进整批。 */
|
||||
private Set<String> existingWordKeys(Long companyId, List<KeywordShareEntity> words) {
|
||||
List<KeywordShareKey> keys = words.stream()
|
||||
.map(row -> KeywordShareKey.of(row.getCountry(), row.getKeyword()))
|
||||
.toList();
|
||||
Set<String> existing = new LinkedHashSet<>();
|
||||
for (List<KeywordShareKey> shard : shard(keys, SYNC_SHARD_SIZE)) {
|
||||
keywordShareMapper.selectExistingKeywords(companyId, shard)
|
||||
.forEach(key -> existing.add(key.getCountry() + "\n" + key.getKeyword()));
|
||||
}
|
||||
return existing;
|
||||
}
|
||||
|
||||
private static <T> List<List<T>> shard(List<T> rows, int size) {
|
||||
List<List<T>> shards = new ArrayList<>();
|
||||
for (int from = 0; from < rows.size(); from += size) {
|
||||
shards.add(rows.subList(from, Math.min(from + size, rows.size())));
|
||||
}
|
||||
return shards;
|
||||
}
|
||||
|
||||
/**
|
||||
* 自报使用人的收口:命中名单才按姓名落库,且**落的是名单里那份写法**。
|
||||
* 名单两侧都做全角/半角括号归一,所以「阿武(吴棋龙)」也能命中「阿武(吴棋龙)」;
|
||||
* 若原样存客户端写法,同一个人会因括号全半角不同被后管的使用者筛选器拆成两个人。
|
||||
*/
|
||||
private UsedUserResult resolveUsedUser(String declared, String tokenUser,
|
||||
Map<String, String> roster, boolean strict) {
|
||||
String name = normalize(declared);
|
||||
if (name.length() > WIDTH_USER_NAME || name.isEmpty()) {
|
||||
return new UsedUserResult(tokenUser, !name.isEmpty(), true);
|
||||
}
|
||||
String listed = roster.get(canonicalName(name));
|
||||
if (listed != null) {
|
||||
return new UsedUserResult(listed, false, false);
|
||||
}
|
||||
return strict
|
||||
? new UsedUserResult(tokenUser, true, true)
|
||||
: new UsedUserResult(name, true, false);
|
||||
}
|
||||
|
||||
/** 客户端 usedTime 的判定:缺失/不可解析回退此刻,未来值钳到此刻,早于下限则丢使用状态。 */
|
||||
private UsedTimeResult resolveUsedTime(String raw, LocalDateTime now) {
|
||||
LocalDateTime parsed = LooseLocalDateTimeParser.parseOrNull(raw);
|
||||
if (parsed == null) {
|
||||
return new UsedTimeResult(now, normalize(raw).isEmpty()
|
||||
? UsedTimeOutcome.MISSING : UsedTimeOutcome.UNPARSEABLE);
|
||||
}
|
||||
if (parsed.isAfter(now)) {
|
||||
return new UsedTimeResult(now, UsedTimeOutcome.FUTURE);
|
||||
}
|
||||
if (parsed.isBefore(MIN_SYNC_USED_TIME)) {
|
||||
return new UsedTimeResult(null, UsedTimeOutcome.TOO_OLD);
|
||||
}
|
||||
return new UsedTimeResult(parsed, UsedTimeOutcome.EXACT);
|
||||
}
|
||||
|
||||
private Map<String, String> canonicalRoster(List<String> configured) {
|
||||
Map<String, String> roster = new LinkedHashMap<>();
|
||||
if (configured == null) {
|
||||
return roster;
|
||||
}
|
||||
for (String name : configured) {
|
||||
String canonical = canonicalName(name);
|
||||
if (!canonical.isEmpty()) {
|
||||
roster.putIfAbsent(canonical, normalize(name));
|
||||
}
|
||||
}
|
||||
return roster;
|
||||
}
|
||||
|
||||
private String canonicalName(String value) {
|
||||
return normalize(value).replace('(', '(').replace(')', ')');
|
||||
}
|
||||
|
||||
private String normalize(String value) {
|
||||
return value == null ? "" : value.trim();
|
||||
}
|
||||
@@ -477,4 +746,13 @@ public class KeywordShareService {
|
||||
private String orEmpty(String value) {
|
||||
return value == null ? "" : value;
|
||||
}
|
||||
|
||||
/** usedTime 判定结果分类,直接对应响应里的降级计数。 */
|
||||
private enum UsedTimeOutcome { EXACT, MISSING, UNPARSEABLE, FUTURE, TOO_OLD }
|
||||
|
||||
/** 解析后的使用时间;{@code value} 为 null 表示这条使用状态不该落库。 */
|
||||
private record UsedTimeResult(LocalDateTime value, UsedTimeOutcome outcome) { }
|
||||
|
||||
/** 自报使用人的收口结果:最终落库名、是否不在名单内、是否被回退成账号名。 */
|
||||
private record UsedUserResult(String value, boolean unknown, boolean fellBack) { }
|
||||
}
|
||||
|
||||
+5
-13
@@ -3,6 +3,7 @@ package com.nanri.aiimage.modules.shopdatacrawl.service;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.common.support.LooseLocalDateTimeParser;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.mapper.ShopDataCrawlAdminTasksMapper;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.mapper.ShopDataCrawlItemMapper;
|
||||
@@ -23,7 +24,6 @@ import org.springframework.stereotype.Service;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.time.format.DateTimeParseException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
@@ -405,19 +405,11 @@ public class ShopDataCrawlAdminTasksService {
|
||||
if (value.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
String normalized = value.replace('T', ' ').replaceAll("[Zz]$", "").replaceAll("\\.\\d+$", "").trim();
|
||||
for (DateTimeFormatter formatter : List.of(
|
||||
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"),
|
||||
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm"),
|
||||
DateTimeFormatter.ofPattern("yyyy-MM-dd HH"),
|
||||
DateTimeFormatter.ofPattern("yyyy-MM-dd"))) {
|
||||
try {
|
||||
return LocalDateTime.parse(normalized, formatter);
|
||||
} catch (DateTimeParseException ignored) {
|
||||
// 尝试下一种格式
|
||||
}
|
||||
LocalDateTime parsed = LooseLocalDateTimeParser.parseOrNull(value);
|
||||
if (parsed == null) {
|
||||
throw new BusinessException(400, name + " 时间格式无效");
|
||||
}
|
||||
throw new BusinessException(400, name + " 时间格式无效");
|
||||
return parsed;
|
||||
}
|
||||
|
||||
private static String formatAdminTime(LocalDateTime value) {
|
||||
|
||||
@@ -131,3 +131,8 @@ AIIMAGE_ZINIAO_SESSION_TTL_HOURS=2
|
||||
AIIMAGE_ZINIAO_SHOPS_CACHE_MINUTES=30
|
||||
AIIMAGE_ZINIAO_CONNECT_TIMEOUT_SECONDS=5
|
||||
AIIMAGE_ZINIAO_READ_TIMEOUT_SECONDS=15
|
||||
|
||||
# 工作台客户端关键词全量同步(POST /api/keyword-share/sync/import)。
|
||||
# 客户端每行自报真实使用人,服务端只接受名单内姓名并按名单写法落库;名单为空时必须保持 strict=false。
|
||||
AIIMAGE_KEYWORD_SHARE_SYNC_STRICT=false
|
||||
AIIMAGE_KEYWORD_SHARE_SYNC_USED_USERS=
|
||||
|
||||
@@ -260,6 +260,12 @@ aiimage:
|
||||
cron: ${AIIMAGE_MODULE_CLEANUP_CRON:0 0 0 * * *}
|
||||
retention-days: ${AIIMAGE_MODULE_CLEANUP_RETENTION_DAYS:7}
|
||||
module-types: ${AIIMAGE_MODULE_CLEANUP_MODULE_TYPES:DEDUPE,SPLIT,CONVERT,DELETE_BRAND,PRODUCT_RISK_RESOLVE,PRICE_TRACK,SHOP_MATCH,PATROL_DELETE,QUERY_ASIN,WITHDRAW,APPEARANCE_PATENT,SIMILAR_ASIN,COLLECT_DATA}
|
||||
# 工作台客户端全量同步(POST /api/keyword-share/sync/import)。客户端每行自报真实使用人,
|
||||
# 服务端只接受名单内姓名、并统一按名单里那份写法落库(避免全/半角括号把同一个人拆成两个使用者)。
|
||||
# 名单为空时必须保持 strict=false:否则首次上线会把全部记录静默改写成同步账号名,比现状更糟。
|
||||
keyword-share-sync:
|
||||
strict: ${AIIMAGE_KEYWORD_SHARE_SYNC_STRICT:false}
|
||||
used-users: ${AIIMAGE_KEYWORD_SHARE_SYNC_USED_USERS:}
|
||||
# 以下为各业务表的保留期清理(2026-09-16 统一补齐):这些表此前只增不删,
|
||||
# 或只有"读的时候顺手删一行"这类碰运气的清理。全部按天分批删除、带分布式锁单实例执行。
|
||||
device-log:
|
||||
|
||||
@@ -36,8 +36,9 @@
|
||||
</select>
|
||||
|
||||
<select id="selectRows" resultType="com.nanri.aiimage.modules.keywordshare.model.vo.KeywordShareRowVo">
|
||||
<!-- used_user 别名不可省:VO 属性叫 usedUser,驼峰映射只会把 user_name 认成不存在的 userName。 -->
|
||||
SELECT k.id, k.country, k.keyword, k.chinese, k.category, k.sub_category, k.category_l3, k.category_l4,
|
||||
(u.country IS NOT NULL) AS used, u.used_time, u.user_name
|
||||
(u.country IS NOT NULL) AS used, u.used_time, u.user_name AS used_user
|
||||
FROM <include refid="usageJoin"/>
|
||||
WHERE <include refid="filterConds"/>
|
||||
ORDER BY (u.country IS NULL) ASC, u.used_time ASC
|
||||
@@ -46,7 +47,7 @@
|
||||
|
||||
<select id="selectRandomUnused" resultType="com.nanri.aiimage.modules.keywordshare.model.vo.KeywordShareRowVo">
|
||||
SELECT k.id, k.country, k.keyword, k.chinese, k.category, k.sub_category, k.category_l3, k.category_l4,
|
||||
(u.country IS NOT NULL) AS used, u.used_time, u.user_name
|
||||
(u.country IS NOT NULL) AS used, u.used_time, u.user_name AS used_user
|
||||
FROM <include refid="usageJoin"/>
|
||||
WHERE <include refid="filterConds"/>
|
||||
ORDER BY RAND()
|
||||
@@ -135,4 +136,28 @@
|
||||
#{row.subCategory}, #{row.categoryL3}, #{row.categoryL4}, #{row.userName}, #{row.addTime}, #{row.companyId})
|
||||
</foreach>
|
||||
</insert>
|
||||
|
||||
<!-- 客户端全量同步专用 UPSERT。与 insertBatchIgnore 的分工:那条只补新词、重复行静默跳过;
|
||||
这条要把客户端本地那份「四级类目」写进已存在的行——add() 从不设类目,主库存量行的类目是空的。
|
||||
命中 uk_comp_cw 时只覆盖非空值(COALESCE(NULLIF(...))):客户端是四级类目的唯一权威源,
|
||||
但一次不带类目的同步不该把已导入的类目永久打回空串,所以「主动清空」不走这条路。
|
||||
user_name/add_time(添加人与添加时间)在更新分支一律不动:那是历史归属,不由同步改写。 -->
|
||||
<insert id="upsertBatchFromSync">
|
||||
INSERT INTO keyword_share
|
||||
(country, keyword, chinese, ktype, source, category, sub_category, category_l3, category_l4,
|
||||
user_name, add_time, company_id)
|
||||
VALUES
|
||||
<foreach collection="rows" item="row" separator=",">
|
||||
(#{row.country}, #{row.keyword}, #{row.chinese}, #{row.ktype}, #{row.source}, #{row.category},
|
||||
#{row.subCategory}, #{row.categoryL3}, #{row.categoryL4}, #{row.userName}, #{row.addTime}, #{row.companyId})
|
||||
</foreach>
|
||||
ON DUPLICATE KEY UPDATE
|
||||
chinese = COALESCE(NULLIF(VALUES(chinese), ''), chinese),
|
||||
ktype = COALESCE(NULLIF(VALUES(ktype), ''), ktype),
|
||||
source = COALESCE(NULLIF(VALUES(source), ''), source),
|
||||
category = COALESCE(NULLIF(VALUES(category), ''), category),
|
||||
sub_category = COALESCE(NULLIF(VALUES(sub_category), ''), sub_category),
|
||||
category_l3 = COALESCE(NULLIF(VALUES(category_l3), ''), category_l3),
|
||||
category_l4 = COALESCE(NULLIF(VALUES(category_l4), ''), category_l4)
|
||||
</insert>
|
||||
</mapper>
|
||||
|
||||
@@ -65,6 +65,8 @@ class MigratedSqlEquivalenceTest {
|
||||
// (SqlLayeringConventionTest.XML_STATEMENT_TOTAL 同轮 128 → 129)。
|
||||
// 89 → 99:关键词使用记录管理端(keywordusage 模块)新写 10 条只读 select,逐条点入
|
||||
// (SqlLayeringConventionTest.XML_STATEMENT_TOTAL 同轮 130 → 140)。
|
||||
// 99 → 99:客户端全量同步新写 1 条 insert(KeywordShareMapper#upsertBatchFromSync),
|
||||
// 点名进 NOT_MIGRATED 但不是 select ⇒ select 总数不动(SqlLayeringConventionTest 同轮 140 → 141)。
|
||||
private static final int XML_SELECT_TOTAL = 99;
|
||||
|
||||
/**
|
||||
@@ -90,6 +92,9 @@ class MigratedSqlEquivalenceTest {
|
||||
"KeywordShareMapper#selectSubCategoryCounts",
|
||||
"KeywordShareMapper#selectCategoryL3Counts",
|
||||
"KeywordShareMapper#selectExistingKeywords",
|
||||
// 工作台客户端全量同步新写的词本体 UPSERT:注解时代没有原文可对撞,按新写语句点名留痕。
|
||||
// 它是 insert 不是 select,故 XML_SELECT_TOTAL 不随之变。
|
||||
"KeywordShareMapper#upsertBatchFromSync",
|
||||
// brandgating 模块(品牌Gating库并入管理后台,2026-10-23)新写 8 条 select。
|
||||
"BrandGatingMapper#countByFilter",
|
||||
"BrandGatingMapper#selectRows",
|
||||
|
||||
@@ -83,7 +83,9 @@ class SqlLayeringConventionTest {
|
||||
// 129 → 130:CR-01 新增 {@code LoginUserMapper#incrementSessionEpoch}(会话世代原子自增),
|
||||
// 新写语句直接进 mapper/auth/LoginUserMapper.xml,同轮登记进 NOT_MIGRATED。
|
||||
// 130 → 140:关键词使用记录管理端(keywordusage 模块,1 份 XML 10 条只读 select)新写语句。
|
||||
private static final int XML_STATEMENT_TOTAL = 140;
|
||||
// 140 → 141:工作台客户端全量同步新增 {@code KeywordShareMapper#upsertBatchFromSync}
|
||||
// (词本体 UPSERT,把客户端的四级类目写进既有行),同轮登记进 NOT_MIGRATED。
|
||||
private static final int XML_STATEMENT_TOTAL = 141;
|
||||
|
||||
/**
|
||||
* wrapper 里夹带函数级 SQL 的现状("这不是简单单表"的证据),文件名 → 条数:
|
||||
|
||||
+212
-1
@@ -1,5 +1,6 @@
|
||||
package com.nanri.aiimage.modules.keywordshare.service;
|
||||
|
||||
import com.nanri.aiimage.config.KeywordShareSyncProperties;
|
||||
import com.nanri.aiimage.modules.keywordshare.mapper.KeywordAddLogMapper;
|
||||
import com.nanri.aiimage.modules.keywordshare.mapper.KeywordDeleteLogMapper;
|
||||
import com.nanri.aiimage.modules.keywordshare.mapper.KeywordShareMapper;
|
||||
@@ -12,9 +13,11 @@ import com.nanri.aiimage.modules.keywordshare.model.entity.KeywordShareEntity;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.entity.KeywordUsageEntity;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.dto.KeywordShareAddRequest;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.dto.KeywordShareDeleteRequest;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.dto.KeywordShareSyncImportRequest;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.dto.KeywordUsageBatchRequest;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.vo.KeywordSharePageVo;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.vo.KeywordShareRowVo;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.vo.KeywordShareSyncResultVo;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
@@ -58,8 +61,19 @@ class KeywordShareServiceTest {
|
||||
private KeywordDeleteLogMapper keywordDeleteLogMapper;
|
||||
|
||||
private KeywordShareService newService() {
|
||||
return newService(new KeywordShareSyncProperties());
|
||||
}
|
||||
|
||||
private KeywordShareService newService(KeywordShareSyncProperties syncProperties) {
|
||||
return new KeywordShareService(keywordShareMapper, keywordUsageMapper,
|
||||
keywordAddLogMapper, keywordDeleteLogMapper);
|
||||
keywordAddLogMapper, keywordDeleteLogMapper, syncProperties);
|
||||
}
|
||||
|
||||
private static KeywordShareSyncProperties rosterOf(boolean strict, String... names) {
|
||||
KeywordShareSyncProperties properties = new KeywordShareSyncProperties();
|
||||
properties.setStrict(strict);
|
||||
properties.setUsedUsers(List.of(names));
|
||||
return properties;
|
||||
}
|
||||
|
||||
private static KeywordShareRowVo row(long id, String keyword, boolean used, String usedUser) {
|
||||
@@ -476,4 +490,201 @@ class KeywordShareServiceTest {
|
||||
.contains("company_id", "user_name")
|
||||
.doesNotContain("${");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- 客户端全量同步
|
||||
|
||||
private static KeywordShareSyncImportRequest.Item syncItem(String country, String keyword, Boolean used,
|
||||
String usedUser, String usedTime) {
|
||||
KeywordShareSyncImportRequest.Item item = new KeywordShareSyncImportRequest.Item();
|
||||
item.setCountry(country);
|
||||
item.setKeyword(keyword);
|
||||
item.setUsed(used);
|
||||
item.setUsedUser(usedUser);
|
||||
item.setUsedTime(usedTime);
|
||||
return item;
|
||||
}
|
||||
|
||||
private static KeywordShareSyncImportRequest syncRequest(List<KeywordShareSyncImportRequest.Item> items) {
|
||||
KeywordShareSyncImportRequest request = new KeywordShareSyncImportRequest();
|
||||
request.setItems(items);
|
||||
return request;
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("sync/import:客户端传来的 usedTime/usedUser 与四级类目原样落库,不被服务端此刻与账号覆盖")
|
||||
void syncImportKeepsClientUsedTimeUserAndCategories() {
|
||||
KeywordShareSyncImportRequest.Item item = syncItem("英国", "wraps", true, "阿武(吴棋龙)", "2026-10-25T22:59:26");
|
||||
item.setChinese("包裹");
|
||||
item.setCategory("衣物鞋类");
|
||||
item.setSubCategory("包包");
|
||||
item.setCategoryL3("其他包包");
|
||||
item.setCategoryL4("其他包包");
|
||||
item.setKtype("主关键词");
|
||||
item.setSource("本地导入");
|
||||
|
||||
KeywordShareService service = newService(rosterOf(true, "阿武(吴棋龙)"));
|
||||
LocalDateTime before = LocalDateTime.now();
|
||||
KeywordShareSyncResultVo vo = service.syncImport(syncRequest(List.of(item)), COMPANY_ID, "kw_share_api");
|
||||
|
||||
// 这条用例存在的理由:mark() 走的是 used_time=now()、user_name=token 账号,
|
||||
// 同步历史时会被改写成「谁在同步那一刻用的」,真实使用人与使用时刻双双丢失。
|
||||
ArgumentCaptor<List<KeywordUsageEntity>> usageCaptor = ArgumentCaptor.forClass(List.class);
|
||||
verify(keywordUsageMapper).upsertBatch(usageCaptor.capture());
|
||||
KeywordUsageEntity usage = usageCaptor.getValue().get(0);
|
||||
assertThat(usage.getUserName()).isEqualTo("阿武(吴棋龙)");
|
||||
assertThat(usage.getUsedTime()).isEqualTo(LocalDateTime.of(2026, 10, 25, 22, 59, 26))
|
||||
.isBefore(before);
|
||||
assertThat(usage.getCompanyId()).isEqualTo(COMPANY_ID);
|
||||
|
||||
ArgumentCaptor<List<KeywordShareEntity>> wordCaptor = ArgumentCaptor.forClass(List.class);
|
||||
verify(keywordShareMapper).upsertBatchFromSync(wordCaptor.capture());
|
||||
KeywordShareEntity word = wordCaptor.getValue().get(0);
|
||||
assertThat(word.getCategory()).isEqualTo("衣物鞋类");
|
||||
assertThat(word.getSubCategory()).isEqualTo("包包");
|
||||
assertThat(word.getCategoryL3()).isEqualTo("其他包包");
|
||||
// 词本体的 user_name 是「添加人」,新词就是本次同步账号——它和使用人是两回事
|
||||
assertThat(word.getUserName()).isEqualTo("kw_share_api");
|
||||
assertThat(vo.getAdded()).isEqualTo(1);
|
||||
assertThat(vo.getUpdated()).isZero();
|
||||
assertThat(vo.getUsedUserFallback()).isZero();
|
||||
assertThat(vo.getUsedUserUnknown()).isZero();
|
||||
assertThat(vo.getUsedTimeFallback()).isZero();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("sync/import:名单外姓名 strict=true 回退账号名并计数,strict=false 放行但仍计 unknown")
|
||||
void syncImportResolvesOperatorAgainstRoster() {
|
||||
// 半角括号写法要能命中全角名单,否则同一个人会被后管拆成两个使用者
|
||||
KeywordShareSyncImportRequest.Item hit = syncItem("英国", "basket", true, "阿武(吴棋龙)", "2026-10-01 08:00:00");
|
||||
KeywordShareSyncImportRequest.Item miss = syncItem("英国", "hanger", true, "陌生人", "2026-10-01 08:00:00");
|
||||
|
||||
ArgumentCaptor<List<KeywordUsageEntity>> captor = ArgumentCaptor.forClass(List.class);
|
||||
|
||||
KeywordShareSyncResultVo strict = newService(rosterOf(true, "阿武(吴棋龙)"))
|
||||
.syncImport(syncRequest(List.of(hit, miss)), COMPANY_ID, "kw_share_api");
|
||||
verify(keywordUsageMapper).upsertBatch(captor.capture());
|
||||
// 命中名单后落的是名单里那份写法:客户端传半角也不能让它把同一个人拆成两个使用者
|
||||
assertThat(captor.getValue()).extracting(KeywordUsageEntity::getUserName)
|
||||
.containsExactly("阿武(吴棋龙)", "kw_share_api");
|
||||
assertThat(strict.getUsedUserFallback()).isEqualTo(1);
|
||||
assertThat(strict.getUsedUserUnknown()).isEqualTo(1);
|
||||
|
||||
KeywordShareSyncResultVo lenient = newService(rosterOf(false, "阿武(吴棋龙)"))
|
||||
.syncImport(syncRequest(List.of(hit, miss)), COMPANY_ID, "kw_share_api");
|
||||
assertThat(lenient.getUsedUserFallback()).isZero();
|
||||
assertThat(lenient.getUsedUserUnknown()).isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("sync/import:脏时间按行降级——1970 丢使用状态但词照收、未来值钳到此刻、缺失回退此刻")
|
||||
void syncImportDegradesDirtyUsedTimePerRow() {
|
||||
KeywordShareSyncImportRequest.Item ancient = syncItem("英国", "ancient", true, "阿武(吴棋龙)", "1970-01-01T00:00:00");
|
||||
KeywordShareSyncImportRequest.Item future = syncItem("英国", "future", true, "阿武(吴棋龙)", "2099-01-01T00:00:00");
|
||||
KeywordShareSyncImportRequest.Item missing = syncItem("英国", "missing", true, "阿武(吴棋龙)", " ");
|
||||
KeywordShareSyncImportRequest.Item unparsable = syncItem("英国", "garbage", true, "阿武(吴棋龙)", "昨天下午");
|
||||
|
||||
LocalDateTime before = LocalDateTime.now();
|
||||
KeywordShareSyncResultVo vo = newService(rosterOf(true, "阿武(吴棋龙)"))
|
||||
.syncImport(syncRequest(List.of(ancient, future, missing, unparsable)), COMPANY_ID, "kw_share_api");
|
||||
|
||||
assertThat(vo.getUsageSkipped()).isEqualTo(1);
|
||||
assertThat(vo.getUsedTimeClamped()).isEqualTo(1);
|
||||
assertThat(vo.getUsedTimeFallback()).isEqualTo(2);
|
||||
// 一颗脏时间戳不该打掉整批:四个词全部入库,只有 ancient 的使用状态被丢
|
||||
assertThat(vo.getAdded()).isEqualTo(4);
|
||||
ArgumentCaptor<List<KeywordShareEntity>> wordCaptor = ArgumentCaptor.forClass(List.class);
|
||||
verify(keywordShareMapper).upsertBatchFromSync(wordCaptor.capture());
|
||||
assertThat(wordCaptor.getValue()).extracting(KeywordShareEntity::getKeyword)
|
||||
.containsExactly("ancient", "future", "missing", "garbage");
|
||||
|
||||
ArgumentCaptor<List<KeywordUsageEntity>> usageCaptor = ArgumentCaptor.forClass(List.class);
|
||||
verify(keywordUsageMapper).upsertBatch(usageCaptor.capture());
|
||||
assertThat(usageCaptor.getValue()).hasSize(3)
|
||||
.allSatisfy(row -> assertThat(row.getUsedTime()).isAfterOrEqualTo(before));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("sync/import:used=false 删使用记录;dryRun 只统计不写;空键与批内重复计入 skipped")
|
||||
void syncImportDeletesUsageAndHonoursDryRun() {
|
||||
List<KeywordShareSyncImportRequest.Item> items = List.of(
|
||||
syncItem("英国", "keep", false, null, null),
|
||||
syncItem("英国", "drop", true, "阿武(吴棋龙)", "2026-10-01T08:00:00"),
|
||||
syncItem("英国", "drop", true, "阿武(吴棋龙)", "2026-10-01T08:00:00"),
|
||||
syncItem("", "no-country", true, "阿武(吴棋龙)", "2026-10-01T08:00:00"));
|
||||
KeywordShareSyncImportRequest request = syncRequest(items);
|
||||
|
||||
KeywordShareService service = newService(rosterOf(true, "阿武(吴棋龙)"));
|
||||
KeywordShareSyncImportRequest dryRunRequest = syncRequest(items);
|
||||
dryRunRequest.setDryRun(true);
|
||||
KeywordShareSyncResultVo dryRun = service.syncImport(dryRunRequest, COMPANY_ID, "kw_share_api");
|
||||
|
||||
assertThat(dryRun.isDryRun()).isTrue();
|
||||
assertThat(dryRun.getSkipped()).isEqualTo(2);
|
||||
assertThat(dryRun.getAdded()).isEqualTo(2);
|
||||
verify(keywordShareMapper, never()).upsertBatchFromSync(anyList());
|
||||
verify(keywordUsageMapper, never()).upsertBatch(anyList());
|
||||
verify(keywordUsageMapper, never()).deleteBatch(any(), anyList());
|
||||
|
||||
KeywordShareSyncResultVo real = service.syncImport(request, COMPANY_ID, "kw_share_api");
|
||||
assertThat(real.isDryRun()).isFalse();
|
||||
ArgumentCaptor<List<KeywordShareKey>> deleteCaptor = ArgumentCaptor.forClass(List.class);
|
||||
verify(keywordUsageMapper).deleteBatch(eq(COMPANY_ID), deleteCaptor.capture());
|
||||
assertThat(deleteCaptor.getValue()).extracting(KeywordShareKey::getKeyword).containsExactly("keep");
|
||||
assertThat(real.getUsageDeleted()).isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("sync/import:used 不传=不碰使用状态(只同步词库不能把后台已有使用记录删掉)")
|
||||
void syncImportLeavesUsageUntouchedWhenUsedFlagOmitted() {
|
||||
KeywordShareSyncImportRequest.Item wordOnly = syncItem("英国", "wraps", null, null, null);
|
||||
wordOnly.setCategory("衣物鞋类");
|
||||
|
||||
KeywordShareSyncResultVo vo = newService(rosterOf(true, "阿武(吴棋龙)"))
|
||||
.syncImport(syncRequest(List.of(wordOnly)), COMPANY_ID, "kw_share_api");
|
||||
|
||||
assertThat(vo.getAdded()).isEqualTo(1);
|
||||
assertThat(vo.getUsageUpserted()).isZero();
|
||||
assertThat(vo.getUsageDeleted()).isZero();
|
||||
verify(keywordUsageMapper, never()).upsertBatch(anyList());
|
||||
verify(keywordUsageMapper, never()).deleteBatch(any(), anyList());
|
||||
ArgumentCaptor<List<KeywordShareEntity>> captor = ArgumentCaptor.forClass(List.class);
|
||||
verify(keywordShareMapper).upsertBatchFromSync(captor.capture());
|
||||
assertThat(captor.getValue().get(0).getCategory()).isEqualTo("衣物鞋类");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("sync/import:写侧按 500 行分片,不把整批塞进一条 SQL 的占位符里")
|
||||
void syncImportShardsWrites() {
|
||||
List<KeywordShareSyncImportRequest.Item> items = new java.util.ArrayList<>();
|
||||
for (int i = 0; i < 1200; i++) {
|
||||
items.add(syncItem("英国", "kw-" + i, true, "阿武(吴棋龙)", "2026-10-01T08:00:00"));
|
||||
}
|
||||
|
||||
KeywordShareSyncResultVo vo = newService(rosterOf(true, "阿武(吴棋龙)"))
|
||||
.syncImport(syncRequest(items), COMPANY_ID, "kw_share_api");
|
||||
|
||||
assertThat(vo.getAdded()).isEqualTo(1200);
|
||||
ArgumentCaptor<List<KeywordShareEntity>> wordCaptor = ArgumentCaptor.forClass(List.class);
|
||||
verify(keywordShareMapper, org.mockito.Mockito.times(3)).upsertBatchFromSync(wordCaptor.capture());
|
||||
assertThat(wordCaptor.getAllValues()).extracting(List::size).containsExactly(500, 500, 200);
|
||||
ArgumentCaptor<List<KeywordUsageEntity>> usageCaptor = ArgumentCaptor.forClass(List.class);
|
||||
verify(keywordUsageMapper, org.mockito.Mockito.times(3)).upsertBatch(usageCaptor.capture());
|
||||
assertThat(usageCaptor.getAllValues()).extracting(List::size).containsExactly(500, 500, 200);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("sync/import 的 UPSERT SQL:类目非空才覆盖,且更新分支不得改写添加人与添加时间")
|
||||
void syncUpsertKeepsCategoriesAndProvenance() throws Exception {
|
||||
String xml = Files.readString(Path.of(
|
||||
"src/main/resources/mapper/keywordshare/KeywordShareMapper.xml"), StandardCharsets.UTF_8);
|
||||
String upsert = xml.substring(xml.indexOf("<insert id=\"upsertBatchFromSync\">"));
|
||||
assertThat(upsert).contains("ON DUPLICATE KEY UPDATE")
|
||||
.contains("category = COALESCE(NULLIF(VALUES(category), ''), category)")
|
||||
.contains("sub_category = COALESCE(NULLIF(VALUES(sub_category), ''), sub_category)")
|
||||
.doesNotContain("${");
|
||||
// 更新分支只碰类目/释义/类型/来源:user_name、add_time 是历史归属,同步无权改写
|
||||
String updateBranch = upsert.substring(upsert.indexOf("ON DUPLICATE KEY UPDATE"));
|
||||
assertThat(updateBranch).doesNotContain("user_name =")
|
||||
.doesNotContain("add_time =");
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user