feat(keyword-share): 新增客户端批量同步端点,保住真实使用人/时间与四级类目
工作台「关键词使用记录」与后台管理对不上,根因不是链路不通(后台现存词 426 与桌面 426 逐字相等),而是「谁、何时」两个字段在推拉两个方向上各丢一次: - 推:markInternal 把 user_name/used_time 硬写成 token 账号与服务器此刻, 历史使用记录推上去一律变成「同步账号 + 同步那一刻」; - 拉:selectRows/selectRandomUnused 投影 u.user_name,而 VO 属性叫 usedUser, 驼峰映射只会认成不存在的 userName ⇒ GET /keywords 的 usedUser 恒为 null; - 词本体:add() 从不写四级类目,且 keyword_share 侧只有 INSERT IGNORE、 没有任何 UPSERT,重复词静默 no-op。 处置: - 修 used_user 别名(两处投影); - 新增 KeywordShareMapper#upsertBatchFromSync,类目/chinese 按 COALESCE(NULLIF(VALUES(x),''),x) 非空才覆盖,更新分支不碰 user_name/add_time (那是历史归属,不由同步改写); - 新增 POST /api/keyword-share/sync/import:每行携带 usedUser/usedTime, 绕开 markInternal 复用现成 KeywordUsageMapper#upsertBatch(本就是 =VALUES(...)), 不改表结构、不加 Flyway 迁移; - used 取三态:null=不改动使用状态。按 false 处理会让「只同步词库」的一次 调用把后台已有使用记录整批删掉; - 共享 API 账号下的归属失真用服务端名单收口:命中 aiimage.keyword-share-sync .used-users 才按姓名落库且落名单里那份写法(全/半角括号归一,否则同一人被 后管拆成两个使用者);strict 默认 false,名单未填时放行并计 usedUserUnknown, 避免首次上线把全部记录静默改写成账号名。 顺带:CatchSilenceRatchetTest 要求新增 catch 必须同批净减,而宽松时间解析需要 try/catch ⇒ 把 ShopDataCrawlAdminTasksService 已有的同形状实现抽成 common/support/LooseLocalDateTimeParser 两处共用,净新增 catch 为 0,未放宽基线。 尺子:XML_STATEMENT_TOTAL 140→141,NOT_MIGRATED 点名新 insert(非 select, XML_SELECT_TOTAL 不动)。 验证:mvn -o test-compile rc=0;后端整面 mvn -o test 4892 passed/0 failed/18 skipped。 本机无 MySQL,端到端未跑,真链路待客户端联调(ops BACKLOG B44)。
This commit is contained in:
+51
@@ -0,0 +1,51 @@
|
||||
package com.nanri.aiimage.common.support;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.time.format.DateTimeParseException;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 宽松本地日期时间解析:兼容 {@code T}/空格分隔、可选秒与小时、尾随 {@code Z} 与小数秒。
|
||||
*
|
||||
* <p>存在的理由是「调用方给的从来不是同一个时间形状」:Flask 侧的管理接口会传
|
||||
* {@code 2026-10-25 22:59:26},Python 客户端 {@code isoformat()} 会传 {@code 2026-10-25T22:59:26},
|
||||
* 带毫秒和 UTC 后缀的也见过。收在一处,是为了让"试几种格式"这件事只有一份实现、只有一处 try/catch。</p>
|
||||
*
|
||||
* <p>解析不出来返回 {@code null},不抛异常——要不要把 null 变成 400、还是降级成服务器此刻,
|
||||
* 是调用方的业务判断,不该由解析器替它决定。</p>
|
||||
*/
|
||||
public final class LooseLocalDateTimeParser {
|
||||
|
||||
/** 按优先级逐个试;顺序即语义(先信息全的后退化到只到日)。 */
|
||||
private static final List<DateTimeFormatter> FORMATS = List.of(
|
||||
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"),
|
||||
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm"),
|
||||
DateTimeFormatter.ofPattern("yyyy-MM-dd HH"),
|
||||
DateTimeFormatter.ofPattern("yyyy-MM-dd"));
|
||||
|
||||
private LooseLocalDateTimeParser() {
|
||||
}
|
||||
|
||||
/**
|
||||
* @return 解析结果;{@code raw} 为空或任何格式都对不上时返回 {@code null}
|
||||
*/
|
||||
public static LocalDateTime parseOrNull(String raw) {
|
||||
String value = raw == null ? "" : raw.trim();
|
||||
if (value.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
String normalized = value.replace('T', ' ')
|
||||
.replaceAll("[Zz]$", "")
|
||||
.replaceAll("\\.\\d+$", "")
|
||||
.trim();
|
||||
for (DateTimeFormatter formatter : FORMATS) {
|
||||
try {
|
||||
return LocalDateTime.parse(normalized, formatter);
|
||||
} catch (DateTimeParseException ignored) {
|
||||
// 尝试下一种格式
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package com.nanri.aiimage.config;
|
||||
|
||||
import lombok.Data;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 五国关键词客户端同步({@code /api/keyword-share/sync/import})的服务端参数。
|
||||
*
|
||||
* <p>存在的主要理由:工作台按对接文档口径使用**共享专用 API 账号**,而 {@code keyword_usage.user_name}
|
||||
* 是「谁用了哪个词」的唯一出处。若一律取 token 账号名,多人使用会被记成同一个人,后台按使用者筛选
|
||||
* 就失去意义;若无条件信任客户端自报姓名,又会把「阿武(扶金凡)/阿武(吴棋龙)」这类同音异写
|
||||
* 直接灌进留痕表。名单校验是这两者之间的收口:命中名单才按自报姓名落库。</p>
|
||||
*/
|
||||
@Data
|
||||
@ConfigurationProperties(prefix = "aiimage.keyword-share-sync")
|
||||
public class KeywordShareSyncProperties {
|
||||
|
||||
/**
|
||||
* 可信操作人姓名名单(真人显示名,与 {@code keyword_usage.user_name} 同形)。
|
||||
* 比对时两侧都做 trim + 全角括号归一,所以名单里写「阿武(吴棋龙)」也能匹配半角写法。
|
||||
*/
|
||||
private List<String> usedUsers = new ArrayList<>();
|
||||
|
||||
/**
|
||||
* 名单外姓名的处置。
|
||||
* <ul>
|
||||
* <li>{@code false}(默认):放行自报姓名,只在响应里计数回报——首次上线时名单还没填,
|
||||
* 直接回退会把全部记录静默改写成账号名,比现状更糟;</li>
|
||||
* <li>{@code true}:名单外一律回退为本次调用的账号名,并计入 {@code usedUserFallback}。</li>
|
||||
* </ul>
|
||||
*/
|
||||
private boolean strict = false;
|
||||
}
|
||||
@@ -4,6 +4,6 @@ import org.springframework.boot.context.properties.EnableConfigurationProperties
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
@Configuration
|
||||
@EnableConfigurationProperties({OssProperties.class, TransientStorageProperties.class, StorageProperties.class, BrandProgressProperties.class, DeleteBrandProgressProperties.class, BrandCheckProperties.class, ZiniaoProperties.class, ModuleCleanupProperties.class, TaskPressureProperties.class, TaskImageCacheCleanupProperties.class, AppearancePatentProperties.class, SimilarAsinProperties.class, ImageVideoProperties.class, InstanceRoutingProperties.class, CapacityPlanProperties.class, UserSecretProperties.class, NotificationProperties.class, DeviceLogOssProperties.class, CollectDataBrandRecheckProperties.class, ClientCredentialProperties.class, CostGuardProperties.class})
|
||||
@EnableConfigurationProperties({OssProperties.class, TransientStorageProperties.class, StorageProperties.class, BrandProgressProperties.class, DeleteBrandProgressProperties.class, BrandCheckProperties.class, ZiniaoProperties.class, ModuleCleanupProperties.class, TaskPressureProperties.class, TaskImageCacheCleanupProperties.class, AppearancePatentProperties.class, SimilarAsinProperties.class, ImageVideoProperties.class, InstanceRoutingProperties.class, CapacityPlanProperties.class, UserSecretProperties.class, NotificationProperties.class, DeviceLogOssProperties.class, CollectDataBrandRecheckProperties.class, ClientCredentialProperties.class, CostGuardProperties.class, KeywordShareSyncProperties.class})
|
||||
public class PropertiesConfig {
|
||||
}
|
||||
|
||||
+20
@@ -5,6 +5,7 @@ import com.nanri.aiimage.common.model.entity.AdminUserEntity;
|
||||
import com.nanri.aiimage.common.security.AdminAuthSupport;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.dto.KeywordShareAddRequest;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.dto.KeywordShareDeleteRequest;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.dto.KeywordShareSyncImportRequest;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.dto.KeywordUsageBatchRequest;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.vo.KeywordShareAddResultVo;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.vo.KeywordShareDeleteResultVo;
|
||||
@@ -14,6 +15,7 @@ import com.nanri.aiimage.modules.keywordshare.model.vo.KeywordShareMarkResultVo;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.vo.KeywordSharePageVo;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.vo.KeywordShareRowVo;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.vo.KeywordShareStatsGroupedVo;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.vo.KeywordShareSyncResultVo;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.vo.KeywordShareUnmarkResultVo;
|
||||
import com.nanri.aiimage.modules.keywordshare.service.KeywordShareService;
|
||||
import com.nanri.aiimage.modules.permission.service.PermissionMenuService;
|
||||
@@ -154,6 +156,24 @@ public class KeywordShareController {
|
||||
country, category, subCategory, categoryL3, categoryL4, search, count));
|
||||
}
|
||||
|
||||
/**
|
||||
* 客户端全量同步导入。与 {@code POST /keywords} + {@code /usage/mark} 的分工:那两条是网页侧
|
||||
* 交互式单条操作(使用人/时间取服务端此刻),这条专供工作台将本地历史批量搬入,
|
||||
* 必须保住客户端传来的四级类目与真实使用人/使用时间。
|
||||
*/
|
||||
@PostMapping("/sync/import")
|
||||
@Operation(summary = "客户端批量同步导入",
|
||||
description = "按 (company_id, country, keyword) UPSERT 词本体并同步使用状态,"
|
||||
+ "保留客户端传入的 usedTime/usedUser;used=false 删除对应使用记录。"
|
||||
+ "幂等,单请求不超过 2000 行,失败只需重传该批。")
|
||||
public ApiResponse<KeywordShareSyncResultVo> syncImport(
|
||||
HttpServletRequest request,
|
||||
@Valid @RequestBody KeywordShareSyncImportRequest body) {
|
||||
Operator operator = requireKeywordShareAccess(request);
|
||||
return ApiResponse.success("同步完成",
|
||||
keywordShareService.syncImport(body, operator.companyId(), operator.userName()));
|
||||
}
|
||||
|
||||
/**
|
||||
* 逐端点把门:登录态 + 后台菜单权限;公司 id 与用户名都从登录人实体取,前端传参不参与归属判定。
|
||||
*/
|
||||
|
||||
+6
@@ -59,4 +59,10 @@ public interface KeywordShareMapper extends BaseMapper<KeywordShareEntity> {
|
||||
|
||||
/** 批量 INSERT IGNORE:命中唯一键 uk_comp_cw 的重复行静默跳过,兜底与预检双保险。 */
|
||||
int insertBatchIgnore(@Param("rows") List<KeywordShareEntity> rows);
|
||||
|
||||
/**
|
||||
* 客户端全量同步专用 UPSERT:新词整行插入,已存在的词只把非空的四级类目/chinese/ktype/source
|
||||
* 覆盖进去,添加人与添加时间不动。与 {@link #insertBatchIgnore} 的区别正是这条会更新已有行。
|
||||
*/
|
||||
int upsertBatchFromSync(@Param("rows") List<KeywordShareEntity> rows);
|
||||
}
|
||||
|
||||
+80
@@ -0,0 +1,80 @@
|
||||
package com.nanri.aiimage.modules.keywordshare.model.dto;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import jakarta.validation.Valid;
|
||||
import jakarta.validation.constraints.NotEmpty;
|
||||
import jakarta.validation.constraints.Size;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 客户端全量同步导入请求。与 {@link KeywordShareAddRequest} 的分工:那条只补新词、由服务端
|
||||
* 定添加人与时间;这条要把客户端本地那份四级类目与真实使用历史原样搬进主库。
|
||||
*/
|
||||
@Data
|
||||
@Schema(description = "五国关键词客户端批量同步导入请求")
|
||||
public class KeywordShareSyncImportRequest {
|
||||
|
||||
@Schema(description = "只统计不写库,用于导入前对账", example = "false")
|
||||
private Boolean dryRun;
|
||||
|
||||
@Schema(description = "是否为新词写 keyword_add_log 留痕;几十万级全量导入默认不写,避免刷屏",
|
||||
example = "false")
|
||||
private Boolean writeAddLogs;
|
||||
|
||||
@Valid
|
||||
@NotEmpty(message = "items 不能为空")
|
||||
@Size(max = KeywordShareSyncImportRequest.MAX_ITEMS, message = "items 单次不超过 2000 条")
|
||||
@Schema(description = "待同步的词行列表", requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
private List<Item> items;
|
||||
|
||||
/** 单请求上限:2000 行 × 12 列约 0.6–1MB,留出网关与请求体日志的余量。 */
|
||||
public static final int MAX_ITEMS = 2000;
|
||||
|
||||
@Data
|
||||
@Schema(description = "单个待同步词行")
|
||||
public static class Item {
|
||||
|
||||
@Schema(description = "国家", requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
private String country;
|
||||
|
||||
@Schema(description = "关键词", requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
private String keyword;
|
||||
|
||||
@Schema(description = "中文释义;传空不会清空主库已有值")
|
||||
private String chinese;
|
||||
|
||||
@Schema(description = "关键词类型;传空不会清空主库已有值")
|
||||
private String ktype;
|
||||
|
||||
@Schema(description = "来源;传空不会清空主库已有值")
|
||||
private String source;
|
||||
|
||||
@Schema(description = "一级类目;传空不会清空主库已有值")
|
||||
private String category;
|
||||
|
||||
@Schema(description = "二级类目;传空不会清空主库已有值")
|
||||
private String subCategory;
|
||||
|
||||
@Schema(description = "三级类目;传空不会清空主库已有值")
|
||||
private String categoryL3;
|
||||
|
||||
@Schema(description = "四级类目;传空不会清空主库已有值")
|
||||
private String categoryL4;
|
||||
|
||||
@Schema(description = "使用状态三态:true 写/刷 keyword_usage,false 删该行,"
|
||||
+ "**不传(null)=完全不改动该词的使用状态**。只同步词库时必须整批省略此字段,"
|
||||
+ "否则会把后台已有使用记录删掉",
|
||||
requiredMode = Schema.RequiredMode.NOT_REQUIRED)
|
||||
private Boolean used;
|
||||
|
||||
@Schema(description = "真实使用人;命中服务端名单才落库,否则回退为本次调用的账号名",
|
||||
example = "阿武(吴棋龙)")
|
||||
private String usedUser;
|
||||
|
||||
@Schema(description = "真实使用时间,ISO-8601 本地日期时间;服务端按此保留历史时刻而非同步时刻",
|
||||
example = "2026-10-25T22:59:26")
|
||||
private String usedTime;
|
||||
}
|
||||
}
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
package com.nanri.aiimage.modules.keywordshare.model.vo;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.Data;
|
||||
|
||||
/**
|
||||
* 客户端批量同步导入结果。计数是观测值而非事务保证:预检与写入之间有并发窗口,
|
||||
* 同一批被两个客户端同时推时 added/updated 会漂,但落库内容幂等。
|
||||
*/
|
||||
@Data
|
||||
@Schema(description = "五国关键词客户端批量同步导入结果")
|
||||
public class KeywordShareSyncResultVo {
|
||||
|
||||
@Schema(description = "新插入的词行数")
|
||||
private int added;
|
||||
|
||||
@Schema(description = "命中唯一键、被刷了类目/释义的既有词行数")
|
||||
private int updated;
|
||||
|
||||
@Schema(description = "整行丢弃数(国家或关键词为空、批内重复、字段超长)")
|
||||
private int skipped;
|
||||
|
||||
@Schema(description = "本批请求写入或刷新 keyword_usage 的行数(dryRun 下为预计数)")
|
||||
private int usageUpserted;
|
||||
|
||||
@Schema(description = "本批请求按 used=false 取消使用的行数(dryRun 下为预计数;"
|
||||
+ "实际删除条数不会多于它,主库可能本就没有该 usage 行)")
|
||||
private int usageDeleted;
|
||||
|
||||
@Schema(description = "词已入库但使用状态被丢弃的行数(usedTime 解析失败或早于下限)")
|
||||
private int usageSkipped;
|
||||
|
||||
@Schema(description = "usedUser 未命中服务端名单、回退为调用账号名的行数(仅 strict=true 时会产生)")
|
||||
private int usedUserFallback;
|
||||
|
||||
@Schema(description = "usedUser 自报姓名不在名单内的行数;非 strict 下照原样落库但仍计数,"
|
||||
+ "供运维据此判断名单是否该补齐、能否切 strict")
|
||||
private int usedUserUnknown;
|
||||
|
||||
@Schema(description = "usedTime 缺失、回退为服务端当前时间的行数")
|
||||
private int usedTimeFallback;
|
||||
|
||||
@Schema(description = "usedTime 落在未来、被钳到当前时间的行数")
|
||||
private int usedTimeClamped;
|
||||
|
||||
@Schema(description = "写入 keyword_add_log 留痕的行数(writeAddLogs=true 时才可能非 0)")
|
||||
private int addLogged;
|
||||
|
||||
@Schema(description = "本次是否为 dryRun")
|
||||
private boolean dryRun;
|
||||
}
|
||||
+278
@@ -2,6 +2,8 @@ package com.nanri.aiimage.modules.keywordshare.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.common.support.LooseLocalDateTimeParser;
|
||||
import com.nanri.aiimage.config.KeywordShareSyncProperties;
|
||||
import com.nanri.aiimage.modules.keywordshare.mapper.KeywordAddLogMapper;
|
||||
import com.nanri.aiimage.modules.keywordshare.mapper.KeywordDeleteLogMapper;
|
||||
import com.nanri.aiimage.modules.keywordshare.mapper.KeywordShareMapper;
|
||||
@@ -14,6 +16,7 @@ import com.nanri.aiimage.modules.keywordshare.model.dto.KeywordShareAddRequest;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.dto.KeywordShareDeleteRequest;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.dto.KeywordShareFilterQuery;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.dto.KeywordShareKey;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.dto.KeywordShareSyncImportRequest;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.dto.KeywordUsageBatchRequest;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.entity.KeywordAddLogEntity;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.entity.KeywordDeleteLogEntity;
|
||||
@@ -29,6 +32,7 @@ import com.nanri.aiimage.modules.keywordshare.model.vo.KeywordSharePageVo;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.vo.KeywordShareRowVo;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.vo.KeywordShareStatsGroupedVo;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.vo.KeywordShareStatsVo;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.vo.KeywordShareSyncResultVo;
|
||||
import com.nanri.aiimage.modules.keywordshare.model.vo.KeywordShareUnmarkResultVo;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.stereotype.Service;
|
||||
@@ -75,10 +79,33 @@ public class KeywordShareService {
|
||||
private static final String KTYPE_MANUAL = "主关键词";
|
||||
private static final String SOURCE_MANUAL = "手动添加";
|
||||
|
||||
/**
|
||||
* 同步写入的分片行数:与仓内批量写惯例一致,控制单条 SQL 的占位符总量
|
||||
* (12 列 × 500 行 = 6000 个占位符)。
|
||||
*/
|
||||
private static final int SYNC_SHARD_SIZE = 500;
|
||||
/**
|
||||
* 客户端自报使用时间的可信下限。早于此的一律当作脏值丢掉使用状态(1970/0000 这类
|
||||
* SQLite 零值日期),但词本体照收——不该为一个坏时间戳丢掉一个词。
|
||||
*/
|
||||
private static final LocalDateTime MIN_SYNC_USED_TIME = LocalDateTime.of(2000, 1, 1, 0, 0);
|
||||
/**
|
||||
* 各列字符上限,对齐 V177 建表口径(线上真表若更窄,由部署前的 SHOW CREATE TABLE 预检兜住)。
|
||||
* 超限一律整行拒绝而不是截断入库:静默截断会让客户端以为同步成功,而词已经被改坏了。
|
||||
*/
|
||||
private static final int WIDTH_COUNTRY = 32;
|
||||
private static final int WIDTH_KEYWORD = 255;
|
||||
private static final int WIDTH_CHINESE = 255;
|
||||
private static final int WIDTH_KTYPE = 64;
|
||||
private static final int WIDTH_SOURCE = 64;
|
||||
private static final int WIDTH_CATEGORY = 128;
|
||||
private static final int WIDTH_USER_NAME = 128;
|
||||
|
||||
private final KeywordShareMapper keywordShareMapper;
|
||||
private final KeywordUsageMapper keywordUsageMapper;
|
||||
private final KeywordAddLogMapper keywordAddLogMapper;
|
||||
private final KeywordDeleteLogMapper keywordDeleteLogMapper;
|
||||
private final KeywordShareSyncProperties syncProperties;
|
||||
|
||||
// ---------------------------------------------------------------- 列表与统计
|
||||
|
||||
@@ -300,6 +327,135 @@ public class KeywordShareService {
|
||||
return vo;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- 客户端全量同步
|
||||
|
||||
/**
|
||||
* 客户端全量同步导入:把工作台本地那份词库+使用状态搬进主库,并且**保住客户端传来的
|
||||
* 真实使用人与真实使用时间**——这正是 {@link #add} 与 {@link #mark} 给不了的两件事:
|
||||
* add() 从不写四级类目且会自动标记已使用,mark() 把 user_name/used_time 硬写成 token 账号与此刻。
|
||||
*
|
||||
* <p>幂等:词按 {@code uk_comp_cw} UPSERT、使用状态按同键 UPSERT/删除,同一批重复推送结果一致。
|
||||
* 一个请求=一个事务=一批(items ≤ 2000),整批要么全落要么全不回;客户端只需重传失败的那一批,
|
||||
* 不做逐行重放——事务内一次 SQLException 已把整笔标记为 rollback-only,重放不出第二结果。</p>
|
||||
*
|
||||
* @param companyId 一律由服务端从 token 取,与其余写侧同口径
|
||||
* @param tokenUser 本次调用的账号名,用作自报姓名不可信时的落点
|
||||
*/
|
||||
@Transactional
|
||||
public KeywordShareSyncResultVo syncImport(KeywordShareSyncImportRequest request,
|
||||
Long companyId, String tokenUser) {
|
||||
KeywordShareSyncResultVo vo = new KeywordShareSyncResultVo();
|
||||
boolean dryRun = Boolean.TRUE.equals(request.getDryRun());
|
||||
boolean writeAddLogs = Boolean.TRUE.equals(request.getWriteAddLogs());
|
||||
vo.setDryRun(dryRun);
|
||||
|
||||
Map<String, String> roster = canonicalRoster(syncProperties.getUsedUsers());
|
||||
boolean strict = syncProperties.isStrict();
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
|
||||
List<KeywordShareEntity> words = new ArrayList<>();
|
||||
List<KeywordUsageEntity> usageUpserts = new ArrayList<>();
|
||||
List<KeywordShareKey> usageDeletes = new ArrayList<>();
|
||||
Set<String> seen = new LinkedHashSet<>();
|
||||
int usedUserFallback = 0;
|
||||
int usedUserUnknown = 0;
|
||||
int usedTimeFallback = 0;
|
||||
int usedTimeClamped = 0;
|
||||
int usageSkipped = 0;
|
||||
|
||||
for (KeywordShareSyncImportRequest.Item item : request.getItems()) {
|
||||
String country = normalize(item == null ? null : item.getCountry());
|
||||
String keyword = normalize(item == null ? null : item.getKeyword());
|
||||
if (country.isEmpty() || keyword.isEmpty()
|
||||
|| !seen.add(country + "\n" + keyword)
|
||||
|| !withinSyncColumnWidths(country, keyword, item)) {
|
||||
vo.setSkipped(vo.getSkipped() + 1);
|
||||
continue;
|
||||
}
|
||||
words.add(syncWordRow(companyId, tokenUser, now, country, keyword, item));
|
||||
|
||||
// used 三态:null=客户端没表达使用状态,一律不改动 keyword_usage。
|
||||
// 不能把缺省当 false——那等于「只同步词库」的一次调用会把后台已有使用记录整批删掉。
|
||||
Boolean used = item.getUsed();
|
||||
if (used == null) {
|
||||
continue;
|
||||
}
|
||||
if (!used) {
|
||||
usageDeletes.add(KeywordShareKey.of(country, keyword));
|
||||
continue;
|
||||
}
|
||||
UsedUserResult usedUser = resolveUsedUser(item.getUsedUser(), tokenUser, roster, strict);
|
||||
if (usedUser.fellBack()) {
|
||||
usedUserFallback++;
|
||||
}
|
||||
if (usedUser.unknown()) {
|
||||
usedUserUnknown++;
|
||||
}
|
||||
UsedTimeResult usedTime = resolveUsedTime(item.getUsedTime(), now);
|
||||
switch (usedTime.outcome()) {
|
||||
case MISSING, UNPARSEABLE -> usedTimeFallback++;
|
||||
case FUTURE -> usedTimeClamped++;
|
||||
case TOO_OLD -> usageSkipped++;
|
||||
case EXACT -> { }
|
||||
}
|
||||
if (usedTime.value() == null) {
|
||||
continue;
|
||||
}
|
||||
KeywordUsageEntity usage = new KeywordUsageEntity();
|
||||
usage.setCompanyId(companyId);
|
||||
usage.setCountry(country);
|
||||
usage.setKeyword(keyword);
|
||||
usage.setUserName(usedUser.value());
|
||||
usage.setUsedTime(usedTime.value());
|
||||
usageUpserts.add(usage);
|
||||
}
|
||||
vo.setUsedUserFallback(usedUserFallback);
|
||||
vo.setUsedUserUnknown(usedUserUnknown);
|
||||
vo.setUsedTimeFallback(usedTimeFallback);
|
||||
vo.setUsedTimeClamped(usedTimeClamped);
|
||||
vo.setUsageSkipped(usageSkipped);
|
||||
|
||||
// added/updated 走预检分组:UPSERT 的 affected-rows 是 1/2/0(插入/值变/值未变),
|
||||
// 求和后不可逆,拿它计数会把「重复推同一批」报成 0 新增或 2 倍新增。
|
||||
Set<String> existing = existingWordKeys(companyId, words);
|
||||
List<KeywordShareEntity> freshWords = words.stream()
|
||||
.filter(row -> !existing.contains(row.getCountry() + "\n" + row.getKeyword()))
|
||||
.toList();
|
||||
vo.setAdded(freshWords.size());
|
||||
vo.setUpdated(words.size() - freshWords.size());
|
||||
vo.setUsageUpserted(usageUpserts.size());
|
||||
vo.setUsageDeleted(usageDeletes.size());
|
||||
if (dryRun) {
|
||||
return vo;
|
||||
}
|
||||
|
||||
for (List<KeywordShareEntity> shard : shard(words, SYNC_SHARD_SIZE)) {
|
||||
keywordShareMapper.upsertBatchFromSync(shard);
|
||||
}
|
||||
if (writeAddLogs && !freshWords.isEmpty()) {
|
||||
List<KeywordAddLogEntity> logs = freshWords.stream().map(row -> {
|
||||
KeywordAddLogEntity log = new KeywordAddLogEntity();
|
||||
log.setCompanyId(companyId);
|
||||
log.setCountry(row.getCountry());
|
||||
log.setKeyword(row.getKeyword());
|
||||
log.setUserName(tokenUser);
|
||||
log.setAddTime(row.getAddTime());
|
||||
return log;
|
||||
}).toList();
|
||||
for (List<KeywordAddLogEntity> shard : shard(logs, SYNC_SHARD_SIZE)) {
|
||||
keywordAddLogMapper.insertBatch(shard);
|
||||
}
|
||||
vo.setAddLogged(logs.size());
|
||||
}
|
||||
for (List<KeywordUsageEntity> shard : shard(usageUpserts, SYNC_SHARD_SIZE)) {
|
||||
keywordUsageMapper.upsertBatch(shard);
|
||||
}
|
||||
for (List<KeywordShareKey> shard : shard(usageDeletes, SYNC_SHARD_SIZE)) {
|
||||
keywordUsageMapper.deleteBatch(companyId, shard);
|
||||
}
|
||||
return vo;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- 共享记录(只读)
|
||||
|
||||
public KeywordShareLogPageVo logs(Long companyId, String type, String country, Long page, Long size) {
|
||||
@@ -465,6 +621,119 @@ public class KeywordShareService {
|
||||
return size == null ? DEFAULT_PAGE_SIZE : Math.min(Math.max(size, 1), MAX_PAGE_SIZE);
|
||||
}
|
||||
|
||||
private KeywordShareEntity syncWordRow(Long companyId, String tokenUser, LocalDateTime now,
|
||||
String country, String keyword,
|
||||
KeywordShareSyncImportRequest.Item item) {
|
||||
KeywordShareEntity row = new KeywordShareEntity();
|
||||
row.setCompanyId(companyId);
|
||||
row.setCountry(country);
|
||||
row.setKeyword(keyword);
|
||||
row.setChinese(normalize(item.getChinese()));
|
||||
row.setKtype(normalize(item.getKtype()));
|
||||
row.setSource(normalize(item.getSource()));
|
||||
row.setCategory(normalize(item.getCategory()));
|
||||
row.setSubCategory(normalize(item.getSubCategory()));
|
||||
row.setCategoryL3(normalize(item.getCategoryL3()));
|
||||
row.setCategoryL4(normalize(item.getCategoryL4()));
|
||||
// 词本体的 user_name 是「添加人」,与 usedUser(使用人)不是一回事:客户端没有自报添加人的通道,
|
||||
// 新词的添加人就是本次同步账号。UPSERT 的更新分支不碰这两列,既有归属不会被同步改写。
|
||||
row.setUserName(tokenUser);
|
||||
row.setAddTime(now);
|
||||
return row;
|
||||
}
|
||||
|
||||
/**
|
||||
* 逐列比宽。用 {@code String.length()} 数的是 UTF-16 码元,MySQL 数的是字符,
|
||||
* 差异只在增补平面字符(emoji)上出现且这里偏大 ⇒ 只会多拒不会漏拒。
|
||||
*/
|
||||
private boolean withinSyncColumnWidths(String country, String keyword,
|
||||
KeywordShareSyncImportRequest.Item item) {
|
||||
return country.length() <= WIDTH_COUNTRY
|
||||
&& keyword.length() <= WIDTH_KEYWORD
|
||||
&& normalize(item.getChinese()).length() <= WIDTH_CHINESE
|
||||
&& normalize(item.getKtype()).length() <= WIDTH_KTYPE
|
||||
&& normalize(item.getSource()).length() <= WIDTH_SOURCE
|
||||
&& normalize(item.getCategory()).length() <= WIDTH_CATEGORY
|
||||
&& normalize(item.getSubCategory()).length() <= WIDTH_CATEGORY
|
||||
&& normalize(item.getCategoryL3()).length() <= WIDTH_CATEGORY
|
||||
&& normalize(item.getCategoryL4()).length() <= WIDTH_CATEGORY;
|
||||
}
|
||||
|
||||
/** 预检哪些 (country, keyword) 已在主库:分片查,避免 IN 列表一次塞进整批。 */
|
||||
private Set<String> existingWordKeys(Long companyId, List<KeywordShareEntity> words) {
|
||||
List<KeywordShareKey> keys = words.stream()
|
||||
.map(row -> KeywordShareKey.of(row.getCountry(), row.getKeyword()))
|
||||
.toList();
|
||||
Set<String> existing = new LinkedHashSet<>();
|
||||
for (List<KeywordShareKey> shard : shard(keys, SYNC_SHARD_SIZE)) {
|
||||
keywordShareMapper.selectExistingKeywords(companyId, shard)
|
||||
.forEach(key -> existing.add(key.getCountry() + "\n" + key.getKeyword()));
|
||||
}
|
||||
return existing;
|
||||
}
|
||||
|
||||
private static <T> List<List<T>> shard(List<T> rows, int size) {
|
||||
List<List<T>> shards = new ArrayList<>();
|
||||
for (int from = 0; from < rows.size(); from += size) {
|
||||
shards.add(rows.subList(from, Math.min(from + size, rows.size())));
|
||||
}
|
||||
return shards;
|
||||
}
|
||||
|
||||
/**
|
||||
* 自报使用人的收口:命中名单才按姓名落库,且**落的是名单里那份写法**。
|
||||
* 名单两侧都做全角/半角括号归一,所以「阿武(吴棋龙)」也能命中「阿武(吴棋龙)」;
|
||||
* 若原样存客户端写法,同一个人会因括号全半角不同被后管的使用者筛选器拆成两个人。
|
||||
*/
|
||||
private UsedUserResult resolveUsedUser(String declared, String tokenUser,
|
||||
Map<String, String> roster, boolean strict) {
|
||||
String name = normalize(declared);
|
||||
if (name.length() > WIDTH_USER_NAME || name.isEmpty()) {
|
||||
return new UsedUserResult(tokenUser, !name.isEmpty(), true);
|
||||
}
|
||||
String listed = roster.get(canonicalName(name));
|
||||
if (listed != null) {
|
||||
return new UsedUserResult(listed, false, false);
|
||||
}
|
||||
return strict
|
||||
? new UsedUserResult(tokenUser, true, true)
|
||||
: new UsedUserResult(name, true, false);
|
||||
}
|
||||
|
||||
/** 客户端 usedTime 的判定:缺失/不可解析回退此刻,未来值钳到此刻,早于下限则丢使用状态。 */
|
||||
private UsedTimeResult resolveUsedTime(String raw, LocalDateTime now) {
|
||||
LocalDateTime parsed = LooseLocalDateTimeParser.parseOrNull(raw);
|
||||
if (parsed == null) {
|
||||
return new UsedTimeResult(now, normalize(raw).isEmpty()
|
||||
? UsedTimeOutcome.MISSING : UsedTimeOutcome.UNPARSEABLE);
|
||||
}
|
||||
if (parsed.isAfter(now)) {
|
||||
return new UsedTimeResult(now, UsedTimeOutcome.FUTURE);
|
||||
}
|
||||
if (parsed.isBefore(MIN_SYNC_USED_TIME)) {
|
||||
return new UsedTimeResult(null, UsedTimeOutcome.TOO_OLD);
|
||||
}
|
||||
return new UsedTimeResult(parsed, UsedTimeOutcome.EXACT);
|
||||
}
|
||||
|
||||
private Map<String, String> canonicalRoster(List<String> configured) {
|
||||
Map<String, String> roster = new LinkedHashMap<>();
|
||||
if (configured == null) {
|
||||
return roster;
|
||||
}
|
||||
for (String name : configured) {
|
||||
String canonical = canonicalName(name);
|
||||
if (!canonical.isEmpty()) {
|
||||
roster.putIfAbsent(canonical, normalize(name));
|
||||
}
|
||||
}
|
||||
return roster;
|
||||
}
|
||||
|
||||
private String canonicalName(String value) {
|
||||
return normalize(value).replace('(', '(').replace(')', ')');
|
||||
}
|
||||
|
||||
private String normalize(String value) {
|
||||
return value == null ? "" : value.trim();
|
||||
}
|
||||
@@ -477,4 +746,13 @@ public class KeywordShareService {
|
||||
private String orEmpty(String value) {
|
||||
return value == null ? "" : value;
|
||||
}
|
||||
|
||||
/** usedTime 判定结果分类,直接对应响应里的降级计数。 */
|
||||
private enum UsedTimeOutcome { EXACT, MISSING, UNPARSEABLE, FUTURE, TOO_OLD }
|
||||
|
||||
/** 解析后的使用时间;{@code value} 为 null 表示这条使用状态不该落库。 */
|
||||
private record UsedTimeResult(LocalDateTime value, UsedTimeOutcome outcome) { }
|
||||
|
||||
/** 自报使用人的收口结果:最终落库名、是否不在名单内、是否被回退成账号名。 */
|
||||
private record UsedUserResult(String value, boolean unknown, boolean fellBack) { }
|
||||
}
|
||||
|
||||
+5
-13
@@ -3,6 +3,7 @@ package com.nanri.aiimage.modules.shopdatacrawl.service;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.common.exception.BusinessException;
|
||||
import com.nanri.aiimage.common.support.LooseLocalDateTimeParser;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.mapper.ShopDataCrawlAdminTasksMapper;
|
||||
import com.nanri.aiimage.modules.shopdatacrawl.mapper.ShopDataCrawlItemMapper;
|
||||
@@ -23,7 +24,6 @@ import org.springframework.stereotype.Service;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.time.format.DateTimeParseException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
@@ -405,19 +405,11 @@ public class ShopDataCrawlAdminTasksService {
|
||||
if (value.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
String normalized = value.replace('T', ' ').replaceAll("[Zz]$", "").replaceAll("\\.\\d+$", "").trim();
|
||||
for (DateTimeFormatter formatter : List.of(
|
||||
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"),
|
||||
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm"),
|
||||
DateTimeFormatter.ofPattern("yyyy-MM-dd HH"),
|
||||
DateTimeFormatter.ofPattern("yyyy-MM-dd"))) {
|
||||
try {
|
||||
return LocalDateTime.parse(normalized, formatter);
|
||||
} catch (DateTimeParseException ignored) {
|
||||
// 尝试下一种格式
|
||||
}
|
||||
LocalDateTime parsed = LooseLocalDateTimeParser.parseOrNull(value);
|
||||
if (parsed == null) {
|
||||
throw new BusinessException(400, name + " 时间格式无效");
|
||||
}
|
||||
throw new BusinessException(400, name + " 时间格式无效");
|
||||
return parsed;
|
||||
}
|
||||
|
||||
private static String formatAdminTime(LocalDateTime value) {
|
||||
|
||||
Reference in New Issue
Block a user