refactor+perf+fix: G5 模块边界 SPI 化、C5 流式解析、C6/C7 查询优化、D13 队列持久化、A3/A4/A7 鉴权
模块边界(G5 / G7) - 新增 task/spi/TaskModuleHeartbeatSpi + 13 个模块实现:TaskHeartbeatService 不再 import 任何 业务模块(原先注入 12 个 CacheService 并用 switch 分发);启动校验重复注册 - 新增 task/spi/BrandTaskHeartbeatSpi(品牌任务心跳/中断)、BrandTaskStaleRepairSpi(陈旧修复)、 CollectDataItemCleanupSpi(历史清理):跨模块 Mapper 操作收回业务模块 - G7:10 个被跨模块借用的 productrisk VO 迁至 common/model/vo - 架构棘轮收紧:TASK_TO_BUSINESS_BASELINE 119 → 6(实测) - 新增 TaskModuleHeartbeatSpiCoverageTest(moduleType 覆盖与拼写) 性能与容量(C5/C6/C7/D13) - C5 流式解析:SkipPriceAsinService(含两行表头语义)、AppearancePatentExcelParser、 BrandTaskService、DeleteBrandRunService、LocalFileStorageService.getExcelInfo 改 ExcelStreamReader; 行数上限改为迭代中生效 - C6 去重总数据列表:关键字改前缀匹配(命中 uk_data_value);V124 删除永不生效的 idx_country - C7 撞款扫描:按店逐批取数(索引前缀),不再全表 GROUP BY + JOIN + 全量拉内存 - D13 待删对象本地日志 PendingDeleteJournal(启动回放 + 收敛重写),RustfsDeleteRetryService 与 TransientPayloadDeleteOrchestrator 接入;异步删除失败对象写回日志 - V123 删除 biz_file_result 两个被复合索引覆盖的单列索引 安全(A3/A4/A7 + 守卫名单) - A4 数字人版本写操作要求管理员;A7 视频密钥按登录身份(超管例外) - A3 上传接口加危险扩展名黑名单(可配置) - AdminApiGuardFilter 用户态名单补 /api/ziniao(controller 已 requireAdmin,此处为开关打开后的第二层) 容量(明细表保留期) - 新增 ShopDataCrawlItemRetentionService:biz_shop_data_crawl_item 按保留期(默认 30 天)分批清理 (该表此前无任何清理策略,是增长最快的表),job 锁 + 单轮批次上限
This commit is contained in:
@@ -56,6 +56,13 @@ public final class ExcelStreamReader {
|
||||
default void onHeader(String sheetName, Integer sheetNo, Map<Integer, String> headerMap) throws Exception {
|
||||
}
|
||||
|
||||
/**
|
||||
* 表头之后回调本次 sheet 的近似总行数(EasyExcel 基于 sheet 尺寸,可能为 null)。
|
||||
* 流式解析无法在读完前得到精确行数,需要展示进度/做前置上限校验的调用方可用它近似。
|
||||
*/
|
||||
default void onSheetTotal(String sheetName, Integer sheetNo, Integer approximateTotalRows) throws Exception {
|
||||
}
|
||||
|
||||
void onRow(String sheetName, Integer sheetNo, int rowIndex, Map<Integer, String> headerMap, Map<Integer, String> rowMap) throws Exception;
|
||||
}
|
||||
|
||||
@@ -77,6 +84,7 @@ public final class ExcelStreamReader {
|
||||
currentHeaderMap = normalizedHeadMap;
|
||||
try {
|
||||
handler.onHeader(sheetName(context), sheetNo(context), currentHeaderMap);
|
||||
handler.onSheetTotal(sheetName(context), sheetNo(context), approximateTotalRows(context));
|
||||
} catch (RuntimeException ex) {
|
||||
throw ex;
|
||||
} catch (Exception ex) {
|
||||
@@ -109,6 +117,15 @@ public final class ExcelStreamReader {
|
||||
public void doAfterAllAnalysed(AnalysisContext context) {
|
||||
}
|
||||
|
||||
private Integer approximateTotalRows(AnalysisContext context) {
|
||||
try {
|
||||
return context.readSheetHolder() == null ? null
|
||||
: context.readSheetHolder().getApproximateTotalRowNumber();
|
||||
} catch (Exception ex) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private String sheetName(AnalysisContext context) {
|
||||
return context.readSheetHolder() == null ? "" : context.readSheetHolder().getSheetName();
|
||||
}
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
package com.nanri.aiimage.common.util;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
import java.nio.file.StandardOpenOption;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* 待删对象队列的本地落盘日志:进程重启后仍能恢复「待删除对象」清单。
|
||||
*
|
||||
* <p>使用场景:对象存储删除失败后的补偿队列原本只在节点内存里,重启即丢,
|
||||
* 对应对象会一直残留在桶里直到生命周期规则过期。这里用一行一个 objectKey 的
|
||||
* 追加日志做最小持久化——入队追加、队列收敛后整体重写、启动时回放。
|
||||
*
|
||||
* <p>并发:所有方法内部同步,调用方无需额外加锁;文件损坏/读写异常只记日志不抛出,
|
||||
* 保证补偿链路本身不会因为落盘失败而中断业务。
|
||||
*
|
||||
* <p>上限:{@code maxEntries} 用于防止日志在异常堆积时无界增长(超出后丢弃最早的记录,
|
||||
* 与内存队列的容量准入语义一致——丢的是「待删对象」,最坏结果是对象残留)。
|
||||
*/
|
||||
@Slf4j
|
||||
public final class PendingDeleteJournal {
|
||||
|
||||
private final Path path;
|
||||
private final int maxEntries;
|
||||
private final Object lock = new Object();
|
||||
|
||||
public PendingDeleteJournal(Path path, int maxEntries) {
|
||||
this.path = path;
|
||||
this.maxEntries = Math.max(1, maxEntries);
|
||||
}
|
||||
|
||||
public Path getPath() {
|
||||
return path;
|
||||
}
|
||||
|
||||
/** 追加一条待删对象(重复追加由回放时的 Set 语义去重)。 */
|
||||
public void record(String objectKey) {
|
||||
if (objectKey == null || objectKey.isBlank()) {
|
||||
return;
|
||||
}
|
||||
synchronized (lock) {
|
||||
try {
|
||||
Path parent = path.getParent();
|
||||
if (parent != null) {
|
||||
Files.createDirectories(parent);
|
||||
}
|
||||
Files.writeString(path, sanitize(objectKey) + System.lineSeparator(),
|
||||
StandardCharsets.UTF_8, StandardOpenOption.CREATE, StandardOpenOption.APPEND);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[delete-journal] 追加待删对象失败 path={} msg={}", path, ex.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** 启动回放:返回日志中的待删对象(按首次出现顺序去重)。 */
|
||||
public List<String> readAll() {
|
||||
synchronized (lock) {
|
||||
if (!Files.isRegularFile(path)) {
|
||||
return List.of();
|
||||
}
|
||||
List<String> lines;
|
||||
try {
|
||||
lines = Files.readAllLines(path, StandardCharsets.UTF_8);
|
||||
} catch (Exception ex) {
|
||||
log.warn("[delete-journal] 读取待删对象日志失败 path={} msg={}", path, ex.getMessage());
|
||||
return List.of();
|
||||
}
|
||||
Set<String> unique = new LinkedHashSet<>();
|
||||
for (String line : lines) {
|
||||
if (line == null) {
|
||||
continue;
|
||||
}
|
||||
String key = line.trim();
|
||||
if (!key.isEmpty()) {
|
||||
unique.add(key);
|
||||
}
|
||||
}
|
||||
return new ArrayList<>(unique);
|
||||
}
|
||||
}
|
||||
|
||||
/** 队列收敛(成功删除/引用仍在)后整体重写为剩余的待删对象;剩余为空则删除日志。 */
|
||||
public void rewrite(Collection<String> remaining) {
|
||||
synchronized (lock) {
|
||||
Set<String> keep = new LinkedHashSet<>();
|
||||
if (remaining != null) {
|
||||
for (String key : remaining) {
|
||||
if (key != null && !key.isBlank()) {
|
||||
keep.add(sanitize(key));
|
||||
if (keep.size() >= maxEntries) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
try {
|
||||
if (keep.isEmpty()) {
|
||||
Files.deleteIfExists(path);
|
||||
return;
|
||||
}
|
||||
Path parent = path.getParent();
|
||||
if (parent != null) {
|
||||
Files.createDirectories(parent);
|
||||
}
|
||||
Path tmp = path.resolveSibling(path.getFileName() + ".tmp");
|
||||
StringBuilder content = new StringBuilder();
|
||||
for (String key : keep) {
|
||||
content.append(key).append(System.lineSeparator());
|
||||
}
|
||||
Files.writeString(tmp, content.toString(), StandardCharsets.UTF_8,
|
||||
StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);
|
||||
try {
|
||||
Files.move(tmp, path, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
|
||||
} catch (IOException atomicUnsupported) {
|
||||
// 少数文件系统不支持 ATOMIC_MOVE,退化为普通替换
|
||||
Files.move(tmp, path, StandardCopyOption.REPLACE_EXISTING);
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
log.warn("[delete-journal] 重写待删对象日志失败 path={} msg={}", path, ex.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** 单行一条记录:去掉换行避免破坏行结构。 */
|
||||
private static String sanitize(String objectKey) {
|
||||
return objectKey.replace('\n', ' ').replace('\r', ' ').trim();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user