fix(撞款扫描): 保留期 90→7 天,并单独兜住最新 SUCCESS 行
该表每行含整份聚合 payload(线上实测约 2.3MB/行,24 行占 55MB), 而读取侧只认最新一行(selectLatestFullRow/LightRow 都是 ORDER BY id DESC LIMIT 1), 历史行纯占磁盘。保留期降到 7 天,行数下限仍由 PROTECTED_ROWS=10 兜底。 同时补一条保护线:连续失败多日时,最新 SUCCESS 行会滑出「最新 N 行」保护窗口, 再按时间线被删掉,界面就空白了——现在取「第 N 新行」与「最新 SUCCESS 行」 两者更靠前的那个 id 作为保护线,确保它永不被删。
This commit is contained in:
+17
-4
@@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.shopduplicatecheck.service;
|
||||
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.modules.shopduplicatecheck.mapper.ShopDataDuplicateScanMapper;
|
||||
import com.nanri.aiimage.modules.shopduplicatecheck.model.dto.ScanLightRowDto;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
@@ -35,8 +36,8 @@ public class ShopDataDuplicateScanRetentionService {
|
||||
private final ShopDataDuplicateScanMapper scanMapper;
|
||||
private final DistributedJobLockService distributedJobLockService;
|
||||
|
||||
@Value("${aiimage.shop-duplicate-check.scan-retention-days:90}")
|
||||
private int retentionDays = 90;
|
||||
@Value("${aiimage.shop-duplicate-check.scan-retention-days:7}")
|
||||
private int retentionDays = 7;
|
||||
|
||||
@Value("${aiimage.shop-duplicate-check.scan-retention-batch-size:200}")
|
||||
private int retentionBatchSize = 200;
|
||||
@@ -54,9 +55,13 @@ public class ShopDataDuplicateScanRetentionService {
|
||||
return;
|
||||
}
|
||||
try (lockHandle) {
|
||||
// 两条保护线取更靠前(更小)的那个 id——保护线之内的行一律不删:
|
||||
// ① 最新的 PROTECTED_ROWS 行:万一扫描停摆很久,界面上仍能看到最后一份结果;
|
||||
// ② 最新 SUCCESS 行:读侧只认它。连续失败多日时它可能已滑出保护窗口,
|
||||
// 按时间线会被删掉、界面直接空白,所以单独兜一条。
|
||||
Long nthNewestId = scanMapper.selectNthNewestId(PROTECTED_ROWS - 1);
|
||||
// 行数不足保护量时没什么可删,直接返回
|
||||
long protectFromId = nthNewestId == null ? Long.MAX_VALUE : nthNewestId;
|
||||
ScanLightRowDto latestSuccess = scanMapper.selectLatestLightRow();
|
||||
long protectFromId = minId(nthNewestId, latestSuccess == null ? null : latestSuccess.getId());
|
||||
int totalDeleted = 0;
|
||||
int batches = 0;
|
||||
while (batches < MAX_BATCHES_PER_RUN) {
|
||||
@@ -73,4 +78,12 @@ public class ShopDataDuplicateScanRetentionService {
|
||||
log.warn("[shop-duplicate-check] 扫描结果保留清理失败 cutoff={} msg={}", cutoff, ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
|
||||
/** 取两条保护线里更靠前(更小)的 id;都为 null 表示无需保护(等价于不设限)。 */
|
||||
private static long minId(Long first, Long second) {
|
||||
if (first == null) {
|
||||
return second == null ? Long.MAX_VALUE : second;
|
||||
}
|
||||
return second == null ? first : Math.min(first, second);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -233,7 +233,9 @@ aiimage:
|
||||
loop-run-retention-batch-size: ${AIIMAGE_PRICE_TRACK_LOOP_RUN_RETENTION_BATCH_SIZE:500}
|
||||
loop-run-retention-cron: ${AIIMAGE_PRICE_TRACK_LOOP_RUN_RETENTION_CRON:0 20 4 * * *}
|
||||
shop-duplicate-check:
|
||||
scan-retention-days: ${AIIMAGE_SHOP_DUPLICATE_SCAN_RETENTION_DAYS:90}
|
||||
# 每行含整份聚合 payload(实测约 2.3MB/行),而读取侧只认最新一行,
|
||||
# 保留期给 7 天足够排查;行数下限由服务里的 PROTECTED_ROWS 兜底
|
||||
scan-retention-days: ${AIIMAGE_SHOP_DUPLICATE_SCAN_RETENTION_DAYS:7}
|
||||
scan-retention-batch-size: ${AIIMAGE_SHOP_DUPLICATE_SCAN_RETENTION_BATCH_SIZE:200}
|
||||
scan-retention-cron: ${AIIMAGE_SHOP_DUPLICATE_SCAN_RETENTION_CRON:0 50 3 * * *}
|
||||
permission-schema-init:
|
||||
|
||||
Reference in New Issue
Block a user