fix(撞款扫描): 保留期 90→7 天,并单独兜住最新 SUCCESS 行

该表每行含整份聚合 payload(线上实测约 2.3MB/行,24 行占 55MB),
而读取侧只认最新一行(selectLatestFullRow/LightRow 都是 ORDER BY id DESC LIMIT 1),
历史行纯占磁盘。保留期降到 7 天,行数下限仍由 PROTECTED_ROWS=10 兜底。

同时补一条保护线:连续失败多日时,最新 SUCCESS 行会滑出「最新 N 行」保护窗口,
再按时间线被删掉,界面就空白了——现在取「第 N 新行」与「最新 SUCCESS 行」
两者更靠前的那个 id 作为保护线,确保它永不被删。
This commit is contained in:
2026-09-16 09:25:30 +08:00
parent ed0d6575c8
commit 8fcceb3226
3 changed files with 47 additions and 5 deletions
@@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.shopduplicatecheck.service;
import com.nanri.aiimage.common.service.DistributedJobLockService;
import com.nanri.aiimage.modules.shopduplicatecheck.mapper.ShopDataDuplicateScanMapper;
import com.nanri.aiimage.modules.shopduplicatecheck.model.dto.ScanLightRowDto;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
@@ -35,8 +36,8 @@ public class ShopDataDuplicateScanRetentionService {
private final ShopDataDuplicateScanMapper scanMapper;
private final DistributedJobLockService distributedJobLockService;
@Value("${aiimage.shop-duplicate-check.scan-retention-days:90}")
private int retentionDays = 90;
@Value("${aiimage.shop-duplicate-check.scan-retention-days:7}")
private int retentionDays = 7;
@Value("${aiimage.shop-duplicate-check.scan-retention-batch-size:200}")
private int retentionBatchSize = 200;
@@ -54,9 +55,13 @@ public class ShopDataDuplicateScanRetentionService {
return;
}
try (lockHandle) {
// 两条保护线取更靠前(更小)的那个 id——保护线之内的行一律不删:
// ① 最新的 PROTECTED_ROWS 行:万一扫描停摆很久,界面上仍能看到最后一份结果;
// ② 最新 SUCCESS 行:读侧只认它。连续失败多日时它可能已滑出保护窗口,
// 按时间线会被删掉、界面直接空白,所以单独兜一条。
Long nthNewestId = scanMapper.selectNthNewestId(PROTECTED_ROWS - 1);
// 行数不足保护量时没什么可删,直接返回
long protectFromId = nthNewestId == null ? Long.MAX_VALUE : nthNewestId;
ScanLightRowDto latestSuccess = scanMapper.selectLatestLightRow();
long protectFromId = minId(nthNewestId, latestSuccess == null ? null : latestSuccess.getId());
int totalDeleted = 0;
int batches = 0;
while (batches < MAX_BATCHES_PER_RUN) {
@@ -73,4 +78,12 @@ public class ShopDataDuplicateScanRetentionService {
log.warn("[shop-duplicate-check] 扫描结果保留清理失败 cutoff={} msg={}", cutoff, ex.getMessage(), ex);
}
}
/** 取两条保护线里更靠前(更小)的 id;都为 null 表示无需保护(等价于不设限)。 */
private static long minId(Long first, Long second) {
if (first == null) {
return second == null ? Long.MAX_VALUE : second;
}
return second == null ? first : Math.min(first, second);
}
}
@@ -233,7 +233,9 @@ aiimage:
loop-run-retention-batch-size: ${AIIMAGE_PRICE_TRACK_LOOP_RUN_RETENTION_BATCH_SIZE:500}
loop-run-retention-cron: ${AIIMAGE_PRICE_TRACK_LOOP_RUN_RETENTION_CRON:0 20 4 * * *}
shop-duplicate-check:
scan-retention-days: ${AIIMAGE_SHOP_DUPLICATE_SCAN_RETENTION_DAYS:90}
# 每行含整份聚合 payload(实测约 2.3MB/行),而读取侧只认最新一行,
# 保留期给 7 天足够排查;行数下限由服务里的 PROTECTED_ROWS 兜底
scan-retention-days: ${AIIMAGE_SHOP_DUPLICATE_SCAN_RETENTION_DAYS:7}
scan-retention-batch-size: ${AIIMAGE_SHOP_DUPLICATE_SCAN_RETENTION_BATCH_SIZE:200}
scan-retention-cron: ${AIIMAGE_SHOP_DUPLICATE_SCAN_RETENTION_CRON:0 50 3 * * *}
permission-schema-init:
@@ -2,10 +2,12 @@ package com.nanri.aiimage.modules.shopduplicatecheck.service;
import com.nanri.aiimage.common.service.DistributedJobLockService;
import com.nanri.aiimage.modules.shopduplicatecheck.mapper.ShopDataDuplicateScanMapper;
import com.nanri.aiimage.modules.shopduplicatecheck.model.dto.ScanLightRowDto;
import org.junit.jupiter.api.Test;
import java.time.LocalDateTime;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyInt;
@@ -94,4 +96,29 @@ class ShopDataDuplicateScanRetentionServiceTest {
verify(scanMapper).deleteOlderThanBatch(any(LocalDateTime.class), protectLine.capture(), anyInt());
assertTrue(protectLine.getValue() == Long.MAX_VALUE, "保护线为 Long.MAX_VALUE,等于不额外设限");
}
/**
* 连续失败多日时,最新 SUCCESS 行可能已滑出「最新 N 行」保护窗口;
* 读侧只认它,被删掉界面就空白了,必须单独兜一条更靠前的保护线。
*/
@Test
void latestSuccessRowIsAlwaysProtected() {
lockAvailable();
when(scanMapper.selectNthNewestId(anyInt())).thenReturn(500L);
when(scanMapper.selectLatestLightRow()).thenReturn(successRow(120L));
when(scanMapper.deleteOlderThanBatch(any(LocalDateTime.class), anyLong(), anyInt())).thenReturn(0);
service().purgeExpiredScans();
ArgumentCaptor<Long> protectLine = ArgumentCaptor.forClass(Long.class);
verify(scanMapper).deleteOlderThanBatch(any(LocalDateTime.class), protectLine.capture(), anyInt());
assertEquals(120L, protectLine.getValue(), "取更靠前的保护线,SUCCESS 行不会被删");
}
private static ScanLightRowDto successRow(Long id) {
ScanLightRowDto row = new ScanLightRowDto();
row.setId(id);
row.setCreatedAt(LocalDateTime.now().minusDays(30));
return row;
}
}