fix(撞款扫描): 保留期 90→7 天,并单独兜住最新 SUCCESS 行
该表每行含整份聚合 payload(线上实测约 2.3MB/行,24 行占 55MB), 而读取侧只认最新一行(selectLatestFullRow/LightRow 都是 ORDER BY id DESC LIMIT 1), 历史行纯占磁盘。保留期降到 7 天,行数下限仍由 PROTECTED_ROWS=10 兜底。 同时补一条保护线:连续失败多日时,最新 SUCCESS 行会滑出「最新 N 行」保护窗口, 再按时间线被删掉,界面就空白了——现在取「第 N 新行」与「最新 SUCCESS 行」 两者更靠前的那个 id 作为保护线,确保它永不被删。
This commit is contained in:
+17
-4
@@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.shopduplicatecheck.service;
|
||||
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.modules.shopduplicatecheck.mapper.ShopDataDuplicateScanMapper;
|
||||
import com.nanri.aiimage.modules.shopduplicatecheck.model.dto.ScanLightRowDto;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
@@ -35,8 +36,8 @@ public class ShopDataDuplicateScanRetentionService {
|
||||
private final ShopDataDuplicateScanMapper scanMapper;
|
||||
private final DistributedJobLockService distributedJobLockService;
|
||||
|
||||
@Value("${aiimage.shop-duplicate-check.scan-retention-days:90}")
|
||||
private int retentionDays = 90;
|
||||
@Value("${aiimage.shop-duplicate-check.scan-retention-days:7}")
|
||||
private int retentionDays = 7;
|
||||
|
||||
@Value("${aiimage.shop-duplicate-check.scan-retention-batch-size:200}")
|
||||
private int retentionBatchSize = 200;
|
||||
@@ -54,9 +55,13 @@ public class ShopDataDuplicateScanRetentionService {
|
||||
return;
|
||||
}
|
||||
try (lockHandle) {
|
||||
// 两条保护线取更靠前(更小)的那个 id——保护线之内的行一律不删:
|
||||
// ① 最新的 PROTECTED_ROWS 行:万一扫描停摆很久,界面上仍能看到最后一份结果;
|
||||
// ② 最新 SUCCESS 行:读侧只认它。连续失败多日时它可能已滑出保护窗口,
|
||||
// 按时间线会被删掉、界面直接空白,所以单独兜一条。
|
||||
Long nthNewestId = scanMapper.selectNthNewestId(PROTECTED_ROWS - 1);
|
||||
// 行数不足保护量时没什么可删,直接返回
|
||||
long protectFromId = nthNewestId == null ? Long.MAX_VALUE : nthNewestId;
|
||||
ScanLightRowDto latestSuccess = scanMapper.selectLatestLightRow();
|
||||
long protectFromId = minId(nthNewestId, latestSuccess == null ? null : latestSuccess.getId());
|
||||
int totalDeleted = 0;
|
||||
int batches = 0;
|
||||
while (batches < MAX_BATCHES_PER_RUN) {
|
||||
@@ -73,4 +78,12 @@ public class ShopDataDuplicateScanRetentionService {
|
||||
log.warn("[shop-duplicate-check] 扫描结果保留清理失败 cutoff={} msg={}", cutoff, ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
|
||||
/** 取两条保护线里更靠前(更小)的 id;都为 null 表示无需保护(等价于不设限)。 */
|
||||
private static long minId(Long first, Long second) {
|
||||
if (first == null) {
|
||||
return second == null ? Long.MAX_VALUE : second;
|
||||
}
|
||||
return second == null ? first : Math.min(first, second);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -233,7 +233,9 @@ aiimage:
|
||||
loop-run-retention-batch-size: ${AIIMAGE_PRICE_TRACK_LOOP_RUN_RETENTION_BATCH_SIZE:500}
|
||||
loop-run-retention-cron: ${AIIMAGE_PRICE_TRACK_LOOP_RUN_RETENTION_CRON:0 20 4 * * *}
|
||||
shop-duplicate-check:
|
||||
scan-retention-days: ${AIIMAGE_SHOP_DUPLICATE_SCAN_RETENTION_DAYS:90}
|
||||
# 每行含整份聚合 payload(实测约 2.3MB/行),而读取侧只认最新一行,
|
||||
# 保留期给 7 天足够排查;行数下限由服务里的 PROTECTED_ROWS 兜底
|
||||
scan-retention-days: ${AIIMAGE_SHOP_DUPLICATE_SCAN_RETENTION_DAYS:7}
|
||||
scan-retention-batch-size: ${AIIMAGE_SHOP_DUPLICATE_SCAN_RETENTION_BATCH_SIZE:200}
|
||||
scan-retention-cron: ${AIIMAGE_SHOP_DUPLICATE_SCAN_RETENTION_CRON:0 50 3 * * *}
|
||||
permission-schema-init:
|
||||
|
||||
+27
@@ -2,10 +2,12 @@ package com.nanri.aiimage.modules.shopduplicatecheck.service;
|
||||
|
||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||
import com.nanri.aiimage.modules.shopduplicatecheck.mapper.ShopDataDuplicateScanMapper;
|
||||
import com.nanri.aiimage.modules.shopduplicatecheck.model.dto.ScanLightRowDto;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.anyInt;
|
||||
@@ -94,4 +96,29 @@ class ShopDataDuplicateScanRetentionServiceTest {
|
||||
verify(scanMapper).deleteOlderThanBatch(any(LocalDateTime.class), protectLine.capture(), anyInt());
|
||||
assertTrue(protectLine.getValue() == Long.MAX_VALUE, "保护线为 Long.MAX_VALUE,等于不额外设限");
|
||||
}
|
||||
|
||||
/**
|
||||
* 连续失败多日时,最新 SUCCESS 行可能已滑出「最新 N 行」保护窗口;
|
||||
* 读侧只认它,被删掉界面就空白了,必须单独兜一条更靠前的保护线。
|
||||
*/
|
||||
@Test
|
||||
void latestSuccessRowIsAlwaysProtected() {
|
||||
lockAvailable();
|
||||
when(scanMapper.selectNthNewestId(anyInt())).thenReturn(500L);
|
||||
when(scanMapper.selectLatestLightRow()).thenReturn(successRow(120L));
|
||||
when(scanMapper.deleteOlderThanBatch(any(LocalDateTime.class), anyLong(), anyInt())).thenReturn(0);
|
||||
|
||||
service().purgeExpiredScans();
|
||||
|
||||
ArgumentCaptor<Long> protectLine = ArgumentCaptor.forClass(Long.class);
|
||||
verify(scanMapper).deleteOlderThanBatch(any(LocalDateTime.class), protectLine.capture(), anyInt());
|
||||
assertEquals(120L, protectLine.getValue(), "取更靠前的保护线,SUCCESS 行不会被删");
|
||||
}
|
||||
|
||||
private static ScanLightRowDto successRow(Long id) {
|
||||
ScanLightRowDto row = new ScanLightRowDto();
|
||||
row.setId(id);
|
||||
row.setCreatedAt(LocalDateTime.now().minusDays(30));
|
||||
return row;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user