fix(撞款扫描): 保留期 90→7 天,并单独兜住最新 SUCCESS 行
该表每行含整份聚合 payload(线上实测约 2.3MB/行,24 行占 55MB), 而读取侧只认最新一行(selectLatestFullRow/LightRow 都是 ORDER BY id DESC LIMIT 1), 历史行纯占磁盘。保留期降到 7 天,行数下限仍由 PROTECTED_ROWS=10 兜底。 同时补一条保护线:连续失败多日时,最新 SUCCESS 行会滑出「最新 N 行」保护窗口, 再按时间线被删掉,界面就空白了——现在取「第 N 新行」与「最新 SUCCESS 行」 两者更靠前的那个 id 作为保护线,确保它永不被删。
This commit is contained in:
+17
-4
@@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.shopduplicatecheck.service;
|
|||||||
|
|
||||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||||
import com.nanri.aiimage.modules.shopduplicatecheck.mapper.ShopDataDuplicateScanMapper;
|
import com.nanri.aiimage.modules.shopduplicatecheck.mapper.ShopDataDuplicateScanMapper;
|
||||||
|
import com.nanri.aiimage.modules.shopduplicatecheck.model.dto.ScanLightRowDto;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.beans.factory.annotation.Value;
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
@@ -35,8 +36,8 @@ public class ShopDataDuplicateScanRetentionService {
|
|||||||
private final ShopDataDuplicateScanMapper scanMapper;
|
private final ShopDataDuplicateScanMapper scanMapper;
|
||||||
private final DistributedJobLockService distributedJobLockService;
|
private final DistributedJobLockService distributedJobLockService;
|
||||||
|
|
||||||
@Value("${aiimage.shop-duplicate-check.scan-retention-days:90}")
|
@Value("${aiimage.shop-duplicate-check.scan-retention-days:7}")
|
||||||
private int retentionDays = 90;
|
private int retentionDays = 7;
|
||||||
|
|
||||||
@Value("${aiimage.shop-duplicate-check.scan-retention-batch-size:200}")
|
@Value("${aiimage.shop-duplicate-check.scan-retention-batch-size:200}")
|
||||||
private int retentionBatchSize = 200;
|
private int retentionBatchSize = 200;
|
||||||
@@ -54,9 +55,13 @@ public class ShopDataDuplicateScanRetentionService {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
try (lockHandle) {
|
try (lockHandle) {
|
||||||
|
// 两条保护线取更靠前(更小)的那个 id——保护线之内的行一律不删:
|
||||||
|
// ① 最新的 PROTECTED_ROWS 行:万一扫描停摆很久,界面上仍能看到最后一份结果;
|
||||||
|
// ② 最新 SUCCESS 行:读侧只认它。连续失败多日时它可能已滑出保护窗口,
|
||||||
|
// 按时间线会被删掉、界面直接空白,所以单独兜一条。
|
||||||
Long nthNewestId = scanMapper.selectNthNewestId(PROTECTED_ROWS - 1);
|
Long nthNewestId = scanMapper.selectNthNewestId(PROTECTED_ROWS - 1);
|
||||||
// 行数不足保护量时没什么可删,直接返回
|
ScanLightRowDto latestSuccess = scanMapper.selectLatestLightRow();
|
||||||
long protectFromId = nthNewestId == null ? Long.MAX_VALUE : nthNewestId;
|
long protectFromId = minId(nthNewestId, latestSuccess == null ? null : latestSuccess.getId());
|
||||||
int totalDeleted = 0;
|
int totalDeleted = 0;
|
||||||
int batches = 0;
|
int batches = 0;
|
||||||
while (batches < MAX_BATCHES_PER_RUN) {
|
while (batches < MAX_BATCHES_PER_RUN) {
|
||||||
@@ -73,4 +78,12 @@ public class ShopDataDuplicateScanRetentionService {
|
|||||||
log.warn("[shop-duplicate-check] 扫描结果保留清理失败 cutoff={} msg={}", cutoff, ex.getMessage(), ex);
|
log.warn("[shop-duplicate-check] 扫描结果保留清理失败 cutoff={} msg={}", cutoff, ex.getMessage(), ex);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** 取两条保护线里更靠前(更小)的 id;都为 null 表示无需保护(等价于不设限)。 */
|
||||||
|
private static long minId(Long first, Long second) {
|
||||||
|
if (first == null) {
|
||||||
|
return second == null ? Long.MAX_VALUE : second;
|
||||||
|
}
|
||||||
|
return second == null ? first : Math.min(first, second);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -233,7 +233,9 @@ aiimage:
|
|||||||
loop-run-retention-batch-size: ${AIIMAGE_PRICE_TRACK_LOOP_RUN_RETENTION_BATCH_SIZE:500}
|
loop-run-retention-batch-size: ${AIIMAGE_PRICE_TRACK_LOOP_RUN_RETENTION_BATCH_SIZE:500}
|
||||||
loop-run-retention-cron: ${AIIMAGE_PRICE_TRACK_LOOP_RUN_RETENTION_CRON:0 20 4 * * *}
|
loop-run-retention-cron: ${AIIMAGE_PRICE_TRACK_LOOP_RUN_RETENTION_CRON:0 20 4 * * *}
|
||||||
shop-duplicate-check:
|
shop-duplicate-check:
|
||||||
scan-retention-days: ${AIIMAGE_SHOP_DUPLICATE_SCAN_RETENTION_DAYS:90}
|
# 每行含整份聚合 payload(实测约 2.3MB/行),而读取侧只认最新一行,
|
||||||
|
# 保留期给 7 天足够排查;行数下限由服务里的 PROTECTED_ROWS 兜底
|
||||||
|
scan-retention-days: ${AIIMAGE_SHOP_DUPLICATE_SCAN_RETENTION_DAYS:7}
|
||||||
scan-retention-batch-size: ${AIIMAGE_SHOP_DUPLICATE_SCAN_RETENTION_BATCH_SIZE:200}
|
scan-retention-batch-size: ${AIIMAGE_SHOP_DUPLICATE_SCAN_RETENTION_BATCH_SIZE:200}
|
||||||
scan-retention-cron: ${AIIMAGE_SHOP_DUPLICATE_SCAN_RETENTION_CRON:0 50 3 * * *}
|
scan-retention-cron: ${AIIMAGE_SHOP_DUPLICATE_SCAN_RETENTION_CRON:0 50 3 * * *}
|
||||||
permission-schema-init:
|
permission-schema-init:
|
||||||
|
|||||||
+27
@@ -2,10 +2,12 @@ package com.nanri.aiimage.modules.shopduplicatecheck.service;
|
|||||||
|
|
||||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||||
import com.nanri.aiimage.modules.shopduplicatecheck.mapper.ShopDataDuplicateScanMapper;
|
import com.nanri.aiimage.modules.shopduplicatecheck.mapper.ShopDataDuplicateScanMapper;
|
||||||
|
import com.nanri.aiimage.modules.shopduplicatecheck.model.dto.ScanLightRowDto;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
import java.time.LocalDateTime;
|
import java.time.LocalDateTime;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
import static org.mockito.ArgumentMatchers.anyLong;
|
import static org.mockito.ArgumentMatchers.anyLong;
|
||||||
import static org.mockito.ArgumentMatchers.anyInt;
|
import static org.mockito.ArgumentMatchers.anyInt;
|
||||||
@@ -94,4 +96,29 @@ class ShopDataDuplicateScanRetentionServiceTest {
|
|||||||
verify(scanMapper).deleteOlderThanBatch(any(LocalDateTime.class), protectLine.capture(), anyInt());
|
verify(scanMapper).deleteOlderThanBatch(any(LocalDateTime.class), protectLine.capture(), anyInt());
|
||||||
assertTrue(protectLine.getValue() == Long.MAX_VALUE, "保护线为 Long.MAX_VALUE,等于不额外设限");
|
assertTrue(protectLine.getValue() == Long.MAX_VALUE, "保护线为 Long.MAX_VALUE,等于不额外设限");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 连续失败多日时,最新 SUCCESS 行可能已滑出「最新 N 行」保护窗口;
|
||||||
|
* 读侧只认它,被删掉界面就空白了,必须单独兜一条更靠前的保护线。
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
void latestSuccessRowIsAlwaysProtected() {
|
||||||
|
lockAvailable();
|
||||||
|
when(scanMapper.selectNthNewestId(anyInt())).thenReturn(500L);
|
||||||
|
when(scanMapper.selectLatestLightRow()).thenReturn(successRow(120L));
|
||||||
|
when(scanMapper.deleteOlderThanBatch(any(LocalDateTime.class), anyLong(), anyInt())).thenReturn(0);
|
||||||
|
|
||||||
|
service().purgeExpiredScans();
|
||||||
|
|
||||||
|
ArgumentCaptor<Long> protectLine = ArgumentCaptor.forClass(Long.class);
|
||||||
|
verify(scanMapper).deleteOlderThanBatch(any(LocalDateTime.class), protectLine.capture(), anyInt());
|
||||||
|
assertEquals(120L, protectLine.getValue(), "取更靠前的保护线,SUCCESS 行不会被删");
|
||||||
|
}
|
||||||
|
|
||||||
|
private static ScanLightRowDto successRow(Long id) {
|
||||||
|
ScanLightRowDto row = new ScanLightRowDto();
|
||||||
|
row.setId(id);
|
||||||
|
row.setCreatedAt(LocalDateTime.now().minusDays(30));
|
||||||
|
return row;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user