From 8fcceb32267e629fb752b65c1c3dbd628b5c2651 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E8=87=AA=E8=BE=BE?= <980324341@qq.com> Date: Wed, 16 Sep 2026 09:25:30 +0800 Subject: [PATCH] =?UTF-8?q?fix(=E6=92=9E=E6=AC=BE=E6=89=AB=E6=8F=8F):=20?= =?UTF-8?q?=E4=BF=9D=E7=95=99=E6=9C=9F=2090=E2=86=927=20=E5=A4=A9=EF=BC=8C?= =?UTF-8?q?=E5=B9=B6=E5=8D=95=E7=8B=AC=E5=85=9C=E4=BD=8F=E6=9C=80=E6=96=B0?= =?UTF-8?q?=20SUCCESS=20=E8=A1=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 该表每行含整份聚合 payload（线上实测约 2.3MB/行，24 行占 55MB）， 而读取侧只认最新一行（selectLatestFullRow/LightRow 都是 ORDER BY id DESC LIMIT 1）， 历史行纯占磁盘。保留期降到 7 天，行数下限仍由 PROTECTED_ROWS=10 兜底。 同时补一条保护线：连续失败多日时，最新 SUCCESS 行会滑出「最新 N 行」保护窗口， 再按时间线被删掉，界面就空白了——现在取「第 N 新行」与「最新 SUCCESS 行」 两者更靠前的那个 id 作为保护线，确保它永不被删。 --- ...ShopDataDuplicateScanRetentionService.java | 21 ++++++++++++--- .../src/main/resources/application.yml | 4 ++- ...DataDuplicateScanRetentionServiceTest.java | 27 +++++++++++++++++++ 3 files changed, 47 insertions(+), 5 deletions(-) diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/service/ShopDataDuplicateScanRetentionService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/service/ShopDataDuplicateScanRetentionService.java index 217c63cb..03d5e411 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/service/ShopDataDuplicateScanRetentionService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/shopduplicatecheck/service/ShopDataDuplicateScanRetentionService.java @@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.shopduplicatecheck.service; import com.nanri.aiimage.common.service.DistributedJobLockService; import com.nanri.aiimage.modules.shopduplicatecheck.mapper.ShopDataDuplicateScanMapper; +import com.nanri.aiimage.modules.shopduplicatecheck.model.dto.ScanLightRowDto; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; @@ -35,8 +36,8 @@ public class ShopDataDuplicateScanRetentionService { private final ShopDataDuplicateScanMapper scanMapper; private final DistributedJobLockService distributedJobLockService; - @Value("${aiimage.shop-duplicate-check.scan-retention-days:90}") - private int retentionDays = 90; + @Value("${aiimage.shop-duplicate-check.scan-retention-days:7}") + private int retentionDays = 7; @Value("${aiimage.shop-duplicate-check.scan-retention-batch-size:200}") private int retentionBatchSize = 200; @@ -54,9 +55,13 @@ public class ShopDataDuplicateScanRetentionService { return; } try (lockHandle) { + // 两条保护线取更靠前（更小）的那个 id——保护线之内的行一律不删： + // ① 最新的 PROTECTED_ROWS 行：万一扫描停摆很久，界面上仍能看到最后一份结果； + // ② 最新 SUCCESS 行：读侧只认它。连续失败多日时它可能已滑出保护窗口， + // 按时间线会被删掉、界面直接空白，所以单独兜一条。 Long nthNewestId = scanMapper.selectNthNewestId(PROTECTED_ROWS - 1); - // 行数不足保护量时没什么可删，直接返回 - long protectFromId = nthNewestId == null ? Long.MAX_VALUE : nthNewestId; + ScanLightRowDto latestSuccess = scanMapper.selectLatestLightRow(); + long protectFromId = minId(nthNewestId, latestSuccess == null ? null : latestSuccess.getId()); int totalDeleted = 0; int batches = 0; while (batches < MAX_BATCHES_PER_RUN) { @@ -73,4 +78,12 @@ public class ShopDataDuplicateScanRetentionService { log.warn("[shop-duplicate-check] 扫描结果保留清理失败 cutoff={} msg={}", cutoff, ex.getMessage(), ex); } } + + /** 取两条保护线里更靠前（更小）的 id；都为 null 表示无需保护（等价于不设限）。 */ + private static long minId(Long first, Long second) { + if (first == null) { + return second == null ? Long.MAX_VALUE : second; + } + return second == null ? first : Math.min(first, second); + } } diff --git a/backend-java/src/main/resources/application.yml b/backend-java/src/main/resources/application.yml index 2eecbe7d..07776513 100644 --- a/backend-java/src/main/resources/application.yml +++ b/backend-java/src/main/resources/application.yml @@ -233,7 +233,9 @@ aiimage: loop-run-retention-batch-size: ${AIIMAGE_PRICE_TRACK_LOOP_RUN_RETENTION_BATCH_SIZE:500} loop-run-retention-cron: ${AIIMAGE_PRICE_TRACK_LOOP_RUN_RETENTION_CRON:0 20 4 * * *} shop-duplicate-check: - scan-retention-days: ${AIIMAGE_SHOP_DUPLICATE_SCAN_RETENTION_DAYS:90} + # 每行含整份聚合 payload（实测约 2.3MB/行），而读取侧只认最新一行， + # 保留期给 7 天足够排查；行数下限由服务里的 PROTECTED_ROWS 兜底 + scan-retention-days: ${AIIMAGE_SHOP_DUPLICATE_SCAN_RETENTION_DAYS:7} scan-retention-batch-size: ${AIIMAGE_SHOP_DUPLICATE_SCAN_RETENTION_BATCH_SIZE:200} scan-retention-cron: ${AIIMAGE_SHOP_DUPLICATE_SCAN_RETENTION_CRON:0 50 3 * * *} permission-schema-init: diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/shopduplicatecheck/service/ShopDataDuplicateScanRetentionServiceTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/shopduplicatecheck/service/ShopDataDuplicateScanRetentionServiceTest.java index d7877dbf..4cbc67ce 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/shopduplicatecheck/service/ShopDataDuplicateScanRetentionServiceTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/shopduplicatecheck/service/ShopDataDuplicateScanRetentionServiceTest.java @@ -2,10 +2,12 @@ package com.nanri.aiimage.modules.shopduplicatecheck.service; import com.nanri.aiimage.common.service.DistributedJobLockService; import com.nanri.aiimage.modules.shopduplicatecheck.mapper.ShopDataDuplicateScanMapper; +import com.nanri.aiimage.modules.shopduplicatecheck.model.dto.ScanLightRowDto; import org.junit.jupiter.api.Test; import java.time.LocalDateTime; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyInt; @@ -94,4 +96,29 @@ class ShopDataDuplicateScanRetentionServiceTest { verify(scanMapper).deleteOlderThanBatch(any(LocalDateTime.class), protectLine.capture(), anyInt()); assertTrue(protectLine.getValue() == Long.MAX_VALUE, "保护线为 Long.MAX_VALUE，等于不额外设限"); } + + /** + * 连续失败多日时，最新 SUCCESS 行可能已滑出「最新 N 行」保护窗口； + * 读侧只认它，被删掉界面就空白了，必须单独兜一条更靠前的保护线。 + */ + @Test + void latestSuccessRowIsAlwaysProtected() { + lockAvailable(); + when(scanMapper.selectNthNewestId(anyInt())).thenReturn(500L); + when(scanMapper.selectLatestLightRow()).thenReturn(successRow(120L)); + when(scanMapper.deleteOlderThanBatch(any(LocalDateTime.class), anyLong(), anyInt())).thenReturn(0); + + service().purgeExpiredScans(); + + ArgumentCaptor protectLine = ArgumentCaptor.forClass(Long.class); + verify(scanMapper).deleteOlderThanBatch(any(LocalDateTime.class), protectLine.capture(), anyInt()); + assertEquals(120L, protectLine.getValue(), "取更靠前的保护线，SUCCESS 行不会被删"); + } + + private static ScanLightRowDto successRow(Long id) { + ScanLightRowDto row = new ScanLightRowDto(); + row.setId(id); + row.setCreatedAt(LocalDateTime.now().minusDays(30)); + return row; + } }