后台更新 服务优化

backend-java/src/test/java/com/nanri/aiimage/modules/dedupe/service/DedupeTotalDataServiceTest.java

@@ -43,6 +43,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
@@ -172,7 +173,8 @@ class DedupeTotalDataServiceTest {
     void duplicateExcelValueKeepsOriginalUploader() throws Exception {
         when(adminUserMapper.selectById(23L)).thenReturn(user(23L, "normal", "member-a"));
         stubWritableGroup(23L, 7L);
-        when(dedupeTotalDataMapper.selectOne(any())).thenReturn(data(91L, 99L));
+        when(dedupeTotalDataMapper.selectExistingDataValues(List.of("B012345678")))
+                .thenReturn(List.of("B012345678"));
         MockMultipartFile file = asinWorkbook("B012345678");
 
         var result = service.importFromExcel(file, 7L, 23L);
@@ -187,7 +189,10 @@ class DedupeTotalDataServiceTest {
         when(adminUserMapper.selectById(23L)).thenReturn(user(23L, "normal", "member-a"));
         stubWritableGroup(23L, 7L);
         when(dedupeTotalDataMapper.selectOne(any())).thenReturn(null);
+        when(dedupeTotalDataMapper.selectExistingDataValues(List.of("B012345678"))).thenReturn(List.of());
         when(transactionManager.getTransaction(any())).thenReturn(transactionStatus);
+        when(dedupeTotalDataMapper.insertBatchIgnore(any()))
+                .thenThrow(new DuplicateKeyException("duplicate batch"));
         when(dedupeTotalDataMapper.insert(any(DedupeTotalDataEntity.class)))
                 .thenThrow(new DuplicateKeyException("duplicate"));
 
@@ -195,7 +200,7 @@ class DedupeTotalDataServiceTest {
 
         assertEquals(0, result.getInsertedCount());
         assertEquals(1, result.getSkippedCount());
-        verify(transactionManager).rollback(transactionStatus);
+        verify(transactionManager, times(2)).rollback(transactionStatus);
     }
 
     @Test

backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskServiceChunkTest.java

@@ -47,6 +47,7 @@ import java.util.Objects;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
@@ -124,7 +125,8 @@ class ShopDataCrawlTaskServiceChunkTest {
                 taskScopeStateMapper,
                 transientPayloadStorageService,
                 instanceMetadata,
-                dailyFileService);
+                dailyFileService,
+                null);
 
         storedChunks.clear();
         storedScopes.clear();
@@ -183,6 +185,10 @@ class ShopDataCrawlTaskServiceChunkTest {
         assertTrue(task.getResultJson().indexOf("B001") < task.getResultJson().indexOf("B002"));
         verify(taskFileJobService).enqueueAssembleResult(task.getId(), MODULE_TYPE, result.getId(),
                 "task:" + task.getId() + ":owner:instance-a");
+        assertNull(result.getResultFileUrl());
+        verify(excelAssemblyService, never()).writeWorkbook(any(), any());
+        verify(excelAssemblyService, never()).appendWorkbook(any(), any(), any());
+        verify(ossStorageService, never()).uploadResultFile(any(), eq(MODULE_TYPE));
     }
 
     @Test

backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskServiceRetentionTest.java

@@ -36,15 +36,18 @@ import org.mockito.junit.jupiter.MockitoSettings;
 import org.mockito.quality.Strictness;
 import org.mockito.Spy;
 import org.mockito.junit.jupiter.MockitoExtension;
-import org.springframework.transaction.support.TransactionSynchronization;
-import org.springframework.transaction.support.TransactionSynchronizationManager;
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.TransactionStatus;
 
 import java.time.LocalDate;
 import java.time.LocalDateTime;
 import java.util.List;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.mockito.ArgumentMatchers.any;
@@ -94,6 +97,8 @@ class ShopDataCrawlTaskServiceRetentionTest {
     @Mock private TransientPayloadStorageService transientPayloadStorageService;
     @Mock private InstanceMetadata instanceMetadata;
     @Mock private ShopDataCrawlDailyFileService dailyFileService;
+    @Mock private PlatformTransactionManager transactionManager;
+    @Mock private TransactionStatus transactionStatus;
     @Spy private final ObjectMapper objectMapper = new ObjectMapper().findAndRegisterModules();
     @InjectMocks private ShopDataCrawlTaskService service;
 
@@ -128,6 +133,7 @@ class ShopDataCrawlTaskServiceRetentionTest {
         when(dailyFileService.findOlder(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(List.of());
         when(dailyFileService.countObjectReferences(anyString())).thenReturn(0L);
         when(dailyFileService.addMember(anyLong(), anyLong(), anyLong())).thenReturn(true);
+        when(transactionManager.getTransaction(any())).thenReturn(transactionStatus);
         doAnswer(invocation -> {
             ShopDataCrawlDailyFileEntity entity = invocation.getArgument(0);
             entity.setId(301L);
@@ -153,6 +159,7 @@ class ShopDataCrawlTaskServiceRetentionTest {
         verify(dailyFileService).insert(captor.capture());
         assertEquals(BUSINESS_DATE, captor.getValue().getBusinessDate());
         assertEquals(1L, captor.getValue().getVersion());
+        assertEquals(currentRow.getResultFileUrl(), captor.getValue().getResultFileUrl());
     }
 
     @Test
@@ -169,25 +176,60 @@ class ShopDataCrawlTaskServiceRetentionTest {
         when(ossStorageService.readObjectBytes("result/old.xlsx")).thenReturn(new byte[]{1, 2, 3});
         when(ossStorageService.uploadResultFile(any(), eq(MODULE_TYPE))).thenReturn("result/new.xlsx");
 
-        TransactionSynchronizationManager.initSynchronization();
-        try {
-            service.processResultFileJob(job);
+        service.processResultFileJob(job);
 
-            verify(excelAssemblyService).appendWorkbook(any(), any(), eq(List.of(snapshot)));
-            verify(dailyFileService).update(daily);
-            verify(ossStorageService, never()).deleteObject("result/old.xlsx");
-            assertNull(previous.getResultFileUrl());
-            assertEquals("result/new.xlsx", currentRow.getResultFileUrl());
-            assertEquals(3, currentRow.getRowCount());
+        verify(excelAssemblyService).appendWorkbook(any(), any(), eq(List.of(snapshot)));
+        verify(dailyFileService).update(daily);
+        verify(ossStorageService).deleteObject("result/old.xlsx");
+        assertNull(previous.getResultFileUrl());
+        assertEquals("result/new.xlsx", currentRow.getResultFileUrl());
+        assertEquals(3, currentRow.getRowCount());
+    }
 
-            List<TransactionSynchronization> synchronizations = TransactionSynchronizationManager.getSynchronizations();
-            synchronizations.forEach(TransactionSynchronization::afterCommit);
-            verify(ossStorageService).deleteObject("result/old.xlsx");
-            synchronizations.forEach(synchronization ->
-                    synchronization.afterCompletion(TransactionSynchronization.STATUS_COMMITTED));
-        } finally {
-            TransactionSynchronizationManager.clearSynchronization();
-        }
+    @Test
+    void workbookAndSnapshotStorageRunOutsideShortTransactions() {
+        AtomicBoolean transactionActive = new AtomicBoolean();
+        doAnswer(invocation -> {
+            assertFalse(transactionActive.getAndSet(true), "short transactions must not overlap");
+            return transactionStatus;
+        }).when(transactionManager).getTransaction(any());
+        doAnswer(invocation -> {
+            transactionActive.set(false);
+            return null;
+        }).when(transactionManager).commit(any());
+        doAnswer(invocation -> {
+            transactionActive.set(false);
+            return null;
+        }).when(transactionManager).rollback(any());
+
+        ShopDataCrawlDailyFileEntity daily = daily("result/old.xlsx", 2);
+        when(fileResultMapper.selectList(any())).thenReturn(List.of(currentRow));
+        when(dailyFileService.findForUpdate(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(daily);
+        when(dailyFileService.containsResult(301L, RESULT_ID)).thenReturn(false);
+        doAnswer(invocation -> {
+            assertFalse(transactionActive.get(), "row counting must run outside the database transaction");
+            return 1;
+        }).when(excelAssemblyService).countRows(any());
+        doAnswer(invocation -> {
+            assertFalse(transactionActive.get(), "workbook download must run outside the database transaction");
+            return new byte[]{1, 2, 3};
+        }).when(ossStorageService).readObjectBytes("result/old.xlsx");
+        doAnswer(invocation -> {
+            assertFalse(transactionActive.get(), "workbook assembly must run outside the database transaction");
+            return null;
+        }).when(excelAssemblyService).appendWorkbook(any(), any(), any());
+        doAnswer(invocation -> {
+            assertFalse(transactionActive.get(), "workbook upload must run outside the database transaction");
+            return "result/new.xlsx";
+        }).when(ossStorageService).uploadResultFile(any(), eq(MODULE_TYPE));
+        doAnswer(invocation -> {
+            assertFalse(transactionActive.get(), "snapshot payload storage must run outside the database transaction");
+            return null;
+        }).when(taskResultItemService).replaceTaskSnapshots(anyLong(), eq(MODULE_TYPE), any(), any());
+
+        service.processResultFileJob(job);
+
+        assertFalse(transactionActive.get());
     }
 
     @Test
@@ -207,6 +249,26 @@ class ShopDataCrawlTaskServiceRetentionTest {
         assertEquals(3, currentRow.getRowCount());
     }
 
+    @Test
+    void zeroNewRowsReuseDailyObjectWithoutWorkbookIo() {
+        ShopDataCrawlDailyFileEntity daily = daily("result/current.xlsx", 3);
+        when(fileResultMapper.selectList(any())).thenReturn(List.of(currentRow));
+        when(dailyFileService.findForUpdate(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(daily);
+        when(dailyFileService.containsResult(301L, RESULT_ID)).thenReturn(false);
+        when(excelAssemblyService.countRows(any())).thenReturn(0);
+
+        service.processResultFileJob(job);
+
+        verify(ossStorageService, never()).readObjectBytes(anyString());
+        verify(excelAssemblyService, never()).writeWorkbook(any(), any());
+        verify(excelAssemblyService, never()).appendWorkbook(any(), any(), any());
+        verify(ossStorageService, never()).uploadResultFile(any(), anyString());
+        verify(dailyFileService).addMember(301L, TASK_ID, RESULT_ID);
+        assertEquals("result/current.xlsx", currentRow.getResultFileUrl());
+        assertEquals(10L, currentRow.getResultFileSize());
+        assertEquals(3, currentRow.getRowCount());
+    }
+
     @Test
     void sameDaySuccessKeepsLegacyPointerOwnedByAnotherUser() {
         FileResultEntity previous = result(200L, 100L, "result/old.xlsx");
@@ -287,6 +349,29 @@ class ShopDataCrawlTaskServiceRetentionTest {
         verify(ossStorageService, never()).deleteObject("result/yesterday.xlsx");
     }
 
+    @Test
+    void failedDatabaseCommitDeletesNewUploadButKeepsPreviousObject() {
+        ShopDataCrawlDailyFileEntity yesterday = daily("result/yesterday.xlsx", 4);
+        yesterday.setId(300L);
+        yesterday.setBusinessDate(BUSINESS_DATE.minusDays(1));
+        when(fileResultMapper.selectList(any())).thenReturn(List.of(currentRow));
+        when(dailyFileService.findForUpdate(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(null);
+        when(dailyFileService.findOlder(USER_ID, "hash-1", BUSINESS_DATE)).thenReturn(List.of(yesterday));
+        when(ossStorageService.uploadResultFile(any(), eq(MODULE_TYPE))).thenReturn("result/new.xlsx");
+        AtomicInteger commitCount = new AtomicInteger();
+        doAnswer(invocation -> {
+            if (commitCount.incrementAndGet() == 2) {
+                throw new IllegalStateException("commit failed");
+            }
+            return null;
+        }).when(transactionManager).commit(any());
+
+        assertThrows(IllegalStateException.class, () -> service.processResultFileJob(job));
+
+        verify(ossStorageService).deleteObject("result/new.xlsx");
+        verify(ossStorageService, never()).deleteObject("result/yesterday.xlsx");
+    }
+
     @Test
     void deletingOneMemberRebuildsDailyWorkbookFromRemainingResults() {
         task.setStatus("SUCCESS");

backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/util/SimilarAsinImageEmbedderTest.java

@@ -28,6 +28,9 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -64,9 +67,77 @@ class SimilarAsinImageEmbedderTest {
     }
 
     @Test
-    void defaultsImageDownloadPoolToEight() {
-        assertEquals(8, new SimilarAsinProperties().getImageDownloadPoolSize());
-        assertEquals(8, embedder.downloadPoolSize());
+    void defaultsImageDownloadPoolToTwoAndCapsItByVisibleCpu() {
+        assertEquals(2, new SimilarAsinProperties().getImageDownloadPoolSize());
+        assertEquals(Math.min(2, SimilarAsinImageEmbedder.cpuBoundPoolLimit()), embedder.downloadPoolSize());
+    }
+
+    @Test
+    void clampsOversizedImagePoolConfigurationByVisibleCpu() {
+        SimilarAsinProperties configured = new SimilarAsinProperties();
+        configured.setImageDownloadPoolSize(Integer.MAX_VALUE);
+        SimilarAsinImageEmbedder limited = new SimilarAsinImageEmbedder(configured, createOssStorageService());
+        try {
+            assertEquals(SimilarAsinImageEmbedder.cpuBoundPoolLimit(), limited.downloadPoolSize());
+        } finally {
+            limited.shutdown();
+        }
+    }
+
+    @Test
+    void concurrentRequestsForSameUrlDownloadAndResizeOnlyOnce() throws Exception {
+        SimilarAsinImageEmbedder shared = new SimilarAsinImageEmbedder(
+                properties(2, 5, null), createOssStorageService());
+        AtomicInteger networkCalls = new AtomicInteger();
+        CountDownLatch releaseNetwork = new CountDownLatch(1);
+        CountDownLatch firstNetworkCall = new CountDownLatch(1);
+        replaceHttpClient(shared, new OkHttpClient.Builder()
+                .addInterceptor(chain -> {
+                    networkCalls.incrementAndGet();
+                    firstNetworkCall.countDown();
+                    try {
+                        if (!releaseNetwork.await(2, TimeUnit.SECONDS)) {
+                            throw new IOException("test network release timed out");
+                        }
+                    } catch (InterruptedException ex) {
+                        Thread.currentThread().interrupt();
+                        throw new IOException("test network interrupted", ex);
+                    }
+                    return response(chain.request(), 200, "OK", createJpegBytes());
+                })
+                .build());
+        ExecutorService callers = Executors.newFixedThreadPool(2);
+        CountDownLatch start = new CountDownLatch(1);
+        try {
+            Future<SimilarAsinImageEmbedder.ResizedImage> first = callers.submit(() -> {
+                start.await();
+                return shared.fetchAndResizeForCache("https://images.example.com/shared.jpg");
+            });
+            Future<SimilarAsinImageEmbedder.ResizedImage> second = callers.submit(() -> {
+                start.await();
+                return shared.fetchAndResizeForCache("https://images.example.com/shared.jpg");
+            });
+
+            start.countDown();
+            assertTrue(firstNetworkCall.await(1, TimeUnit.SECONDS));
+            Thread.sleep(100L);
+            releaseNetwork.countDown();
+
+            assertNotNull(first.get(2, TimeUnit.SECONDS));
+            assertNotNull(second.get(2, TimeUnit.SECONDS));
+            assertEquals(1, networkCalls.get());
+        } finally {
+            releaseNetwork.countDown();
+            callers.shutdownNow();
+            shared.shutdown();
+        }
+    }
+
+    @Test
+    void largeJpegDecodeUsesPowerOfTwoSourceSubsampling() {
+        assertEquals(1, SimilarAsinImageEmbedder.jpegSourceSubsampling(2559, 1200));
+        assertEquals(2, SimilarAsinImageEmbedder.jpegSourceSubsampling(3200, 2400));
+        assertEquals(4, SimilarAsinImageEmbedder.jpegSourceSubsampling(6000, 4000));
     }
 
     @Test

backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskHeartbeatServiceTest.java

@@ -112,11 +112,12 @@ class TaskHeartbeatServiceTest {
         task.setId(taskId);
         task.setModuleType("SHOP_DATA_CRAWL");
         task.setStatus("RUNNING");
+        TaskHeartbeatRequest request = new TaskHeartbeatRequest();
+        request.setModuleType("SHOP_DATA_CRAWL");
         when(fileTaskMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(task);
-        when(brandCrawlTaskMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(null);
         when(fileTaskMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1);
 
-        TaskHeartbeatVo result = service.heartbeat(taskId, new TaskHeartbeatRequest());
+        TaskHeartbeatVo result = service.heartbeat(taskId, request);
 
         assertTrue(result.isAlive());
         InOrder order = inOrder(shopDataCrawlTaskService, fileTaskMapper);
@@ -125,6 +126,7 @@ class TaskHeartbeatServiceTest {
         order.verify(fileTaskMapper).update(isNull(), any(LambdaUpdateWrapper.class));
         verify(shopDataCrawlTaskCacheService).touchTaskHeartbeat(taskId);
         verify(shopDataCrawlTaskCacheService).saveTaskCache(task);
+        verify(brandCrawlTaskMapper, never()).selectOne(any(LambdaQueryWrapper.class));
     }
 
     @Test