super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

性能优化：N+1查询改批量、串行远程调用并发化、紫鸟索引巡检批量回写
Build Backend JAR / build (push) Has been cancelled
Details

Browse Source
This commit is contained in:
黄自达
2026-08-25 19:42:43 +08:00
parent 338493ecaa
commit 5f519f9cd5
14 changed files with 425 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-java/src/test/java/com/nanri/aiimage/modules/shopdatacrawl/service/ShopDataCrawlTaskServiceRetentionTest.java
+1 -1
View File
@@ -388,7 +388,7 @@ class ShopDataCrawlTaskServiceRetentionTest {
when(taskDistributedLockService.acquire(MODULE_TYPE, TASK_ID))
.thenReturn(mock(TaskDistributedLockService.LockHandle.class));
when(fileResultMapper.selectById(RESULT_ID)).thenReturn(currentRow);
when(fileResultMapper.selectById(200L)).thenReturn(previous);
when(fileResultMapper.selectBatchIds(List.of(200L))).thenReturn(List.of(previous));
when(fileResultMapper.selectList(any())).thenReturn(List.of());
when(dailyFileService.findByLatestResultId(RESULT_ID)).thenReturn(List.of(daily));
when(dailyFileService.findMembersByResultId(RESULT_ID)).thenReturn(List.of(removedMember));
backend-java/src/test/java/com/nanri/aiimage/modules/ziniao/service/ZiniaoShopIndexServiceTest.java
+107
View File
@@ -3,6 +3,7 @@ package com.nanri.aiimage.modules.ziniao.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.config.ZiniaoProperties;
import com.nanri.aiimage.modules.ziniao.memory.model.entity.ZiniaoMemoryStoreEntity;
import com.nanri.aiimage.modules.ziniao.memory.service.ZiniaoMemoryStoreService;
import com.nanri.aiimage.modules.ziniao.memory.service.ZiniaoTransientCacheService;
import com.nanri.aiimage.modules.ziniao.model.cache.ZiniaoShopIndexEntryDto;
@@ -17,6 +18,7 @@ import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Optional;
@@ -24,6 +26,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.ArgumentMatchers.argThat;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.never;
@@ -209,6 +212,110 @@ class ZiniaoShopIndexServiceTest {
assertEquals(Integer.valueOf(0), cursor.getNextApiKeyOffset());
}
@Test
void staleRowsWithoutPayloadChangeSkipRewriteAndOnlyTouchExpiry() throws Exception {
ZiniaoApiKeyProvider.ApiKeyAccount key = new ZiniaoApiKeyProvider.ApiKeyAccount("key", "acct");
when(ziniaoApiKeyProvider.listApiKeyAccounts()).thenReturn(List.of(key));
when(ziniaoAuthService.resolveCompanyIdForIndex("key")).thenReturn(1L);
when(ziniaoAuthService.getOrLoadStaffForIndex("key", 1L)).thenReturn(List.of(staff(11L)));
when(ziniaoAuthService.getOrLoadUserStoresForIndex("key", 1L, 11L)).thenReturn(List.of());
ZiniaoShopIndexEntryDto dto = new ZiniaoShopIndexEntryDto();
dto.setNormalizedShopName("shop-a");
dto.setCompanyName("acct");
dto.setStatus("STALE");
dto.setMessage("店铺索引已过期，请等待后台刷新");
dto.setLastRefreshedAt(1000L);
String payload = new ObjectMapper().writeValueAsString(dto);
ZiniaoMemoryStoreEntity row = new ZiniaoMemoryStoreEntity();
row.setId(1L);
row.setCacheType("SHOP_INDEX_ENTRY");
row.setCacheKey("n:shop-a");
row.setCompanyName("acct");
row.setPayloadJson(payload);
row.setExpiresAt(LocalDateTime.now().plusHours(12));
when(ziniaoMemoryStoreService.listAliveEntitiesByType("SHOP_INDEX_ENTRY", 10000)).thenReturn(List.of(row));
service.refreshAllShopIndex();
verify(ziniaoMemoryStoreService, never()).updateStaleMarks(anyList());
verify(ziniaoMemoryStoreService).touchExpiryBatch(anyList(), any(Duration.class));
verify(ziniaoMemoryStoreService, never()).put(any(), any(), any(), any());
}
@Test
void activeEntryMissingFromThisRoundGetsRewrittenAsStale() throws Exception {
ZiniaoApiKeyProvider.ApiKeyAccount key = new ZiniaoApiKeyProvider.ApiKeyAccount("key", "acct");
when(ziniaoApiKeyProvider.listApiKeyAccounts()).thenReturn(List.of(key));
when(ziniaoAuthService.resolveCompanyIdForIndex("key")).thenReturn(1L);
when(ziniaoAuthService.getOrLoadStaffForIndex("key", 1L)).thenReturn(List.of(staff(11L)));
when(ziniaoAuthService.getOrLoadUserStoresForIndex("key", 1L, 11L)).thenReturn(List.of());
ZiniaoShopIndexEntryDto dto = new ZiniaoShopIndexEntryDto();
dto.setNormalizedShopName("shop-a");
dto.setStatus("ACTIVE");
dto.setLastRefreshedAt(1000L);
String payload = new ObjectMapper().writeValueAsString(dto);
ZiniaoMemoryStoreEntity row = new ZiniaoMemoryStoreEntity();
row.setId(1L);
row.setCacheType("SHOP_INDEX_ENTRY");
row.setCacheKey("n:shop-a");
row.setCompanyName("acct");
row.setPayloadJson(payload);
row.setExpiresAt(LocalDateTime.now().plusHours(12));
when(ziniaoMemoryStoreService.listAliveEntitiesByType("SHOP_INDEX_ENTRY", 10000)).thenReturn(List.of(row));
service.refreshAllShopIndex();
ArgumentCaptor<List<ZiniaoMemoryStoreEntity>> captor = ArgumentCaptor.forClass(List.class);
verify(ziniaoMemoryStoreService).updateStaleMarks(captor.capture());
assertEquals(1, captor.getValue().size());
ZiniaoMemoryStoreEntity updated = captor.getValue().get(0);
assertEquals(1L, updated.getId());
assertTrue(updated.getPayloadJson().contains("\"STALE\""));
assertEquals("店铺索引已过期，请等待后台刷新",
new ObjectMapper().readTree(updated.getPayloadJson()).get("message").asText());
verify(ziniaoMemoryStoreService, never()).touchExpiryBatch(anyList(), any(Duration.class));
}
@Test
void activeKeysFromThisRoundAreSkippedInStaleMarking() throws Exception {
ZiniaoApiKeyProvider.ApiKeyAccount key = new ZiniaoApiKeyProvider.ApiKeyAccount("key", "acct");
when(ziniaoApiKeyProvider.listApiKeyAccounts()).thenReturn(List.of(key));
when(ziniaoAuthService.resolveCompanyIdForIndex("key")).thenReturn(1L);
when(ziniaoAuthService.getOrLoadStaffForIndex("key", 1L)).thenReturn(List.of(staff(11L)));
when(ziniaoAuthService.getOrLoadUserStoresForIndex("key", 1L, 11L)).thenReturn(List.of(shop("s-1", "shop-1")));
ZiniaoShopIndexEntryDto activeDto = new ZiniaoShopIndexEntryDto();
activeDto.setNormalizedShopName("shop-1");
activeDto.setStatus("ACTIVE");
ZiniaoMemoryStoreEntity activeRow = new ZiniaoMemoryStoreEntity();
activeRow.setId(1L);
activeRow.setCacheType("SHOP_INDEX_ENTRY");
activeRow.setCacheKey("s:s-1");
activeRow.setPayloadJson(new ObjectMapper().writeValueAsString(activeDto));
activeRow.setExpiresAt(LocalDateTime.now().plusHours(12));
ZiniaoShopIndexEntryDto staleDto = new ZiniaoShopIndexEntryDto();
staleDto.setNormalizedShopName("shop-2");
staleDto.setStatus("ACTIVE");
ZiniaoMemoryStoreEntity staleRow = new ZiniaoMemoryStoreEntity();
staleRow.setId(2L);
staleRow.setCacheType("SHOP_INDEX_ENTRY");
staleRow.setCacheKey("n:shop-2");
staleRow.setPayloadJson(new ObjectMapper().writeValueAsString(staleDto));
staleRow.setExpiresAt(LocalDateTime.now().plusHours(12));
when(ziniaoMemoryStoreService.listAliveEntitiesByType("SHOP_INDEX_ENTRY", 10000))
.thenReturn(List.of(activeRow, staleRow));
service.refreshAllShopIndex();
ArgumentCaptor<List<ZiniaoMemoryStoreEntity>> captor = ArgumentCaptor.forClass(List.class);
verify(ziniaoMemoryStoreService).updateStaleMarks(captor.capture());
assertEquals(1, captor.getValue().size());
assertEquals("n:shop-2", captor.getValue().get(0).getCacheKey());
}
private ZiniaoShopIndexRefreshCursorDto capturedCursor() {
ArgumentCaptor<Object> captor = ArgumentCaptor.forClass(Object.class);
verify(ziniaoTransientCacheService, times(2)).put(