super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

新需求更新 同步更新

Browse Source
This commit is contained in:
supernijia
2026-08-06 01:11:54 +08:00
parent 9048bbb7f8
commit 28e7fce11c
112 changed files with 7739 additions and 637 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-java/src/main/java/com/nanri/aiimage/modules/ziniao/controller/ZiniaoAuthController.java
+20
View File
@@ -1,6 +1,7 @@
package com.nanri.aiimage.modules.ziniao.controller;
import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.modules.admin.support.AdminAuthSupport;
import com.nanri.aiimage.modules.ziniao.model.cache.ZiniaoShopIndexRefreshCursorDto;
import com.nanri.aiimage.modules.ziniao.model.dto.ZiniaoOpenShopRequest;
import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoOpenShopVo;
@@ -8,17 +9,23 @@ import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoSessionVo;
import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoShopListVo;
import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoStaffListVo;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoAuthService;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopIndexRefreshService;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopIndexService;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.media.Content;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.responses.ApiResponses;
import io.swagger.v3.oas.annotations.security.SecurityRequirement;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor;
import org.springframework.http.HttpHeaders;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestHeader;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
@@ -31,6 +38,8 @@ public class ZiniaoAuthController {
private final ZiniaoAuthService ziniaoAuthService;
private final ZiniaoShopIndexService ziniaoShopIndexService;
private final ZiniaoShopIndexRefreshService ziniaoShopIndexRefreshService;
private final AdminAuthSupport adminAuthSupport;
@GetMapping("/session")
@Operation(summary = "获取紫鸟会话状态", description = "自动获取或复用 appToken，并返回 companyId、当前员工 userId、脱敏 token 和当前店铺信息。")
@@ -56,6 +65,17 @@ public class ZiniaoAuthController {
return ApiResponse.success(ziniaoShopIndexService.getRefreshCursor());
}
@PostMapping("/index-refresh")
@Operation(summary = "手动刷新紫鸟店铺索引", description = "立即执行一轮完整的紫鸟店铺索引刷新，并返回本轮刷新结果。")
@SecurityRequirement(name = "bearerAuth")
public ApiResponse<ZiniaoShopIndexRefreshCursorDto> refreshShopIndex(
HttpServletRequest request,
@Parameter(description = "管理员登录凭证，格式：Bearer <token>")
@RequestHeader(value = HttpHeaders.AUTHORIZATION, required = false) String authorization) {
adminAuthSupport.requireAdmin(request);
return ApiResponse.success("紫鸟店铺索引刷新完成", ziniaoShopIndexRefreshService.refreshShopIndexManually());
}
@GetMapping("/shops")
@Operation(summary = "获取员工可见店铺列表", description = "先通过 API Key 获取 companyId，再按员工 userId 查询该员工有权限的店铺列表，并缓存到当前 session。")
@ApiResponses({
backend-java/src/main/java/com/nanri/aiimage/modules/ziniao/service/ZiniaoApiKeyProvider.java
+59 -13
View File
@@ -1,12 +1,15 @@
package com.nanri.aiimage.modules.ziniao.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.shopkey.mapper.ShopKeyMapper;
import com.nanri.aiimage.modules.shopkey.model.entity.ShopKeyEntity;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -15,29 +18,28 @@ import java.util.Map;
@RequiredArgsConstructor
public class ZiniaoApiKeyProvider {
public static final String IP_WHITELIST_STATUS_ALLOWED = "ALLOWED";
public static final String IP_WHITELIST_STATUS_BLOCKED = "BLOCKED";
private final ShopKeyMapper shopKeyMapper;
public List<ApiKeyAccount> listApiKeyAccounts() {
List<ShopKeyEntity> entities = shopKeyMapper.selectList(new LambdaQueryWrapper<ShopKeyEntity>()
.orderByDesc(ShopKeyEntity::getId));
Map<String, String> accountByApiKey = new LinkedHashMap<>();
Map<String, List<ShopKeyEntity>> entitiesByApiKey = new LinkedHashMap<>();
for (ShopKeyEntity entity : entities) {
String apiKey = normalizeApiKey(entity == null ? null : entity.getZiniaoToken());
if (apiKey == null) {
continue;
}
String accountName = entity.getZiniaoAccountName() == null ? null : entity.getZiniaoAccountName().trim();
if (!accountByApiKey.containsKey(apiKey)) {
accountByApiKey.put(apiKey, accountName);
continue;
}
String existingName = accountByApiKey.get(apiKey);
if ((existingName == null || existingName.isBlank()) && accountName != null && !accountName.isBlank()) {
accountByApiKey.put(apiKey, accountName);
}
entitiesByApiKey.computeIfAbsent(apiKey, ignored -> new ArrayList<>()).add(entity);
}
return accountByApiKey.entrySet().stream()
.map(entry -> new ApiKeyAccount(entry.getKey(), entry.getValue()))
return entitiesByApiKey.entrySet().stream()
.map(entry -> new ApiKeyAccount(
entry.getKey(),
resolveAccountName(entry.getValue()),
entry.getValue().stream().map(ShopKeyEntity::getId).filter(java.util.Objects::nonNull).toList()
))
.toList();
}
@@ -60,6 +62,42 @@ public class ZiniaoApiKeyProvider {
return total != null && total > 0;
}
public void markIpWhitelistAllowed(ApiKeyAccount account) {
updateIpWhitelistStatus(account, IP_WHITELIST_STATUS_ALLOWED, null);
}
public void markIpWhitelistBlocked(ApiKeyAccount account, String message) {
updateIpWhitelistStatus(account, IP_WHITELIST_STATUS_BLOCKED, message);
}
private void updateIpWhitelistStatus(ApiKeyAccount account, String status, String message) {
if (account == null || account.shopKeyIds().isEmpty()) {
return;
}
shopKeyMapper.update(null, new LambdaUpdateWrapper<ShopKeyEntity>()
.in(ShopKeyEntity::getId, account.shopKeyIds())
.set(ShopKeyEntity::getIpWhitelistStatus, status)
.set(ShopKeyEntity::getIpWhitelistCheckedAt, LocalDateTime.now())
.set(ShopKeyEntity::getIpWhitelistMessage, truncateMessage(message)));
}
private String resolveAccountName(List<ShopKeyEntity> entities) {
return entities.stream()
.map(ShopKeyEntity::getZiniaoAccountName)
.filter(name -> name != null && !name.isBlank())
.map(String::trim)
.findFirst()
.orElse(null);
}
private String truncateMessage(String message) {
if (message == null || message.isBlank()) {
return null;
}
String normalized = message.trim();
return normalized.length() <= 500 ? normalized : normalized.substring(0, 500);
}
private String normalizeApiKey(String token) {
if (token == null || token.isBlank()) {
return null;
@@ -71,6 +109,14 @@ public class ZiniaoApiKeyProvider {
return normalized.isBlank() ? null : normalized;
}
public record ApiKeyAccount(String apiKey, String accountName) {
public record ApiKeyAccount(String apiKey, String accountName, List<Long> shopKeyIds) {
public ApiKeyAccount {
shopKeyIds = shopKeyIds == null ? List.of() : List.copyOf(shopKeyIds);
}
public ApiKeyAccount(String apiKey, String accountName) {
this(apiKey, accountName, List.of());
}
}
}
backend-java/src/main/java/com/nanri/aiimage/modules/ziniao/service/ZiniaoShopIndexRefreshService.java
+13
View File
@@ -1,6 +1,8 @@
package com.nanri.aiimage.modules.ziniao.service;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.common.service.DistributedJobLockService;
import com.nanri.aiimage.modules.ziniao.model.cache.ZiniaoShopIndexRefreshCursorDto;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.scheduling.annotation.Scheduled;
@@ -33,4 +35,15 @@ public class ZiniaoShopIndexRefreshService {
}
}
}
public ZiniaoShopIndexRefreshCursorDto refreshShopIndexManually() {
DistributedJobLockService.LockHandle lockHandle = distributedJobLockService.tryLock("ziniao:shop-index-refresh", REFRESH_LOCK_TTL);
if (lockHandle == null) {
throw new BusinessException("紫鸟店铺索引正在刷新，或分布式锁暂时不可用，请稍后重试");
}
try (lockHandle) {
ziniaoShopIndexService.refreshAllShopIndex();
return ziniaoShopIndexService.getRefreshCursor();
}
}
}
backend-java/src/main/java/com/nanri/aiimage/modules/ziniao/service/ZiniaoShopIndexService.java
+93 -20
View File
@@ -159,6 +159,14 @@ public class ZiniaoShopIndexService {
}
public void refreshShopIndex() {
refreshShopIndex(resolveRefreshBatchSize());
}
public void refreshAllShopIndex() {
refreshShopIndex(0);
}
private void refreshShopIndex(int refreshBatchSize) {
long now = Instant.now().toEpochMilli();
ZiniaoShopIndexRefreshCursorDto previousCursor = ziniaoTransientCacheService
.get(CACHE_TYPE_SHOP_INDEX_REFRESH_CURSOR, "global", ZiniaoShopIndexRefreshCursorDto.class)
@@ -177,7 +185,10 @@ public class ZiniaoShopIndexService {
Map<String, List<ZiniaoShopIndexEntryDto>> grouped = new LinkedHashMap<>();
Map<String, Long> storesFingerprintToUserId = new LinkedHashMap<>();
Set<Long> allInvalidUserIds = new LinkedHashSet<>();
int refreshBatchSize = resolveRefreshBatchSize();
int completedApiKeyCount = 0;
int skippedApiKeyCount = 0;
int whitelistSkippedApiKeyCount = 0;
boolean completeCoverage = true;
try {
List<ZiniaoApiKeyProvider.ApiKeyAccount> allApiKeyAccounts = ziniaoApiKeyProvider.listApiKeyAccounts();
List<ZiniaoApiKeyProvider.ApiKeyAccount> apiKeyAccounts = selectApiKeyBatchByOffset(
@@ -186,14 +197,24 @@ public class ZiniaoShopIndexService {
refreshBatchSize
);
int nextOffset = computeNextOffset(allApiKeyAccounts.size(), previousOffset, apiKeyAccounts.size(), refreshBatchSize);
apiKeyLoop:
for (ZiniaoApiKeyProvider.ApiKeyAccount apiKeyAccount : apiKeyAccounts) {
String apiKey = apiKeyAccount.apiKey();
String companyName = apiKeyAccount.accountName();
Map<String, List<ZiniaoShopIndexEntryDto>> apiKeyGrouped = new LinkedHashMap<>();
Map<String, Long> apiKeyFingerprints = new LinkedHashMap<>();
Long companyId;
try {
companyId = ziniaoAuthService.resolveCompanyIdForIndex(apiKey);
} catch (BusinessException ex) {
rethrowIfIpWhitelistError(ex);
skippedApiKeyCount++;
completeCoverage = false;
if (ziniaoAuthService.isIpWhitelistError(ex)) {
whitelistSkippedApiKeyCount++;
markIpWhitelistBlockedSafely(apiKeyAccount, ex.getMessage());
log.warn("[ziniao-index] skip apiKey reason=IP_WHITELIST accountName={} msg={}", companyName, ex.getMessage());
continue;
}
log.warn("[ziniao-index] skip apiKey while resolving companyId, msg={}", ex.getMessage());
continue;
}
@@ -203,7 +224,15 @@ public class ZiniaoShopIndexService {
try {
staff = ziniaoAuthService.getOrLoadStaffForIndex(apiKey, companyId);
} catch (BusinessException ex) {
rethrowIfIpWhitelistError(ex);
if (ziniaoAuthService.isIpWhitelistError(ex)) {
skippedApiKeyCount++;
whitelistSkippedApiKeyCount++;
completeCoverage = false;
markIpWhitelistBlockedSafely(apiKeyAccount, ex.getMessage());
log.warn("[ziniao-index] skip apiKey reason=IP_WHITELIST accountName={} companyId={} msg={}",
companyName, companyId, ex.getMessage());
continue;
}
throw ex;
}
for (Long userId : buildUserIds(staff)) {
@@ -214,11 +243,21 @@ public class ZiniaoShopIndexService {
try {
stores = ziniaoAuthService.getOrLoadUserStoresForIndex(apiKey, companyId, userId);
} catch (BusinessException ex) {
rethrowIfIpWhitelistError(ex);
if (ziniaoAuthService.isIpWhitelistError(ex)) {
skippedApiKeyCount++;
whitelistSkippedApiKeyCount++;
completeCoverage = false;
markIpWhitelistBlockedSafely(apiKeyAccount, ex.getMessage());
log.warn("[ziniao-index] skip apiKey reason=IP_WHITELIST accountName={} companyId={} userId={} msg={}",
companyName, companyId, userId, ex.getMessage());
continue apiKeyLoop;
}
if (ziniaoAuthService.isInvalidUserStoresError(ex)) {
invalidUserIds.add(userId);
allInvalidUserIds.add(userId);
ziniaoAuthService.evictInvalidUserForIndex(apiKey, companyId, userId);
} else {
completeCoverage = false;
}
log.warn("[ziniao-index] skip user stores, companyId={}, userId={}, msg={}", companyId, userId, ex.getMessage());
continue;
@@ -231,7 +270,10 @@ public class ZiniaoShopIndexService {
continue;
}
String storesFingerprint = buildStoresFingerprint(stores);
Long duplicatedUserId = storesFingerprintToUserId.putIfAbsent(storesFingerprint, userId);
Long duplicatedUserId = storesFingerprintToUserId.get(storesFingerprint);
if (duplicatedUserId == null) {
duplicatedUserId = apiKeyFingerprints.putIfAbsent(storesFingerprint, userId);
}
if (duplicatedUserId != null) {
ziniaoTransientCacheService.delete(
CACHE_TYPE_SHOP_INDEX_SCOPE_SNAPSHOT,
@@ -264,7 +306,7 @@ public class ZiniaoShopIndexService {
entry.setStatus(STATUS_ACTIVE);
entry.setLastSeenAt(now);
entry.setLastRefreshedAt(now);
List<ZiniaoShopIndexEntryDto> candidates = grouped.computeIfAbsent(normalizedShopName, ignored -> new ArrayList<>());
List<ZiniaoShopIndexEntryDto> candidates = apiKeyGrouped.computeIfAbsent(normalizedShopName, ignored -> new ArrayList<>());
if (candidates.stream().noneMatch(existing -> sameCandidate(existing, entry))) {
candidates.add(entry);
}
@@ -278,6 +320,17 @@ public class ZiniaoShopIndexService {
);
}
}
for (Map.Entry<String, List<ZiniaoShopIndexEntryDto>> apiKeyEntry : apiKeyGrouped.entrySet()) {
List<ZiniaoShopIndexEntryDto> candidates = grouped.computeIfAbsent(apiKeyEntry.getKey(), ignored -> new ArrayList<>());
for (ZiniaoShopIndexEntryDto candidate : apiKeyEntry.getValue()) {
if (candidates.stream().noneMatch(existing -> sameCandidate(existing, candidate))) {
candidates.add(candidate);
}
}
}
storesFingerprintToUserId.putAll(apiKeyFingerprints);
markIpWhitelistAllowedSafely(apiKeyAccount);
completedApiKeyCount++;
}
List<ZiniaoShopIndexEntryDto> roundEntries = new ArrayList<>();
@@ -314,28 +367,35 @@ public class ZiniaoShopIndexService {
}
log.info("[ziniao-index] refresh persist shopIndex uniqueShopId={} nameAlias={} groupedNames={} activeRows={}",
byShopId.size(), byNameAlias.size(), grouped.size(), activeCacheKeys.size());
boolean fullRefresh = apiKeyAccounts.size() >= allApiKeyAccounts.size();
boolean fullRefresh = completeCoverage && apiKeyAccounts.size() >= allApiKeyAccounts.size();
if (fullRefresh) {
markMissingEntriesAsStale(activeCacheKeys, now);
} else {
log.info("[ziniao-index] skip stale marking for partial refresh processedApiKeys={}/{} nextOffset={}",
apiKeyAccounts.size(), allApiKeyAccounts.size(), nextOffset);
log.info("[ziniao-index] skip stale marking for partial refresh completedApiKeys={}/{} skippedApiKeys={} nextOffset={}",
completedApiKeyCount, allApiKeyAccounts.size(), skippedApiKeyCount, nextOffset);
}
cursor.setStatus("SUCCESS");
cursor.setMessage(allInvalidUserIds.isEmpty()
? null
: "本轮刷新已跳过无效 userId 数: " + allInvalidUserIds.size());
List<String> refreshMessages = new ArrayList<>();
if (!allInvalidUserIds.isEmpty()) {
refreshMessages.add("本轮刷新已跳过无效 userId 数: " + allInvalidUserIds.size());
}
if (skippedApiKeyCount > 0) {
refreshMessages.add("本轮刷新已跳过 apiKey 数: " + skippedApiKeyCount
+ "（IP 白名单: " + whitelistSkippedApiKeyCount + "）");
}
cursor.setMessage(refreshMessages.isEmpty() ? null : String.join("；", refreshMessages));
cursor.setApiKeyTotal(allApiKeyAccounts.size());
cursor.setLastProcessedApiKeyCount(apiKeyAccounts.size());
cursor.setLastProcessedApiKeyCount(completedApiKeyCount);
cursor.setNextApiKeyOffset(nextOffset);
cursor.setInvalidUserCount(allInvalidUserIds.size());
cursor.setSampleInvalidUserIds(allInvalidUserIds.stream().limit(20).toList());
cursor.setLastFinishedAt(now);
cursor.setLastSuccessAt(now);
ziniaoTransientCacheService.put(CACHE_TYPE_SHOP_INDEX_REFRESH_CURSOR, "global", cursor, DEFAULT_CURSOR_TTL);
log.info("[ziniao-index] refresh success invalidUsersSkipped={} apiKeyProcessed={}/{} nextOffset={}",
allInvalidUserIds.size(), apiKeyAccounts.size(), allApiKeyAccounts.size(), nextOffset);
log.info("[ziniao-index] refresh success invalidUsersSkipped={} apiKeyCompleted={}/{} apiKeySkipped={} whitelistSkipped={} nextOffset={}",
allInvalidUserIds.size(), completedApiKeyCount, allApiKeyAccounts.size(), skippedApiKeyCount,
whitelistSkippedApiKeyCount, nextOffset);
} catch (Exception ex) {
cursor.setStatus("FAILED");
cursor.setMessage(ex.getMessage());
@@ -358,6 +418,24 @@ public class ZiniaoShopIndexService {
log.info("[ziniao-index] cursor invalidated (transient only; shop rows unchanged)");
}
private void markIpWhitelistAllowedSafely(ZiniaoApiKeyProvider.ApiKeyAccount account) {
try {
ziniaoApiKeyProvider.markIpWhitelistAllowed(account);
} catch (Exception ex) {
log.warn("[ziniao-index] failed to record IP whitelist status accountName={} status=ALLOWED msg={}",
account.accountName(), ex.getMessage());
}
}
private void markIpWhitelistBlockedSafely(ZiniaoApiKeyProvider.ApiKeyAccount account, String message) {
try {
ziniaoApiKeyProvider.markIpWhitelistBlocked(account, message);
} catch (Exception ex) {
log.warn("[ziniao-index] failed to record IP whitelist status accountName={} status=BLOCKED msg={}",
account.accountName(), ex.getMessage());
}
}
public String normalizeShopName(String value) {
if (value == null) {
return "";
@@ -723,9 +801,4 @@ public class ZiniaoShopIndexService {
}
}
private void rethrowIfIpWhitelistError(BusinessException ex) {
if (ziniaoAuthService.isIpWhitelistError(ex)) {
throw new BusinessException("紫鸟刷新店铺索引失败：当前服务器 IP 未加入紫鸟白名单，已停止本轮刷新且不会更新索引数据");
}
}
}