super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

暂存

Browse Source
This commit is contained in:
super
2026-03-30 20:39:13 +08:00
parent cb3b9e9cfe
commit 53df8b9971
16 changed files with 328 additions and 138 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
backend-java/aiimage-backend.log.2026-03-29.0.gz Normal file
View File

Binary file not shown.

61
backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandRunService.java
View File

@@ -40,6 +40,8 @@ import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.io.File;
import java.io.FileInputStream;
@@ -125,6 +127,8 @@ public class DeleteBrandRunService {
item.setShopId(matchResult.shopId());
item.setPlatform(matchResult.platform());
item.setOpenStoreUrl(matchResult.openStoreUrl());
} else {
throw new BusinessException("未匹配到紫鸟店铺，跳过处理");
}
} catch (BusinessException ex) {
String message = ex.getMessage();
@@ -191,9 +195,10 @@ public class DeleteBrandRunService {
task.setFailedFileCount(parsedFailedCount);
task.setResultJson(JSONUtil.toJsonStr(items));
task.setUpdatedAt(LocalDateTime.now());
task.setSourceFileCount(parsedSuccessCount);
if (parsedSuccessCount <= 0) {
task.setStatus("FAILED");
task.setErrorMessage(parsedFailedCount > 0 ? "删除品牌文件解析失败" : null);
task.setErrorMessage(parsedFailedCount > 0 ? "全部文件解析或匹配失败" : null);
task.setFinishedAt(LocalDateTime.now());
}
fileTaskMapper.updateById(task);
@@ -269,11 +274,20 @@ public class DeleteBrandRunService {
}
if (entity.getSourceFilename().equals(candidate.getSourceFilename())) {
item.setMatched(candidate.isMatched());
// 如果物理结果已经是成功了，说明已经处理过了，强制置为匹配成功
if (item.isSuccess()) {
item.setMatched(true);
}
item.setShopId(candidate.getShopId());
item.setPlatform(candidate.getPlatform());
item.setOpenStoreUrl(candidate.getOpenStoreUrl());
// 历史表可能只有通用 errorMessage；优先返回任务当时的真实 error
if ((item.getError() == null || item.getError().isBlank()) && candidate.getError() != null && !candidate.getError().isBlank()) {
// 但是，如果物理结果是成功的，就不应该显示所谓的“未匹配”报错
if (item.isSuccess()) {
item.setError(null);
} else if ((item.getError() == null || item.getError().isBlank()) && candidate.getError() != null && !candidate.getError().isBlank()) {
item.setError(candidate.getError());
}
break;
@@ -636,7 +650,10 @@ public class DeleteBrandRunService {
}
}
@Transactional
public void submitResult(Long taskId, DeleteBrandSubmitResultRequest request) {
log.info("[DeleteBrand] Received submitResult for taskId: {}, files count: {}", taskId, request == null || request.getFiles() == null ? 0 : request.getFiles().size());
if (taskId == null || taskId <= 0) {
throw new BusinessException("taskId 不合法");
}
@@ -659,24 +676,33 @@ public class DeleteBrandRunService {
throw new BusinessException("任务原始数据已过期，请重新执行解析");
}
task.setStatus("RUNNING");
task.setErrorMessage(null);
task.setUpdatedAt(LocalDateTime.now());
task.setFinishedAt(null);
fileTaskMapper.updateById(task);
if (!"RUNNING".equals(task.getStatus())) {
task.setStatus("RUNNING");
task.setErrorMessage(null);
task.setUpdatedAt(LocalDateTime.now());
task.setFinishedAt(null);
fileTaskMapper.updateById(task);
}
for (DeleteBrandResultFileDto fileDto : request.getFiles()) {
log.info("[DeleteBrand] submitResult processing chunk -> taskId: {}, file: {}, chunkIndex: {}/{}, processedRows: {}/{}",
taskId, fileDto.getSourceFilename(), fileDto.getChunkIndex(), fileDto.getChunkTotal(),
fileDto.getProcessedRows(), fileDto.getTotalRows());
String fileIdentity = resolveFileIdentity(fileDto);
if (fileIdentity.isBlank()) {
throw new BusinessException("fileKey/sourceFilename 不能为空");
}
DeleteBrandParsedFileCacheDto parsedFile = parsedPayload.get(fileIdentity);
if (parsedFile == null) {
log.warn("[DeleteBrand] parsed payload not found for identity: {}", fileIdentity);
throw new BusinessException("文件标识不匹配: " + fileIdentity);
}
// 更早的 duplicate fast-path：尽量在较重的校验/进度写入之前直接短路
if (deleteBrandTaskCacheService.hasSameResultChunk(taskId, fileIdentity, fileDto.getChunkIndex(), fileDto)) {
log.info("[DeleteBrand] Duplicate chunk ignored for taskId: {}, file: {}, chunkIndex: {}", taskId, fileIdentity, fileDto.getChunkIndex());
continue;
}
@@ -699,6 +725,16 @@ public class DeleteBrandRunService {
progress.put("updated_at", String.valueOf(System.currentTimeMillis()));
progress.put("last_heartbeat_at", String.valueOf(System.currentTimeMillis()));
deleteBrandTaskCacheService.saveProgress(taskId, progress);
// 如果该文件已完成，立即更新成品计数的 Redis 缓存「提示」，让后续 tryFinalizeTask 能更精确
if (isFileCompleted(fileDto)) {
log.info("[DeleteBrand] File chunk transmission fully reached via chunkTotal for taskId: {}, file: {}", taskId, fileDto.getSourceFilename());
java.util.Map<Object, Object> currentProgress = deleteBrandTaskCacheService.getProgress(taskId);
int finishedCount = parseInt(currentProgress.get("finished_files")) == null ? 0 : parseInt(currentProgress.get("finished_files"));
// 这里我们暂不直接 increment，而是标记需要重算或在 tryFinalizeTask 中重新计算
// 为了保险，我们直接在 tryFinalizeTask 里重新遍历分片状态
}
}
tryFinalizeTask(taskId, false);
@@ -724,12 +760,10 @@ public class DeleteBrandRunService {
return;
}
Map<Object, Object> progressSnapshot = deleteBrandTaskCacheService.getProgress(taskId);
int expectedFiles = parsedPayload.size();
Integer finishedFilesHint = parseInt(progressSnapshot == null ? null : progressSnapshot.get("finished_files"));
if (finishedFilesHint != null && finishedFilesHint < expectedFiles) {
return;
}
// 移除 finishedFilesHint 强行返回逻辑，因为 Redis 里的 finished_files 可能是旧的，
// 既然进入了 tryFinalizeTask，就说明有新片到达，应该以 loadMergedChunks 为准
Map<String, List<DeleteBrandResultFileDto>> mergedByFile = loadMergedChunks(taskId);
int finishedFiles = countCompletedFiles(mergedByFile);
@@ -747,8 +781,11 @@ public class DeleteBrandRunService {
fileTaskMapper.updateById(task);
if (finishedFiles < expectedFiles) {
log.info("[DeleteBrand] tryFinalizeTask skipping -> finishedFiles ({}) < expectedFiles ({})", finishedFiles, expectedFiles);
return;
}
log.info("[DeleteBrand] tryFinalizeTask calling finalizeTask -> finishedFiles: {}, expectedFiles: {}", finishedFiles, expectedFiles);
finalizeTask(task, parsedPayload, mergedByFile);
}

21
backend-java/src/main/java/com/nanri/aiimage/modules/deletebrand/service/DeleteBrandStaleTaskService.java
View File

@@ -39,6 +39,16 @@ public class DeleteBrandStaleTaskService {
Map<Object, Object> progress = deleteBrandTaskCacheService.getProgress(task.getId());
boolean hasProgress = progress != null && !progress.isEmpty() && progress.containsKey("last_heartbeat_at");
boolean isActive = false;
if (progress != null && progress.containsKey("has_progress")) {
Object hpObj = progress.get("has_progress");
if (hpObj instanceof Boolean) {
isActive = (Boolean) hpObj;
} else if (hpObj instanceof String) {
isActive = Boolean.parseBoolean((String) hpObj);
}
}
long lastHeartbeatAt = 0L;
if (hasProgress) {
try {
@@ -48,18 +58,19 @@ public class DeleteBrandStaleTaskService {
}
LocalDateTime lastActivityTime;
if (lastHeartbeatAt > 0) {
// 有心跳，用最后心跳时间对比 15 分钟
// 只有当 has_progress 为 true 且超过 15 分钟没心跳，才认为是异常卡死
if (lastHeartbeatAt > 0 && isActive) {
// 有心跳，用最后心跳时间对比 15 分钟 (threshold 是 now - 15)
lastActivityTime = LocalDateTime.ofInstant(Instant.ofEpochMilli(lastHeartbeatAt), ZoneId.systemDefault());
if (lastActivityTime.isAfter(threshold)) {
continue; // 没超时
}
} else {
// 没有心跳，说明任务一直没被 Python 取出来执行（处在排队状态中）
// 排队的宽限期我们设为 12 个小时，如果建了 12 个小时都没执行，再杀死
// 没有心跳，或者当前处在 has_progress: false （等待用户点击下一个任务推送的静默期）
// 静默期和排队期的宽限期一样，设为 12 个小时
LocalDateTime queuedThreshold = LocalDateTime.now().minusHours(12);
if (task.getCreatedAt() != null && task.getCreatedAt().isAfter(queuedThreshold)) {
continue; // 排队中，不要杀它！
continue; // 等待下个队列推送中，不要杀它！
}
}

2
backend-java/src/main/java/com/nanri/aiimage/modules/ziniao/service/ZiniaoAuthService.java
View File

@@ -294,7 +294,7 @@ public class ZiniaoAuthService {
.queryParam("storeId", storeId)
.queryParam("openapiToken", loginToken)
.queryParam("userId", userId)
.queryParam("lanuchUrl", encodedLaunchUrl)
.queryParam("launchUrl", encodedLaunchUrl)
.queryParam("autoopen", Boolean.TRUE.equals(ziniaoProperties.getOpenStoreAutoOpen()))
.queryParam("debuggPort", ziniaoProperties.getOpenStoreDebugPort())
.queryParam("notPromptForDownload", Boolean.TRUE.equals(ziniaoProperties.getOpenStoreNotPromptForDownload()) ? 1 : 0);

2
backend-java/src/main/resources/application-local.example.yml
View File

@@ -29,7 +29,7 @@ AIIMAGE_ZINIAO_APP_TOKEN_PATH=/auth/get_app_token
AIIMAGE_ZINIAO_STAFF_LIST_PATH=/superbrowser/rest/v1/erp/staff/list
AIIMAGE_ZINIAO_USER_STORES_PATH=/superbrowser/rest/v1/erp/user/stores
AIIMAGE_ZINIAO_USER_LOGIN_TOKEN_PATH=/superbrowser/rest/v1/token/user-login
AIIMAGE_ZINIAO_OPEN_STORE_SCHEME=superbrowser://OpenStrore
AIIMAGE_ZINIAO_OPEN_STORE_SCHEME=superbrowser://OpenStore
AIIMAGE_ZINIAO_OPEN_STORE_USER_ID=
AIIMAGE_ZINIAO_OPEN_STORE_LAUNCH_URL=https://www.baidu.com
AIIMAGE_ZINIAO_OPEN_STORE_AUTO_OPEN=true

2
backend-java/src/main/resources/application.yml
View File

@@ -96,7 +96,7 @@ aiimage:
staff-list-path: ${AIIMAGE_ZINIAO_STAFF_LIST_PATH:/superbrowser/rest/v1/erp/staff/list}
user-stores-path: ${AIIMAGE_ZINIAO_USER_STORES_PATH:/superbrowser/rest/v1/erp/user/stores}
user-login-token-path: ${AIIMAGE_ZINIAO_USER_LOGIN_TOKEN_PATH:/superbrowser/rest/v1/token/user-login}
open-store-scheme: ${AIIMAGE_ZINIAO_OPEN_STORE_SCHEME:superbrowser://OpenStrore}
open-store-scheme: ${AIIMAGE_ZINIAO_OPEN_STORE_SCHEME:superbrowser://OpenStore}
open-store-user-id: ${AIIMAGE_ZINIAO_OPEN_STORE_USER_ID:}
open-store-launch-url: ${AIIMAGE_ZINIAO_OPEN_STORE_LAUNCH_URL:https://www.baidu.com}
open-store-auto-open: ${AIIMAGE_ZINIAO_OPEN_STORE_AUTO_OPEN:true}