super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修复图片下载问题

Browse Source
This commit is contained in:
super
2026-06-16 22:19:55 +08:00
parent f44110fae5
commit 021b0c618b
11 changed files with 358 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-java/src/main/java/com/nanri/aiimage/modules/permission/service/PermissionMenuSchemaInitializer.java
+2 -1
View File
@@ -36,7 +36,8 @@ public class PermissionMenuSchemaInitializer {
new DefaultAdminMenu("查询ASIN", "admin_query_asin", "query-asin", 65),
new DefaultAdminMenu("商品类目", "admin_product_categories", "product-categories", 66),
new DefaultAdminMenu("查看生成记录", "admin_history", "history", 70),
new DefaultAdminMenu("版本管理", "admin_version", "version", 80)
new DefaultAdminMenu("软件版本管理", "admin_version", "version", 80),
new DefaultAdminMenu("数字人版本管理", "digital_human_version", "digital-human-version", 81)
);
@EventListener(ApplicationReadyEvent.class)
backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskService.java
+6 -2
View File
@@ -78,7 +78,7 @@ public class PriceTrackTaskService {
private FileTaskEntity loadTaskForExecution(Long taskId) {
Map<Long, FileTaskEntity> cachedTasks = priceTrackTaskCacheService.getTaskCacheBatch(List.of(taskId));
FileTaskEntity cached = cachedTasks.get(taskId);
if (cached != null) {
if (cached != null && hasText(cached.getRequestJson())) {
return cached;
}
FileTaskEntity dbTask = fileTaskMapper.selectById(taskId);
@@ -396,7 +396,6 @@ public class PriceTrackTaskService {
task.setCreatedAt(LocalDateTime.now());
task.setUpdatedAt(LocalDateTime.now());
fileTaskMapper.insert(task);
priceTrackTaskCacheService.saveTaskCache(task);
priceTrackTaskCacheService.touchTaskHeartbeat(task.getId());
if (request.getLoopRunId() != null) {
priceTrackLoopRunService.bindChildTask(request.getLoopRunId(), task.getId(), request.getRoundIndex(), request.getShopIndex());
@@ -447,6 +446,7 @@ public class PriceTrackTaskService {
throw new BusinessException("序列化任务数据失败");
}
fileTaskMapper.updateById(task);
priceTrackTaskCacheService.saveTaskCache(task);
// TODO: Python 侧轮询 RUNNING 任务，pickup 后开始处理
// Java 侧通过 submitResult 接口接收 Python 回传结果
@@ -1548,6 +1548,10 @@ public class PriceTrackTaskService {
return taskDistributedLockService.acquire(MODULE_TYPE, taskId);
}
private static boolean hasText(String value) {
return value != null && !value.isBlank();
}
private void cleanupTaskCacheIfTerminal(Long taskId, String taskStatus) {
if (!"SUCCESS".equals(taskStatus) && !"FAILED".equals(taskStatus) && !"DELETE_EMPTY".equals(taskStatus)) {
return;
backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskExcelAssemblyService.java
+27 -1
View File
@@ -51,7 +51,7 @@ public class ProductRiskExcelAssemblyService {
createTextCell(row, 0, firstNonBlank(dto == null ? null : dto.getShopName(), shopDisplayName));
createTextCell(row, 1, dto == null ? null : dto.getProductAsinSku());
createTextCell(row, 2, dto == null ? null : dto.getStatus());
createTextCell(row, 3, hasBusinessData(dto) ? "是" : "否");
createTextCell(row, 3, completionDisplayValue(dto));
createTextCell(row, 4, dto == null ? null : dto.getRemoveAsin());
createTextCell(row, 5, dto == null ? null : dto.getRemoveStatus());
}
@@ -119,6 +119,32 @@ public class ProductRiskExcelAssemblyService {
|| hasText(dto.getRemoveStatus()));
}
private static String completionDisplayValue(ProductRiskRowDto dto) {
if (dto == null || !hasBusinessData(dto)) {
return "";
}
return isCompleted(dto) ? "是" : "否";
}
private static boolean isCompleted(ProductRiskRowDto dto) {
return Boolean.TRUE.equals(dto.getDone())
|| indicatesCompleted(dto.getStatus())
|| indicatesCompleted(dto.getRemoveStatus());
}
private static boolean indicatesCompleted(String value) {
if (!hasText(value)) {
return false;
}
String normalized = value.trim().toUpperCase(java.util.Locale.ROOT);
return normalized.contains("完成")
|| normalized.contains("成功")
|| normalized.contains("已处理")
|| normalized.contains("DONE")
|| normalized.contains("SUCCESS")
|| normalized.contains("COMPLETED");
}
private static boolean hasText(String value) {
return value != null && !value.isBlank();
}
backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/util/SimilarAsinImageEmbedder.java
+58 -5
View File
@@ -35,6 +35,7 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@@ -345,13 +346,20 @@ public class SimilarAsinImageEmbedder {
private byte[] downloadWithRetry(String url) throws IOException, TimeoutException {
validateHttpsUrl(url);
List<String> candidates = downloadCandidates(url);
IOException last = null;
TimeoutException lastTimeout = null;
long waitSeconds = downloadTimeoutSeconds * 2L;
for (int attempt = 0; attempt <= DOWNLOAD_MAX_RETRY; attempt++) {
Future<byte[]> future = downloadPool.submit(() -> doFetch(url));
String attemptUrl = candidates.get(Math.min(attempt, candidates.size() - 1));
Future<byte[]> future = downloadPool.submit(() -> doFetch(attemptUrl));
try {
return future.get(waitSeconds, TimeUnit.SECONDS);
byte[] bytes = future.get(waitSeconds, TimeUnit.SECONDS);
if (!attemptUrl.equals(url)) {
log.info("[similar-asin][image] download-fallback-success originalUrl={} usedUrl={} attempt={}/{}",
url, attemptUrl, attempt + 1, DOWNLOAD_MAX_RETRY + 1);
}
return bytes;
} catch (java.util.concurrent.ExecutionException ee) {
Throwable cause = ee.getCause();
if (cause instanceof DownloadOversizeException doe) {
@@ -360,7 +368,7 @@ public class SimilarAsinImageEmbedder {
if (cause instanceof IOException io) {
last = io;
lastTimeout = null;
logRetry(url, attempt, io);
logRetry(attemptUrl, attempt, io);
continue;
}
if (cause instanceof RuntimeException re) {
@@ -371,11 +379,12 @@ public class SimilarAsinImageEmbedder {
future.cancel(true);
TimeoutException wrapped = new TimeoutException("image download timeout after attempt "
+ (attempt + 1) + "/" + (DOWNLOAD_MAX_RETRY + 1)
+ ", waitSeconds=" + waitSeconds);
+ ", waitSeconds=" + waitSeconds
+ ", url=" + attemptUrl);
wrapped.initCause(te);
lastTimeout = wrapped;
last = null;
logRetry(url, attempt, wrapped);
logRetry(attemptUrl, attempt, wrapped);
} catch (InterruptedException ie) {
Thread.currentThread().interrupt();
throw new IOException("image download interrupted", ie);
@@ -387,6 +396,50 @@ public class SimilarAsinImageEmbedder {
throw last != null ? last : new IOException("image download failed without cause");
}
static List<String> downloadCandidates(String url) {
List<String> single = List.of(url);
URI uri;
try {
uri = new URI(url);
} catch (URISyntaxException ex) {
return single;
}
String host = uri.getHost();
String rawPath = uri.getRawPath();
if (host == null || rawPath == null
|| !host.toLowerCase(Locale.ROOT).endsWith("media-amazon.com")
|| !rawPath.startsWith("/images/I/")) {
return single;
}
int slash = rawPath.lastIndexOf('/');
String dir = rawPath.substring(0, slash + 1);
String file = rawPath.substring(slash + 1);
int markerStart = file.lastIndexOf("._");
int markerEnd = markerStart < 0 ? -1 : file.indexOf("_.", markerStart + 2);
if (markerStart < 0 || markerEnd < 0) {
return single;
}
String prefix = file.substring(0, markerStart);
String ext = file.substring(markerEnd + 1);
LinkedHashSet<String> candidates = new LinkedHashSet<>();
candidates.add(url);
candidates.add(buildUrl(uri, dir + prefix + "._AC_SY450_" + ext));
candidates.add(buildUrl(uri, dir + prefix + "._AC_SX425_" + ext));
return new ArrayList<>(candidates);
}
private static String buildUrl(URI uri, String rawPath) {
StringBuilder sb = new StringBuilder();
sb.append(uri.getScheme()).append("://").append(uri.getRawAuthority()).append(rawPath);
if (uri.getRawQuery() != null) {
sb.append('?').append(uri.getRawQuery());
}
if (uri.getRawFragment() != null) {
sb.append('#').append(uri.getRawFragment());
}
return sb.toString();
}
private void logRetry(String url, int attempt, Exception ex) {
if (attempt < DOWNLOAD_MAX_RETRY) {
log.debug("[similar-asin][image] download-retry url={} nextAttempt={}/{} err={}",