super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修复图片下载问题

Browse Source
This commit is contained in:
super
2026-06-16 22:19:55 +08:00
parent f44110fae5
commit 021b0c618b
11 changed files with 358 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
backend-java/src/main/java/com/nanri/aiimage/modules/permission/service/PermissionMenuSchemaInitializer.java
View File

@@ -36,7 +36,8 @@ public class PermissionMenuSchemaInitializer {
new DefaultAdminMenu("查询ASIN", "admin_query_asin", "query-asin", 65),
new DefaultAdminMenu("商品类目", "admin_product_categories", "product-categories", 66),
new DefaultAdminMenu("查看生成记录", "admin_history", "history", 70),
new DefaultAdminMenu("版本管理", "admin_version", "version", 80)
new DefaultAdminMenu("软件版本管理", "admin_version", "version", 80),
new DefaultAdminMenu("数字人版本管理", "digital_human_version", "digital-human-version", 81)
);
@EventListener(ApplicationReadyEvent.class)

8
backend-java/src/main/java/com/nanri/aiimage/modules/pricetrack/service/PriceTrackTaskService.java
View File

@@ -78,7 +78,7 @@ public class PriceTrackTaskService {
private FileTaskEntity loadTaskForExecution(Long taskId) {
Map<Long, FileTaskEntity> cachedTasks = priceTrackTaskCacheService.getTaskCacheBatch(List.of(taskId));
FileTaskEntity cached = cachedTasks.get(taskId);
if (cached != null) {
if (cached != null && hasText(cached.getRequestJson())) {
return cached;
}
FileTaskEntity dbTask = fileTaskMapper.selectById(taskId);
@@ -396,7 +396,6 @@ public class PriceTrackTaskService {
task.setCreatedAt(LocalDateTime.now());
task.setUpdatedAt(LocalDateTime.now());
fileTaskMapper.insert(task);
priceTrackTaskCacheService.saveTaskCache(task);
priceTrackTaskCacheService.touchTaskHeartbeat(task.getId());
if (request.getLoopRunId() != null) {
priceTrackLoopRunService.bindChildTask(request.getLoopRunId(), task.getId(), request.getRoundIndex(), request.getShopIndex());
@@ -447,6 +446,7 @@ public class PriceTrackTaskService {
throw new BusinessException("序列化任务数据失败");
}
fileTaskMapper.updateById(task);
priceTrackTaskCacheService.saveTaskCache(task);
// TODO: Python 侧轮询 RUNNING 任务，pickup 后开始处理
// Java 侧通过 submitResult 接口接收 Python 回传结果
@@ -1548,6 +1548,10 @@ public class PriceTrackTaskService {
return taskDistributedLockService.acquire(MODULE_TYPE, taskId);
}
private static boolean hasText(String value) {
return value != null && !value.isBlank();
}
private void cleanupTaskCacheIfTerminal(Long taskId, String taskStatus) {
if (!"SUCCESS".equals(taskStatus) && !"FAILED".equals(taskStatus) && !"DELETE_EMPTY".equals(taskStatus)) {
return;

28
backend-java/src/main/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskExcelAssemblyService.java
View File

@@ -51,7 +51,7 @@ public class ProductRiskExcelAssemblyService {
createTextCell(row, 0, firstNonBlank(dto == null ? null : dto.getShopName(), shopDisplayName));
createTextCell(row, 1, dto == null ? null : dto.getProductAsinSku());
createTextCell(row, 2, dto == null ? null : dto.getStatus());
createTextCell(row, 3, hasBusinessData(dto) ? "是" : "否");
createTextCell(row, 3, completionDisplayValue(dto));
createTextCell(row, 4, dto == null ? null : dto.getRemoveAsin());
createTextCell(row, 5, dto == null ? null : dto.getRemoveStatus());
}
@@ -119,6 +119,32 @@ public class ProductRiskExcelAssemblyService {
|| hasText(dto.getRemoveStatus()));
}
private static String completionDisplayValue(ProductRiskRowDto dto) {
if (dto == null || !hasBusinessData(dto)) {
return "";
}
return isCompleted(dto) ? "是" : "否";
}
private static boolean isCompleted(ProductRiskRowDto dto) {
return Boolean.TRUE.equals(dto.getDone())
|| indicatesCompleted(dto.getStatus())
|| indicatesCompleted(dto.getRemoveStatus());
}
private static boolean indicatesCompleted(String value) {
if (!hasText(value)) {
return false;
}
String normalized = value.trim().toUpperCase(java.util.Locale.ROOT);
return normalized.contains("完成")
|| normalized.contains("成功")
|| normalized.contains("已处理")
|| normalized.contains("DONE")
|| normalized.contains("SUCCESS")
|| normalized.contains("COMPLETED");
}
private static boolean hasText(String value) {
return value != null && !value.isBlank();
}

63
backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/util/SimilarAsinImageEmbedder.java
View File

@@ -35,6 +35,7 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@@ -345,13 +346,20 @@ public class SimilarAsinImageEmbedder {
private byte[] downloadWithRetry(String url) throws IOException, TimeoutException {
validateHttpsUrl(url);
List<String> candidates = downloadCandidates(url);
IOException last = null;
TimeoutException lastTimeout = null;
long waitSeconds = downloadTimeoutSeconds * 2L;
for (int attempt = 0; attempt <= DOWNLOAD_MAX_RETRY; attempt++) {
Future<byte[]> future = downloadPool.submit(() -> doFetch(url));
String attemptUrl = candidates.get(Math.min(attempt, candidates.size() - 1));
Future<byte[]> future = downloadPool.submit(() -> doFetch(attemptUrl));
try {
return future.get(waitSeconds, TimeUnit.SECONDS);
byte[] bytes = future.get(waitSeconds, TimeUnit.SECONDS);
if (!attemptUrl.equals(url)) {
log.info("[similar-asin][image] download-fallback-success originalUrl={} usedUrl={} attempt={}/{}",
url, attemptUrl, attempt + 1, DOWNLOAD_MAX_RETRY + 1);
}
return bytes;
} catch (java.util.concurrent.ExecutionException ee) {
Throwable cause = ee.getCause();
if (cause instanceof DownloadOversizeException doe) {
@@ -360,7 +368,7 @@ public class SimilarAsinImageEmbedder {
if (cause instanceof IOException io) {
last = io;
lastTimeout = null;
logRetry(url, attempt, io);
logRetry(attemptUrl, attempt, io);
continue;
}
if (cause instanceof RuntimeException re) {
@@ -371,11 +379,12 @@ public class SimilarAsinImageEmbedder {
future.cancel(true);
TimeoutException wrapped = new TimeoutException("image download timeout after attempt "
+ (attempt + 1) + "/" + (DOWNLOAD_MAX_RETRY + 1)
+ ", waitSeconds=" + waitSeconds);
+ ", waitSeconds=" + waitSeconds
+ ", url=" + attemptUrl);
wrapped.initCause(te);
lastTimeout = wrapped;
last = null;
logRetry(url, attempt, wrapped);
logRetry(attemptUrl, attempt, wrapped);
} catch (InterruptedException ie) {
Thread.currentThread().interrupt();
throw new IOException("image download interrupted", ie);
@@ -387,6 +396,50 @@ public class SimilarAsinImageEmbedder {
throw last != null ? last : new IOException("image download failed without cause");
}
static List<String> downloadCandidates(String url) {
List<String> single = List.of(url);
URI uri;
try {
uri = new URI(url);
} catch (URISyntaxException ex) {
return single;
}
String host = uri.getHost();
String rawPath = uri.getRawPath();
if (host == null || rawPath == null
|| !host.toLowerCase(Locale.ROOT).endsWith("media-amazon.com")
|| !rawPath.startsWith("/images/I/")) {
return single;
}
int slash = rawPath.lastIndexOf('/');
String dir = rawPath.substring(0, slash + 1);
String file = rawPath.substring(slash + 1);
int markerStart = file.lastIndexOf("._");
int markerEnd = markerStart < 0 ? -1 : file.indexOf("_.", markerStart + 2);
if (markerStart < 0 || markerEnd < 0) {
return single;
}
String prefix = file.substring(0, markerStart);
String ext = file.substring(markerEnd + 1);
LinkedHashSet<String> candidates = new LinkedHashSet<>();
candidates.add(url);
candidates.add(buildUrl(uri, dir + prefix + "._AC_SY450_" + ext));
candidates.add(buildUrl(uri, dir + prefix + "._AC_SX425_" + ext));
return new ArrayList<>(candidates);
}
private static String buildUrl(URI uri, String rawPath) {
StringBuilder sb = new StringBuilder();
sb.append(uri.getScheme()).append("://").append(uri.getRawAuthority()).append(rawPath);
if (uri.getRawQuery() != null) {
sb.append('?').append(uri.getRawQuery());
}
if (uri.getRawFragment() != null) {
sb.append('#').append(uri.getRawFragment());
}
return sb.toString();
}
private void logRetry(String url, int attempt, Exception ex) {
if (attempt < DOWNLOAD_MAX_RETRY) {
log.debug("[similar-asin][image] download-retry url={} nextAttempt={}/{} err={}",

25
backend-java/src/main/resources/db/V60__split_version_admin_menu.sql Normal file
View File

@@ -0,0 +1,25 @@
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`)
SELECT '软件版本管理', 'admin_version', 'admin', 'version', 80
WHERE NOT EXISTS (
SELECT 1 FROM columns WHERE column_key = 'admin_version'
);
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`)
SELECT '数字人版本管理', 'digital_human_version', 'admin', 'digital-human-version', 81
WHERE NOT EXISTS (
SELECT 1 FROM columns WHERE column_key = 'digital_human_version'
);
UPDATE `columns`
SET `name` = '软件版本管理',
`route_path` = 'version',
`menu_type` = 'admin',
`sort_order` = 80
WHERE `column_key` = 'admin_version';
UPDATE `columns`
SET `name` = '数字人版本管理',
`route_path` = 'digital-human-version',
`menu_type` = 'admin',
`sort_order` = 81
WHERE `column_key` = 'digital_human_version';

58
backend-java/src/test/java/com/nanri/aiimage/modules/productrisk/service/ProductRiskExcelAssemblyServiceTest.java Normal file
View File

@@ -0,0 +1,58 @@
package com.nanri.aiimage.modules.productrisk.service;
import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskRowDto;
import com.nanri.aiimage.modules.productrisk.model.enums.ProductRiskCountryCode;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.junit.jupiter.api.Test;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
class ProductRiskExcelAssemblyServiceTest {
private final ProductRiskExcelAssemblyService service = new ProductRiskExcelAssemblyService();
@Test
void completionColumnUsesChineseValueAndCompletedStatusWinsOverFalseDone() throws Exception {
Path xlsx = Files.createTempFile("product-risk-result-", ".xlsx");
try {
ProductRiskRowDto completed = row("SKU-1", "处理完成", Boolean.FALSE);
ProductRiskRowDto running = row("SKU-2", "处理中", Boolean.FALSE);
ProductRiskRowDto done = row("SKU-3", null, Boolean.TRUE);
service.writeWorkbook(
xlsx.toFile(),
"店铺A",
Map.of(ProductRiskCountryCode.DE.getSheetName(), List.of(completed, running, done)));
try (InputStream in = Files.newInputStream(xlsx);
Workbook workbook = new XSSFWorkbook(in)) {
Sheet sheet = workbook.getSheet(ProductRiskCountryCode.DE.getSheetName());
assertEquals("完成", sheet.getRow(0).getCell(3).getStringCellValue());
assertEquals("是", sheet.getRow(1).getCell(3).getStringCellValue());
assertNotEquals("false", sheet.getRow(1).getCell(3).getStringCellValue());
assertEquals("否", sheet.getRow(2).getCell(3).getStringCellValue());
assertEquals("是", sheet.getRow(3).getCell(3).getStringCellValue());
}
} finally {
Files.deleteIfExists(xlsx);
}
}
private static ProductRiskRowDto row(String productAsinSku, String status, Boolean done) {
ProductRiskRowDto row = new ProductRiskRowDto();
row.setShopName("店铺A");
row.setProductAsinSku(productAsinSku);
row.setStatus(status);
row.setDone(done);
return row;
}
}

19
backend-java/src/test/java/com/nanri/aiimage/modules/similarasin/util/SimilarAsinImageEmbedderTest.java
View File

@@ -177,6 +177,25 @@ class SimilarAsinImageEmbedderTest {
assertNotNull(req.header("Accept-Language"));
}
@Test
void amazonImageDownloadCandidatesTrySmallerVariantsWithinRetryBudget() {
List<String> candidates = SimilarAsinImageEmbedder.downloadCandidates(
"https://m.media-amazon.com/images/I/71AGS2r4JML._AC_SX679_.jpg");
assertEquals(List.of(
"https://m.media-amazon.com/images/I/71AGS2r4JML._AC_SX679_.jpg",
"https://m.media-amazon.com/images/I/71AGS2r4JML._AC_SY450_.jpg",
"https://m.media-amazon.com/images/I/71AGS2r4JML._AC_SX425_.jpg"), candidates);
}
@Test
void downloadCandidatesLeaveNonAmazonAndUnsignedUrlsUntouched() {
String coze = "https://lf9-bot-platform-tos-sign.coze.cn/bot-studio-bot-platform/bot_files/1/image/jpeg/2/merged_image.jpg?x-expires=1&x-signature=a";
assertEquals(List.of(coze), SimilarAsinImageEmbedder.downloadCandidates(coze));
assertEquals(List.of("https://cbu01.alicdn.com/O1CN01x.jpg"),
SimilarAsinImageEmbedder.downloadCandidates("https://cbu01.alicdn.com/O1CN01x.jpg"));
}
@Test
void buildImageRequestUsesDownloadHeadersForCozeSignedImages() {
var req = SimilarAsinImageEmbedder.buildImageRequest(