修复图片下载问题
This commit is contained in:
+58
-5
@@ -35,6 +35,7 @@ import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.HashSet;
|
||||
import java.util.Iterator;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
@@ -345,13 +346,20 @@ public class SimilarAsinImageEmbedder {
|
||||
|
||||
private byte[] downloadWithRetry(String url) throws IOException, TimeoutException {
|
||||
validateHttpsUrl(url);
|
||||
List<String> candidates = downloadCandidates(url);
|
||||
IOException last = null;
|
||||
TimeoutException lastTimeout = null;
|
||||
long waitSeconds = downloadTimeoutSeconds * 2L;
|
||||
for (int attempt = 0; attempt <= DOWNLOAD_MAX_RETRY; attempt++) {
|
||||
Future<byte[]> future = downloadPool.submit(() -> doFetch(url));
|
||||
String attemptUrl = candidates.get(Math.min(attempt, candidates.size() - 1));
|
||||
Future<byte[]> future = downloadPool.submit(() -> doFetch(attemptUrl));
|
||||
try {
|
||||
return future.get(waitSeconds, TimeUnit.SECONDS);
|
||||
byte[] bytes = future.get(waitSeconds, TimeUnit.SECONDS);
|
||||
if (!attemptUrl.equals(url)) {
|
||||
log.info("[similar-asin][image] download-fallback-success originalUrl={} usedUrl={} attempt={}/{}",
|
||||
url, attemptUrl, attempt + 1, DOWNLOAD_MAX_RETRY + 1);
|
||||
}
|
||||
return bytes;
|
||||
} catch (java.util.concurrent.ExecutionException ee) {
|
||||
Throwable cause = ee.getCause();
|
||||
if (cause instanceof DownloadOversizeException doe) {
|
||||
@@ -360,7 +368,7 @@ public class SimilarAsinImageEmbedder {
|
||||
if (cause instanceof IOException io) {
|
||||
last = io;
|
||||
lastTimeout = null;
|
||||
logRetry(url, attempt, io);
|
||||
logRetry(attemptUrl, attempt, io);
|
||||
continue;
|
||||
}
|
||||
if (cause instanceof RuntimeException re) {
|
||||
@@ -371,11 +379,12 @@ public class SimilarAsinImageEmbedder {
|
||||
future.cancel(true);
|
||||
TimeoutException wrapped = new TimeoutException("image download timeout after attempt "
|
||||
+ (attempt + 1) + "/" + (DOWNLOAD_MAX_RETRY + 1)
|
||||
+ ", waitSeconds=" + waitSeconds);
|
||||
+ ", waitSeconds=" + waitSeconds
|
||||
+ ", url=" + attemptUrl);
|
||||
wrapped.initCause(te);
|
||||
lastTimeout = wrapped;
|
||||
last = null;
|
||||
logRetry(url, attempt, wrapped);
|
||||
logRetry(attemptUrl, attempt, wrapped);
|
||||
} catch (InterruptedException ie) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new IOException("image download interrupted", ie);
|
||||
@@ -387,6 +396,50 @@ public class SimilarAsinImageEmbedder {
|
||||
throw last != null ? last : new IOException("image download failed without cause");
|
||||
}
|
||||
|
||||
static List<String> downloadCandidates(String url) {
|
||||
List<String> single = List.of(url);
|
||||
URI uri;
|
||||
try {
|
||||
uri = new URI(url);
|
||||
} catch (URISyntaxException ex) {
|
||||
return single;
|
||||
}
|
||||
String host = uri.getHost();
|
||||
String rawPath = uri.getRawPath();
|
||||
if (host == null || rawPath == null
|
||||
|| !host.toLowerCase(Locale.ROOT).endsWith("media-amazon.com")
|
||||
|| !rawPath.startsWith("/images/I/")) {
|
||||
return single;
|
||||
}
|
||||
int slash = rawPath.lastIndexOf('/');
|
||||
String dir = rawPath.substring(0, slash + 1);
|
||||
String file = rawPath.substring(slash + 1);
|
||||
int markerStart = file.lastIndexOf("._");
|
||||
int markerEnd = markerStart < 0 ? -1 : file.indexOf("_.", markerStart + 2);
|
||||
if (markerStart < 0 || markerEnd < 0) {
|
||||
return single;
|
||||
}
|
||||
String prefix = file.substring(0, markerStart);
|
||||
String ext = file.substring(markerEnd + 1);
|
||||
LinkedHashSet<String> candidates = new LinkedHashSet<>();
|
||||
candidates.add(url);
|
||||
candidates.add(buildUrl(uri, dir + prefix + "._AC_SY450_" + ext));
|
||||
candidates.add(buildUrl(uri, dir + prefix + "._AC_SX425_" + ext));
|
||||
return new ArrayList<>(candidates);
|
||||
}
|
||||
|
||||
private static String buildUrl(URI uri, String rawPath) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append(uri.getScheme()).append("://").append(uri.getRawAuthority()).append(rawPath);
|
||||
if (uri.getRawQuery() != null) {
|
||||
sb.append('?').append(uri.getRawQuery());
|
||||
}
|
||||
if (uri.getRawFragment() != null) {
|
||||
sb.append('#').append(uri.getRawFragment());
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
private void logRetry(String url, int attempt, Exception ex) {
|
||||
if (attempt < DOWNLOAD_MAX_RETRY) {
|
||||
log.debug("[similar-asin][image] download-retry url={} nextAttempt={}/{} err={}",
|
||||
|
||||
Reference in New Issue
Block a user