super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: update task routing and coze handling

Browse Source
This commit is contained in:
super
2026-06-03 17:14:50 +08:00
parent ea37d82d73
commit e98a1a1207
37 changed files with 1883 additions and 254 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-java/src/main/java/com/nanri/aiimage/config/AppearancePatentProperties.java
+5
View File
@@ -23,6 +23,11 @@ public class AppearancePatentProperties {
private int cozePollTimeoutMillis = 600000;
private int staleTimeoutMinutes = 30;
private String staleFinalizeCron = "0 */2 * * * *";
/**
* 末尾不足一批的数据等待该时长后强制提交 Coze。
* Python 回传超时后会直接封口；这个配置处理 Python 慢回传但仍未超时的零头批次。
*/
private int cozeFlushPendingMinutes = 1;
@Data
public static class CozeCredential {
backend-java/src/main/java/com/nanri/aiimage/config/BrandCheckProperties.java
+15
View File
@@ -0,0 +1,15 @@
package com.nanri.aiimage.config;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
@Data
@ConfigurationProperties(prefix = "aiimage.brand-check")
public class BrandCheckProperties {
private String baseUrl = "http://47.110.241.161:16890";
private String path = "/brand_check";
private String token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9";
private String defaultStrategy = "Terms";
private int connectTimeoutMillis = 10000;
private int readTimeoutMillis = 60000;
}
backend-java/src/main/java/com/nanri/aiimage/config/DeleteBrandProgressProperties.java
+9 -9
View File
@@ -7,7 +7,6 @@ import org.springframework.boot.context.properties.ConfigurationProperties;
@ConfigurationProperties(prefix = "aiimage.delete-brand-progress")
public class DeleteBrandProgressProperties {
private long heartbeatTimeoutMinutes = 15;
private long deleteBrandInitialTimeoutMinutes = 20;
private String staleCheckCron = "*/30 * * * * *";
/**
@@ -19,24 +18,25 @@ public class DeleteBrandProgressProperties {
* 商品风险（PRODUCT_RISK_RESOLVE）RUNNING 超时自动收尾/失败，按分钟计算；
* 与删除品牌共用同一定时调度。
*/
private long productRiskStaleTimeoutMinutes = 20;
private long productRiskInitialTimeoutMinutes = 20;
private long productRiskStaleTimeoutMinutes = 30;
/**
* Price track RUNNING timeout auto-finalize/fail threshold in minutes.
*/
private long priceTrackStaleTimeoutMinutes = 20;
private long priceTrackInitialTimeoutMinutes = 20;
private long priceTrackStaleTimeoutMinutes = 30;
/**
* Shop match RUNNING timeout auto-finalize/fail threshold in minutes.
*/
private long shopMatchStaleTimeoutMinutes = 20;
private long shopMatchInitialTimeoutMinutes = 20;
private long shopMatchStaleTimeoutMinutes = 30;
/**
* Patrol delete RUNNING timeout auto-finalize/fail threshold in minutes.
*/
private long patrolDeleteStaleTimeoutMinutes = 20;
private long patrolDeleteInitialTimeoutMinutes = 20;
private long patrolDeleteStaleTimeoutMinutes = 30;
/**
* Query ASIN RUNNING timeout auto-finalize/fail threshold in minutes.
*/
private long queryAsinStaleTimeoutMinutes = 30;
}
backend-java/src/main/java/com/nanri/aiimage/config/InstanceRoutingProperties.java
+17
View File
@@ -0,0 +1,17 @@
package com.nanri.aiimage.config;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import java.util.LinkedHashMap;
import java.util.Map;
@Data
@ConfigurationProperties(prefix = "aiimage.instance-routing")
public class InstanceRoutingProperties {
private Map<String, String> routes = new LinkedHashMap<>();
private int connectTimeoutMillis = 3000;
private int readTimeoutMillis = 300000;
private int requestBodyCacheLimitBytes = 104857600;
}
backend-java/src/main/java/com/nanri/aiimage/config/PropertiesConfig.java
+1 -1
View File
@@ -4,6 +4,6 @@ import org.springframework.boot.context.properties.EnableConfigurationProperties
import org.springframework.context.annotation.Configuration;
@Configuration
@EnableConfigurationProperties({OssProperties.class, TransientStorageProperties.class, StorageProperties.class, BrandProgressProperties.class, DeleteBrandProgressProperties.class, ZiniaoProperties.class, ModuleCleanupProperties.class, TaskPressureProperties.class, AppearancePatentProperties.class, SimilarAsinProperties.class})
@EnableConfigurationProperties({OssProperties.class, TransientStorageProperties.class, StorageProperties.class, BrandProgressProperties.class, DeleteBrandProgressProperties.class, BrandCheckProperties.class, ZiniaoProperties.class, ModuleCleanupProperties.class, TaskPressureProperties.class, TaskImageCacheCleanupProperties.class, AppearancePatentProperties.class, SimilarAsinProperties.class, InstanceRoutingProperties.class})
public class PropertiesConfig {
}
backend-java/src/main/java/com/nanri/aiimage/config/RequestTraceFilter.java
+34 -14
View File
@@ -6,8 +6,10 @@ import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.web.filter.OncePerRequestFilter;
import org.springframework.web.util.ContentCachingRequestWrapper;
import java.io.IOException;
import java.util.Optional;
@@ -18,27 +20,31 @@ public class RequestTraceFilter extends OncePerRequestFilter {
private static final Logger log = LoggerFactory.getLogger(RequestTraceFilter.class);
private final InstanceMetadata instanceMetadata;
private final int requestBodyCacheLimitBytes;
public RequestTraceFilter(InstanceMetadata instanceMetadata) {
public RequestTraceFilter(InstanceMetadata instanceMetadata,
@Value("${aiimage.instance-routing.request-body-cache-limit-bytes:104857600}") int requestBodyCacheLimitBytes) {
this.instanceMetadata = instanceMetadata;
this.requestBodyCacheLimitBytes = Math.max(1024 * 1024, requestBodyCacheLimitBytes);
}
@Override
protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain filterChain)
throws ServletException, IOException {
long start = System.currentTimeMillis();
HttpServletRequest requestToUse = wrapRequestIfNeeded(request, requestBodyCacheLimitBytes);
String remoteAddr = firstNonBlank(
request.getHeader("X-Forwarded-For"),
request.getHeader("X-Real-IP"),
request.getRemoteAddr()
requestToUse.getHeader("X-Forwarded-For"),
requestToUse.getHeader("X-Real-IP"),
requestToUse.getRemoteAddr()
);
String forwardedProto = request.getHeader("X-Forwarded-Proto");
String forwardedHost = firstNonBlank(request.getHeader("X-Forwarded-Host"), request.getHeader("Host"));
String forwardedPort = request.getHeader("X-Forwarded-Port");
String forwardedProto = requestToUse.getHeader("X-Forwarded-Proto");
String forwardedHost = firstNonBlank(requestToUse.getHeader("X-Forwarded-Host"), requestToUse.getHeader("Host"));
String forwardedPort = requestToUse.getHeader("X-Forwarded-Port");
String requestId = firstNonBlank(
request.getHeader("X-Request-Id"),
request.getHeader("X-Amzn-Trace-Id"),
request.getHeader("Traceparent")
requestToUse.getHeader("X-Request-Id"),
requestToUse.getHeader("X-Amzn-Trace-Id"),
requestToUse.getHeader("Traceparent")
);
response.setHeader("X-AIIMAGE-Instance", instanceMetadata.getInstanceId());
@@ -47,7 +53,7 @@ public class RequestTraceFilter extends OncePerRequestFilter {
response.setHeader("X-AIIMAGE-Instance-Stable", String.valueOf(instanceMetadata.isStable()));
try {
filterChain.doFilter(request, response);
filterChain.doFilter(requestToUse, response);
} finally {
long costMs = System.currentTimeMillis() - start;
log.info(
@@ -56,20 +62,34 @@ public class RequestTraceFilter extends OncePerRequestFilter {
instanceMetadata.getSource(),
instanceMetadata.isStable(),
instanceMetadata.getHostname(),
request.getMethod(),
request.getRequestURI(),
requestToUse.getMethod(),
requestToUse.getRequestURI(),
response.getStatus(),
remoteAddr,
blankToDash(forwardedHost),
blankToDash(forwardedProto),
blankToDash(forwardedPort),
blankToDash(requestId),
blankToDash(request.getHeader("User-Agent")),
blankToDash(requestToUse.getHeader("User-Agent")),
costMs
);
}
}
private static HttpServletRequest wrapRequestIfNeeded(HttpServletRequest request, int requestBodyCacheLimitBytes) {
if (request instanceof ContentCachingRequestWrapper) {
return request;
}
String method = request.getMethod();
if ("POST".equalsIgnoreCase(method)
|| "PUT".equalsIgnoreCase(method)
|| "PATCH".equalsIgnoreCase(method)
|| "DELETE".equalsIgnoreCase(method)) {
return new ContentCachingRequestWrapper(request, requestBodyCacheLimitBytes);
}
return request;
}
private static String firstNonBlank(String... values) {
for (String value : values) {
if (value != null && !value.isBlank()) {
backend-java/src/main/java/com/nanri/aiimage/config/TaskImageCacheCleanupProperties.java
+15
View File
@@ -0,0 +1,15 @@
package com.nanri.aiimage.config;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
@Data
@ConfigurationProperties(prefix = "aiimage.task-image-cache-cleanup")
public class TaskImageCacheCleanupProperties {
private boolean enabled = true;
private String cron = "0 30 2 * * *";
private int retentionDays = 3;
private int batchSize = 5000;
private int maxBatchesPerRun = 200;
}
backend-java/src/main/java/com/nanri/aiimage/config/TaskOperationLockConfig.java
+16 -5
View File
@@ -20,6 +20,8 @@ import java.util.regex.Pattern;
@RequiredArgsConstructor
public class TaskOperationLockConfig implements WebMvcConfigurer {
private static final String LOCK_ATTRIBUTE = TaskOperationLockConfig.class.getName() + ".LOCK";
private final TaskDistributedLockService taskDistributedLockService;
@Override
@@ -28,6 +30,19 @@ public class TaskOperationLockConfig implements WebMvcConfigurer {
.addPathPatterns("/api/**");
}
public static boolean releaseRequestLock(HttpServletRequest request) {
if (request == null) {
return false;
}
Object lockHandle = request.getAttribute(LOCK_ATTRIBUTE);
if (lockHandle instanceof TaskDistributedLockService.LockHandle handle) {
request.removeAttribute(LOCK_ATTRIBUTE);
handle.close();
return true;
}
return false;
}
@Slf4j
private static final class TaskOperationLockInterceptor implements HandlerInterceptor {
@@ -36,7 +51,6 @@ public class TaskOperationLockConfig implements WebMvcConfigurer {
private static final Set<String> MUTATING_METHODS = Set.of("POST", "PUT", "PATCH", "DELETE");
private static final long RESULT_SUBMIT_WAIT_MILLIS = 30 * 1000L;
private static final long DELETE_WAIT_MILLIS = 60 * 1000L;
private static final String LOCK_ATTRIBUTE = TaskOperationLockInterceptor.class.getName() + ".LOCK";
private final TaskDistributedLockService taskDistributedLockService;
@@ -83,10 +97,7 @@ public class TaskOperationLockConfig implements WebMvcConfigurer {
HttpServletResponse response,
Object handler,
Exception ex) {
Object lockHandle = request.getAttribute(LOCK_ATTRIBUTE);
if (lockHandle instanceof TaskDistributedLockService.LockHandle handle) {
handle.close();
}
releaseRequestLock(request);
}
}
}