Compare commits
2 Commits
backup/bac
...
e98a1a1207
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e98a1a1207 | ||
|
|
ea37d82d73 |
7818
2026_05_29.log
7818
2026_05_29.log
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
1
app/assets/appearance-patent-Cb_Fx4da.css
Normal file
1
app/assets/appearance-patent-Cb_Fx4da.css
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -1 +1 @@
|
|||||||
import{bN as r}from"./java-modules-B8c-YG5x.js";const n="";function s(e){return r(`${n}/api/brand/expand-folder-recursive`,{folder:e})}export{s as e};
|
import{bN as r}from"./java-modules-WSWRTEaR.js";const n="";function s(e){return r(`${n}/api/brand/expand-folder-recursive`,{folder:e})}export{s as e};
|
||||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -5,9 +5,9 @@
|
|||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
<title>格式转换 - 数富AI</title>
|
<title>格式转换 - 数富AI</title>
|
||||||
<script type="module" crossorigin src="/assets/convert.js"></script>
|
<script type="module" crossorigin src="/assets/convert.js"></script>
|
||||||
<link rel="modulepreload" crossorigin href="/assets/java-modules-B8c-YG5x.js">
|
<link rel="modulepreload" crossorigin href="/assets/java-modules-WSWRTEaR.js">
|
||||||
<link rel="modulepreload" crossorigin href="/assets/brand-COze15GJ.js">
|
<link rel="modulepreload" crossorigin href="/assets/brand-5GpfAyHD.js">
|
||||||
<link rel="stylesheet" crossorigin href="/assets/java-modules-l5anrOZ2.css">
|
<link rel="stylesheet" crossorigin href="/assets/java-modules-B3hGvRck.css">
|
||||||
<link rel="stylesheet" crossorigin href="/assets/convert-6oxZNMye.css">
|
<link rel="stylesheet" crossorigin href="/assets/convert-6oxZNMye.css">
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
|
|||||||
@@ -5,9 +5,9 @@
|
|||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
<title>数据去重 - 数富AI</title>
|
<title>数据去重 - 数富AI</title>
|
||||||
<script type="module" crossorigin src="/assets/dedupe.js"></script>
|
<script type="module" crossorigin src="/assets/dedupe.js"></script>
|
||||||
<link rel="modulepreload" crossorigin href="/assets/java-modules-B8c-YG5x.js">
|
<link rel="modulepreload" crossorigin href="/assets/java-modules-WSWRTEaR.js">
|
||||||
<link rel="modulepreload" crossorigin href="/assets/brand-COze15GJ.js">
|
<link rel="modulepreload" crossorigin href="/assets/brand-5GpfAyHD.js">
|
||||||
<link rel="stylesheet" crossorigin href="/assets/java-modules-l5anrOZ2.css">
|
<link rel="stylesheet" crossorigin href="/assets/java-modules-B3hGvRck.css">
|
||||||
<link rel="stylesheet" crossorigin href="/assets/dedupe-DpfPYQDv.css">
|
<link rel="stylesheet" crossorigin href="/assets/dedupe-DpfPYQDv.css">
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
|
|||||||
@@ -5,10 +5,10 @@
|
|||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
<title>删除品牌 - 数富AI</title>
|
<title>删除品牌 - 数富AI</title>
|
||||||
<script type="module" crossorigin src="/assets/delete-brand.js"></script>
|
<script type="module" crossorigin src="/assets/delete-brand.js"></script>
|
||||||
<link rel="modulepreload" crossorigin href="/assets/java-modules-B8c-YG5x.js">
|
<link rel="modulepreload" crossorigin href="/assets/java-modules-WSWRTEaR.js">
|
||||||
<link rel="modulepreload" crossorigin href="/assets/brand-COze15GJ.js">
|
<link rel="modulepreload" crossorigin href="/assets/brand-5GpfAyHD.js">
|
||||||
<link rel="modulepreload" crossorigin href="/assets/categorized-timers-JPA-olTr.js">
|
<link rel="modulepreload" crossorigin href="/assets/categorized-timers-JPA-olTr.js">
|
||||||
<link rel="stylesheet" crossorigin href="/assets/java-modules-l5anrOZ2.css">
|
<link rel="stylesheet" crossorigin href="/assets/java-modules-B3hGvRck.css">
|
||||||
<link rel="stylesheet" crossorigin href="/assets/delete-brand-DZ3x5hOX.css">
|
<link rel="stylesheet" crossorigin href="/assets/delete-brand-DZ3x5hOX.css">
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
|
|||||||
@@ -5,9 +5,9 @@
|
|||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
<title>数据拆分 - 数富AI</title>
|
<title>数据拆分 - 数富AI</title>
|
||||||
<script type="module" crossorigin src="/assets/split.js"></script>
|
<script type="module" crossorigin src="/assets/split.js"></script>
|
||||||
<link rel="modulepreload" crossorigin href="/assets/java-modules-B8c-YG5x.js">
|
<link rel="modulepreload" crossorigin href="/assets/java-modules-WSWRTEaR.js">
|
||||||
<link rel="modulepreload" crossorigin href="/assets/brand-COze15GJ.js">
|
<link rel="modulepreload" crossorigin href="/assets/brand-5GpfAyHD.js">
|
||||||
<link rel="stylesheet" crossorigin href="/assets/java-modules-l5anrOZ2.css">
|
<link rel="stylesheet" crossorigin href="/assets/java-modules-B3hGvRck.css">
|
||||||
<link rel="stylesheet" crossorigin href="/assets/split-CR1PtDQE.css">
|
<link rel="stylesheet" crossorigin href="/assets/split-CR1PtDQE.css">
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
|
|||||||
@@ -647,7 +647,7 @@
|
|||||||
<div class="nav-section" data-column-key="brand_front_tools">
|
<div class="nav-section" data-column-key="brand_front_tools">
|
||||||
<div class="nav-section-title">前端工具</div>
|
<div class="nav-section-title">前端工具</div>
|
||||||
<div class="nav-section-items">
|
<div class="nav-section-items">
|
||||||
<span class="nav-item disabled">采集数据</span>
|
<a class="nav-item" href="/new_web_source/collect-data.html">采集数据</a>
|
||||||
<span class="nav-item disabled">变体分析</span>
|
<span class="nav-item disabled">变体分析</span>
|
||||||
<button type="button" class="nav-item active" data-panel="brandCheck">品牌检测</button>
|
<button type="button" class="nav-item active" data-panel="brandCheck">品牌检测</button>
|
||||||
<a class="nav-item" href="/new_web_source/appearance-patent.html">外观专利检测</a>
|
<a class="nav-item" href="/new_web_source/appearance-patent.html">外观专利检测</a>
|
||||||
@@ -1447,4 +1447,4 @@
|
|||||||
</script>
|
</script>
|
||||||
</body>
|
</body>
|
||||||
|
|
||||||
</html>
|
</html>
|
||||||
|
|||||||
@@ -107,7 +107,7 @@ Content-Type: application/json
|
|||||||
当前超时配置来自后端配置项:
|
当前超时配置来自后端配置项:
|
||||||
|
|
||||||
- `aiimage.delete-brand-progress.patrol-delete-stale-timeout-minutes`
|
- `aiimage.delete-brand-progress.patrol-delete-stale-timeout-minutes`
|
||||||
- `aiimage.delete-brand-progress.patrol-delete-initial-timeout-minutes`
|
- Python should call `POST /api/tasks/{taskId}/heartbeat` every minute; timeout now uses only the normal stale-timeout setting.
|
||||||
|
|
||||||
## 6. 历史清理
|
## 6. 历史清理
|
||||||
|
|
||||||
|
|||||||
@@ -1,8 +1,13 @@
|
|||||||
package com.nanri.aiimage.common.exception;
|
package com.nanri.aiimage.common.exception;
|
||||||
|
|
||||||
import com.nanri.aiimage.common.api.ApiResponse;
|
import com.nanri.aiimage.common.api.ApiResponse;
|
||||||
|
import com.nanri.aiimage.common.service.TaskOwnerForwardService;
|
||||||
|
import com.nanri.aiimage.config.TaskOperationLockConfig;
|
||||||
|
import jakarta.servlet.http.HttpServletRequest;
|
||||||
import jakarta.validation.ConstraintViolationException;
|
import jakarta.validation.ConstraintViolationException;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.http.ResponseEntity;
|
||||||
import org.springframework.web.bind.MethodArgumentNotValidException;
|
import org.springframework.web.bind.MethodArgumentNotValidException;
|
||||||
import org.springframework.web.bind.annotation.ExceptionHandler;
|
import org.springframework.web.bind.annotation.ExceptionHandler;
|
||||||
import org.springframework.web.bind.annotation.RestControllerAdvice;
|
import org.springframework.web.bind.annotation.RestControllerAdvice;
|
||||||
@@ -11,9 +16,32 @@ import org.springframework.web.context.request.async.AsyncRequestNotUsableExcept
|
|||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
|
||||||
@Slf4j
|
@Slf4j
|
||||||
|
@RequiredArgsConstructor
|
||||||
@RestControllerAdvice
|
@RestControllerAdvice
|
||||||
public class GlobalExceptionHandler {
|
public class GlobalExceptionHandler {
|
||||||
|
|
||||||
|
private final TaskOwnerForwardService taskOwnerForwardService;
|
||||||
|
|
||||||
|
@ExceptionHandler(TaskOwnerMismatchException.class)
|
||||||
|
public Object handleTaskOwnerMismatchException(TaskOwnerMismatchException ex, HttpServletRequest request) {
|
||||||
|
TaskOperationLockConfig.releaseRequestLock(request);
|
||||||
|
try {
|
||||||
|
ResponseEntity<byte[]> response = taskOwnerForwardService.forwardCurrentRequest(ex, request);
|
||||||
|
return ResponseEntity.status(response.getStatusCode())
|
||||||
|
.headers(response.getHeaders())
|
||||||
|
.body(response.getBody());
|
||||||
|
} catch (BusinessException forwardEx) {
|
||||||
|
return forwardEx.getCode() == null
|
||||||
|
? ApiResponse.fail(forwardEx.getMessage())
|
||||||
|
: ApiResponse.fail(forwardEx.getCode(), forwardEx.getMessage());
|
||||||
|
} catch (Exception forwardEx) {
|
||||||
|
log.warn("[instance-routing] forward failed taskId={} operation={} owner={} current={} msg={}",
|
||||||
|
ex.getTaskId(), ex.getOperation(), ex.getOwnerInstanceId(), ex.getCurrentInstanceId(),
|
||||||
|
forwardEx.getMessage(), forwardEx);
|
||||||
|
return ApiResponse.fail(40903, "任务归属实例转发失败: " + forwardEx.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@ExceptionHandler(BusinessException.class)
|
@ExceptionHandler(BusinessException.class)
|
||||||
public ApiResponse<Void> handleBusinessException(BusinessException ex) {
|
public ApiResponse<Void> handleBusinessException(BusinessException ex) {
|
||||||
if (Integer.valueOf(40901).equals(ex.getCode())) {
|
if (Integer.valueOf(40901).equals(ex.getCode())) {
|
||||||
|
|||||||
@@ -0,0 +1,36 @@
|
|||||||
|
package com.nanri.aiimage.common.exception;
|
||||||
|
|
||||||
|
public class TaskOwnerMismatchException extends BusinessException {
|
||||||
|
|
||||||
|
private final Long taskId;
|
||||||
|
private final String operation;
|
||||||
|
private final String ownerInstanceId;
|
||||||
|
private final String currentInstanceId;
|
||||||
|
|
||||||
|
public TaskOwnerMismatchException(Long taskId,
|
||||||
|
String operation,
|
||||||
|
String ownerInstanceId,
|
||||||
|
String currentInstanceId) {
|
||||||
|
super(40903, "该任务已绑定到另一台服务实例处理,请通过原实例继续处理");
|
||||||
|
this.taskId = taskId;
|
||||||
|
this.operation = operation;
|
||||||
|
this.ownerInstanceId = ownerInstanceId;
|
||||||
|
this.currentInstanceId = currentInstanceId;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Long getTaskId() {
|
||||||
|
return taskId;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getOperation() {
|
||||||
|
return operation;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getOwnerInstanceId() {
|
||||||
|
return ownerInstanceId;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getCurrentInstanceId() {
|
||||||
|
return currentInstanceId;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,156 @@
|
|||||||
|
package com.nanri.aiimage.common.service;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.common.exception.BusinessException;
|
||||||
|
import com.nanri.aiimage.common.exception.TaskOwnerMismatchException;
|
||||||
|
import com.nanri.aiimage.config.InstanceRoutingProperties;
|
||||||
|
import jakarta.servlet.http.HttpServletRequest;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.http.HttpHeaders;
|
||||||
|
import org.springframework.http.HttpMethod;
|
||||||
|
import org.springframework.http.MediaType;
|
||||||
|
import org.springframework.http.ResponseEntity;
|
||||||
|
import org.springframework.http.client.SimpleClientHttpRequestFactory;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
import org.springframework.util.StreamUtils;
|
||||||
|
import org.springframework.web.client.RestClient;
|
||||||
|
import org.springframework.web.util.ContentCachingRequestWrapper;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.Enumeration;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Set;
|
||||||
|
|
||||||
|
@Slf4j
|
||||||
|
@Service
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
public class TaskOwnerForwardService {
|
||||||
|
|
||||||
|
public static final String FORWARDED_HEADER = "X-AIIMAGE-Owner-Forwarded";
|
||||||
|
|
||||||
|
private static final Set<String> HOP_BY_HOP_HEADERS = Set.of(
|
||||||
|
"connection",
|
||||||
|
"keep-alive",
|
||||||
|
"proxy-authenticate",
|
||||||
|
"proxy-authorization",
|
||||||
|
"te",
|
||||||
|
"trailer",
|
||||||
|
"transfer-encoding",
|
||||||
|
"upgrade",
|
||||||
|
"host",
|
||||||
|
"content-length"
|
||||||
|
);
|
||||||
|
|
||||||
|
private final InstanceRoutingProperties properties;
|
||||||
|
|
||||||
|
private volatile RestClient sharedRestClient;
|
||||||
|
|
||||||
|
public ResponseEntity<byte[]> forwardCurrentRequest(TaskOwnerMismatchException ex, HttpServletRequest request) {
|
||||||
|
if (request == null) {
|
||||||
|
throw new BusinessException(40903, "无法获取当前请求,不能转发到任务归属实例");
|
||||||
|
}
|
||||||
|
if (hasAlreadyForwarded(request)) {
|
||||||
|
throw new BusinessException(40903, "任务归属实例转发检测到循环,请检查实例路由配置");
|
||||||
|
}
|
||||||
|
HttpMethod method = HttpMethod.valueOf(request.getMethod());
|
||||||
|
String url = resolveUrl(ex, currentPathAndQuery(request));
|
||||||
|
byte[] body = requestBody(request);
|
||||||
|
HttpHeaders headers = copyForwardHeaders(request, ex.getCurrentInstanceId());
|
||||||
|
log.info("[instance-routing] forward {} taskId={} operation={} owner={} current={} url={}",
|
||||||
|
method, ex.getTaskId(), ex.getOperation(), ex.getOwnerInstanceId(), ex.getCurrentInstanceId(), url);
|
||||||
|
return restClient().method(method)
|
||||||
|
.uri(url)
|
||||||
|
.headers(target -> target.addAll(headers))
|
||||||
|
.body(body)
|
||||||
|
.retrieve()
|
||||||
|
.toEntity(byte[].class);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String resolveUrl(TaskOwnerMismatchException ex, String path) {
|
||||||
|
String owner = ex.getOwnerInstanceId();
|
||||||
|
String baseUrl = properties.getRoutes().get(owner);
|
||||||
|
if (baseUrl == null || baseUrl.isBlank()) {
|
||||||
|
throw new BusinessException(40903, "任务归属实例未配置服务路由:" + owner);
|
||||||
|
}
|
||||||
|
return stripTrailingSlash(baseUrl) + ensureLeadingSlash(path);
|
||||||
|
}
|
||||||
|
|
||||||
|
private RestClient restClient() {
|
||||||
|
RestClient client = sharedRestClient;
|
||||||
|
if (client != null) {
|
||||||
|
return client;
|
||||||
|
}
|
||||||
|
synchronized (this) {
|
||||||
|
if (sharedRestClient == null) {
|
||||||
|
SimpleClientHttpRequestFactory requestFactory = new SimpleClientHttpRequestFactory();
|
||||||
|
requestFactory.setConnectTimeout(properties.getConnectTimeoutMillis());
|
||||||
|
requestFactory.setReadTimeout(properties.getReadTimeoutMillis());
|
||||||
|
sharedRestClient = RestClient.builder().requestFactory(requestFactory).build();
|
||||||
|
}
|
||||||
|
return sharedRestClient;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static boolean hasAlreadyForwarded(HttpServletRequest request) {
|
||||||
|
String value = request.getHeader(FORWARDED_HEADER);
|
||||||
|
return value != null && !value.isBlank();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String currentPathAndQuery(HttpServletRequest request) {
|
||||||
|
String path = request.getRequestURI();
|
||||||
|
String query = request.getQueryString();
|
||||||
|
return query == null || query.isBlank() ? path : path + "?" + query;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static byte[] requestBody(HttpServletRequest request) {
|
||||||
|
if (request instanceof ContentCachingRequestWrapper wrapper) {
|
||||||
|
byte[] body = wrapper.getContentAsByteArray();
|
||||||
|
return body == null ? new byte[0] : body;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return StreamUtils.copyToByteArray(request.getInputStream());
|
||||||
|
} catch (IOException ex) {
|
||||||
|
throw new BusinessException("读取转发请求体失败:" + ex.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static HttpHeaders copyForwardHeaders(HttpServletRequest request, String currentInstanceId) {
|
||||||
|
HttpHeaders headers = new HttpHeaders();
|
||||||
|
Enumeration<String> headerNames = request.getHeaderNames();
|
||||||
|
while (headerNames != null && headerNames.hasMoreElements()) {
|
||||||
|
String headerName = headerNames.nextElement();
|
||||||
|
if (headerName == null || HOP_BY_HOP_HEADERS.contains(headerName.toLowerCase())) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
Enumeration<String> values = request.getHeaders(headerName);
|
||||||
|
while (values != null && values.hasMoreElements()) {
|
||||||
|
headers.add(headerName, values.nextElement());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
headers.set(FORWARDED_HEADER, currentInstanceId == null ? "unknown-instance" : currentInstanceId);
|
||||||
|
if (!headers.containsKey(HttpHeaders.ACCEPT)) {
|
||||||
|
headers.setAccept(List.of(MediaType.APPLICATION_JSON));
|
||||||
|
}
|
||||||
|
if (!headers.containsKey(HttpHeaders.CONTENT_TYPE)
|
||||||
|
&& request.getContentType() != null
|
||||||
|
&& !request.getContentType().isBlank()) {
|
||||||
|
headers.set(HttpHeaders.CONTENT_TYPE, request.getContentType());
|
||||||
|
}
|
||||||
|
return headers;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String stripTrailingSlash(String value) {
|
||||||
|
String result = value == null ? "" : value.trim();
|
||||||
|
while (result.endsWith("/")) {
|
||||||
|
result = result.substring(0, result.length() - 1);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String ensureLeadingSlash(String value) {
|
||||||
|
if (value == null || value.isBlank()) {
|
||||||
|
return "/";
|
||||||
|
}
|
||||||
|
return value.startsWith("/") ? value : "/" + value;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -50,9 +50,9 @@ public final class CozeGroupResultPropagator {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* 否定前缀关键字。若当前值同时包含 standard 和这些关键字之一,则不视为命中。
|
* 否定前缀关键字。若当前值同时包含 standard 和这些关键字之一,则不视为命中。
|
||||||
* 用于规避"没有侵权""不侵权""未发现明显侵权风险"等被 contains 误判为命中"侵权"的情况。
|
* 用于规避"没有侵权""无侵权""不侵权""未发现明显侵权风险"等被 contains 误判为命中"侵权"的情况。
|
||||||
*/
|
*/
|
||||||
private static final List<String> NEGATIVE_KEYWORDS = Arrays.asList("没有", "不", "未");
|
private static final List<String> NEGATIVE_KEYWORDS = Arrays.asList("没有", "无", "不", "未");
|
||||||
|
|
||||||
private CozeGroupResultPropagator() {
|
private CozeGroupResultPropagator() {
|
||||||
}
|
}
|
||||||
@@ -134,8 +134,8 @@ public final class CozeGroupResultPropagator {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// 按 hitValues 顺序检测组内是否有命中(包含语义),命中即定下"标准值"。
|
// 按 hitValues 顺序检测组内是否有命中(包含语义),命中即定下"标准值"。
|
||||||
// 命中需同时满足:当前值包含 standard 且不含任何否定关键字("没有"/"不"/"未"),
|
// 命中需同时满足:当前值包含 standard 且不含任何否定关键字("没有"/"无"/"不"/"未"),
|
||||||
// 避免"没有侵权""不侵权""未发现明显侵权风险"这类被 contains 误判为命中。
|
// 避免"没有侵权""无侵权""不侵权""未发现明显侵权风险"这类被 contains 误判为命中。
|
||||||
String matchedStandardValue = null;
|
String matchedStandardValue = null;
|
||||||
outer:
|
outer:
|
||||||
for (String standard : hitValues) {
|
for (String standard : hitValues) {
|
||||||
|
|||||||
@@ -23,6 +23,11 @@ public class AppearancePatentProperties {
|
|||||||
private int cozePollTimeoutMillis = 600000;
|
private int cozePollTimeoutMillis = 600000;
|
||||||
private int staleTimeoutMinutes = 30;
|
private int staleTimeoutMinutes = 30;
|
||||||
private String staleFinalizeCron = "0 */2 * * * *";
|
private String staleFinalizeCron = "0 */2 * * * *";
|
||||||
|
/**
|
||||||
|
* 末尾不足一批的数据等待该时长后强制提交 Coze。
|
||||||
|
* Python 回传超时后会直接封口;这个配置处理 Python 慢回传但仍未超时的零头批次。
|
||||||
|
*/
|
||||||
|
private int cozeFlushPendingMinutes = 1;
|
||||||
|
|
||||||
@Data
|
@Data
|
||||||
public static class CozeCredential {
|
public static class CozeCredential {
|
||||||
|
|||||||
@@ -0,0 +1,15 @@
|
|||||||
|
package com.nanri.aiimage.config;
|
||||||
|
|
||||||
|
import lombok.Data;
|
||||||
|
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@ConfigurationProperties(prefix = "aiimage.brand-check")
|
||||||
|
public class BrandCheckProperties {
|
||||||
|
private String baseUrl = "http://47.110.241.161:16890";
|
||||||
|
private String path = "/brand_check";
|
||||||
|
private String token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9";
|
||||||
|
private String defaultStrategy = "Terms";
|
||||||
|
private int connectTimeoutMillis = 10000;
|
||||||
|
private int readTimeoutMillis = 60000;
|
||||||
|
}
|
||||||
@@ -7,7 +7,6 @@ import org.springframework.boot.context.properties.ConfigurationProperties;
|
|||||||
@ConfigurationProperties(prefix = "aiimage.delete-brand-progress")
|
@ConfigurationProperties(prefix = "aiimage.delete-brand-progress")
|
||||||
public class DeleteBrandProgressProperties {
|
public class DeleteBrandProgressProperties {
|
||||||
private long heartbeatTimeoutMinutes = 15;
|
private long heartbeatTimeoutMinutes = 15;
|
||||||
private long deleteBrandInitialTimeoutMinutes = 20;
|
|
||||||
private String staleCheckCron = "*/30 * * * * *";
|
private String staleCheckCron = "*/30 * * * * *";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -19,24 +18,25 @@ public class DeleteBrandProgressProperties {
|
|||||||
* 商品风险(PRODUCT_RISK_RESOLVE)RUNNING 超时自动收尾/失败,按分钟计算;
|
* 商品风险(PRODUCT_RISK_RESOLVE)RUNNING 超时自动收尾/失败,按分钟计算;
|
||||||
* 与删除品牌共用同一定时调度。
|
* 与删除品牌共用同一定时调度。
|
||||||
*/
|
*/
|
||||||
private long productRiskStaleTimeoutMinutes = 20;
|
private long productRiskStaleTimeoutMinutes = 30;
|
||||||
private long productRiskInitialTimeoutMinutes = 20;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Price track RUNNING timeout auto-finalize/fail threshold in minutes.
|
* Price track RUNNING timeout auto-finalize/fail threshold in minutes.
|
||||||
*/
|
*/
|
||||||
private long priceTrackStaleTimeoutMinutes = 20;
|
private long priceTrackStaleTimeoutMinutes = 30;
|
||||||
private long priceTrackInitialTimeoutMinutes = 20;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Shop match RUNNING timeout auto-finalize/fail threshold in minutes.
|
* Shop match RUNNING timeout auto-finalize/fail threshold in minutes.
|
||||||
*/
|
*/
|
||||||
private long shopMatchStaleTimeoutMinutes = 20;
|
private long shopMatchStaleTimeoutMinutes = 30;
|
||||||
private long shopMatchInitialTimeoutMinutes = 20;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Patrol delete RUNNING timeout auto-finalize/fail threshold in minutes.
|
* Patrol delete RUNNING timeout auto-finalize/fail threshold in minutes.
|
||||||
*/
|
*/
|
||||||
private long patrolDeleteStaleTimeoutMinutes = 20;
|
private long patrolDeleteStaleTimeoutMinutes = 30;
|
||||||
private long patrolDeleteInitialTimeoutMinutes = 20;
|
|
||||||
|
/**
|
||||||
|
* Query ASIN RUNNING timeout auto-finalize/fail threshold in minutes.
|
||||||
|
*/
|
||||||
|
private long queryAsinStaleTimeoutMinutes = 30;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,17 @@
|
|||||||
|
package com.nanri.aiimage.config;
|
||||||
|
|
||||||
|
import lombok.Data;
|
||||||
|
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||||
|
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@ConfigurationProperties(prefix = "aiimage.instance-routing")
|
||||||
|
public class InstanceRoutingProperties {
|
||||||
|
|
||||||
|
private Map<String, String> routes = new LinkedHashMap<>();
|
||||||
|
private int connectTimeoutMillis = 3000;
|
||||||
|
private int readTimeoutMillis = 300000;
|
||||||
|
private int requestBodyCacheLimitBytes = 104857600;
|
||||||
|
}
|
||||||
@@ -4,6 +4,6 @@ import org.springframework.boot.context.properties.EnableConfigurationProperties
|
|||||||
import org.springframework.context.annotation.Configuration;
|
import org.springframework.context.annotation.Configuration;
|
||||||
|
|
||||||
@Configuration
|
@Configuration
|
||||||
@EnableConfigurationProperties({OssProperties.class, TransientStorageProperties.class, StorageProperties.class, BrandProgressProperties.class, DeleteBrandProgressProperties.class, ZiniaoProperties.class, ModuleCleanupProperties.class, TaskPressureProperties.class, AppearancePatentProperties.class, SimilarAsinProperties.class})
|
@EnableConfigurationProperties({OssProperties.class, TransientStorageProperties.class, StorageProperties.class, BrandProgressProperties.class, DeleteBrandProgressProperties.class, BrandCheckProperties.class, ZiniaoProperties.class, ModuleCleanupProperties.class, TaskPressureProperties.class, TaskImageCacheCleanupProperties.class, AppearancePatentProperties.class, SimilarAsinProperties.class, InstanceRoutingProperties.class})
|
||||||
public class PropertiesConfig {
|
public class PropertiesConfig {
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,8 +6,10 @@ import jakarta.servlet.http.HttpServletRequest;
|
|||||||
import jakarta.servlet.http.HttpServletResponse;
|
import jakarta.servlet.http.HttpServletResponse;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
import org.springframework.web.filter.OncePerRequestFilter;
|
import org.springframework.web.filter.OncePerRequestFilter;
|
||||||
|
import org.springframework.web.util.ContentCachingRequestWrapper;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
@@ -18,27 +20,31 @@ public class RequestTraceFilter extends OncePerRequestFilter {
|
|||||||
private static final Logger log = LoggerFactory.getLogger(RequestTraceFilter.class);
|
private static final Logger log = LoggerFactory.getLogger(RequestTraceFilter.class);
|
||||||
|
|
||||||
private final InstanceMetadata instanceMetadata;
|
private final InstanceMetadata instanceMetadata;
|
||||||
|
private final int requestBodyCacheLimitBytes;
|
||||||
|
|
||||||
public RequestTraceFilter(InstanceMetadata instanceMetadata) {
|
public RequestTraceFilter(InstanceMetadata instanceMetadata,
|
||||||
|
@Value("${aiimage.instance-routing.request-body-cache-limit-bytes:104857600}") int requestBodyCacheLimitBytes) {
|
||||||
this.instanceMetadata = instanceMetadata;
|
this.instanceMetadata = instanceMetadata;
|
||||||
|
this.requestBodyCacheLimitBytes = Math.max(1024 * 1024, requestBodyCacheLimitBytes);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain filterChain)
|
protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain filterChain)
|
||||||
throws ServletException, IOException {
|
throws ServletException, IOException {
|
||||||
long start = System.currentTimeMillis();
|
long start = System.currentTimeMillis();
|
||||||
|
HttpServletRequest requestToUse = wrapRequestIfNeeded(request, requestBodyCacheLimitBytes);
|
||||||
String remoteAddr = firstNonBlank(
|
String remoteAddr = firstNonBlank(
|
||||||
request.getHeader("X-Forwarded-For"),
|
requestToUse.getHeader("X-Forwarded-For"),
|
||||||
request.getHeader("X-Real-IP"),
|
requestToUse.getHeader("X-Real-IP"),
|
||||||
request.getRemoteAddr()
|
requestToUse.getRemoteAddr()
|
||||||
);
|
);
|
||||||
String forwardedProto = request.getHeader("X-Forwarded-Proto");
|
String forwardedProto = requestToUse.getHeader("X-Forwarded-Proto");
|
||||||
String forwardedHost = firstNonBlank(request.getHeader("X-Forwarded-Host"), request.getHeader("Host"));
|
String forwardedHost = firstNonBlank(requestToUse.getHeader("X-Forwarded-Host"), requestToUse.getHeader("Host"));
|
||||||
String forwardedPort = request.getHeader("X-Forwarded-Port");
|
String forwardedPort = requestToUse.getHeader("X-Forwarded-Port");
|
||||||
String requestId = firstNonBlank(
|
String requestId = firstNonBlank(
|
||||||
request.getHeader("X-Request-Id"),
|
requestToUse.getHeader("X-Request-Id"),
|
||||||
request.getHeader("X-Amzn-Trace-Id"),
|
requestToUse.getHeader("X-Amzn-Trace-Id"),
|
||||||
request.getHeader("Traceparent")
|
requestToUse.getHeader("Traceparent")
|
||||||
);
|
);
|
||||||
|
|
||||||
response.setHeader("X-AIIMAGE-Instance", instanceMetadata.getInstanceId());
|
response.setHeader("X-AIIMAGE-Instance", instanceMetadata.getInstanceId());
|
||||||
@@ -47,7 +53,7 @@ public class RequestTraceFilter extends OncePerRequestFilter {
|
|||||||
response.setHeader("X-AIIMAGE-Instance-Stable", String.valueOf(instanceMetadata.isStable()));
|
response.setHeader("X-AIIMAGE-Instance-Stable", String.valueOf(instanceMetadata.isStable()));
|
||||||
|
|
||||||
try {
|
try {
|
||||||
filterChain.doFilter(request, response);
|
filterChain.doFilter(requestToUse, response);
|
||||||
} finally {
|
} finally {
|
||||||
long costMs = System.currentTimeMillis() - start;
|
long costMs = System.currentTimeMillis() - start;
|
||||||
log.info(
|
log.info(
|
||||||
@@ -56,20 +62,34 @@ public class RequestTraceFilter extends OncePerRequestFilter {
|
|||||||
instanceMetadata.getSource(),
|
instanceMetadata.getSource(),
|
||||||
instanceMetadata.isStable(),
|
instanceMetadata.isStable(),
|
||||||
instanceMetadata.getHostname(),
|
instanceMetadata.getHostname(),
|
||||||
request.getMethod(),
|
requestToUse.getMethod(),
|
||||||
request.getRequestURI(),
|
requestToUse.getRequestURI(),
|
||||||
response.getStatus(),
|
response.getStatus(),
|
||||||
remoteAddr,
|
remoteAddr,
|
||||||
blankToDash(forwardedHost),
|
blankToDash(forwardedHost),
|
||||||
blankToDash(forwardedProto),
|
blankToDash(forwardedProto),
|
||||||
blankToDash(forwardedPort),
|
blankToDash(forwardedPort),
|
||||||
blankToDash(requestId),
|
blankToDash(requestId),
|
||||||
blankToDash(request.getHeader("User-Agent")),
|
blankToDash(requestToUse.getHeader("User-Agent")),
|
||||||
costMs
|
costMs
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static HttpServletRequest wrapRequestIfNeeded(HttpServletRequest request, int requestBodyCacheLimitBytes) {
|
||||||
|
if (request instanceof ContentCachingRequestWrapper) {
|
||||||
|
return request;
|
||||||
|
}
|
||||||
|
String method = request.getMethod();
|
||||||
|
if ("POST".equalsIgnoreCase(method)
|
||||||
|
|| "PUT".equalsIgnoreCase(method)
|
||||||
|
|| "PATCH".equalsIgnoreCase(method)
|
||||||
|
|| "DELETE".equalsIgnoreCase(method)) {
|
||||||
|
return new ContentCachingRequestWrapper(request, requestBodyCacheLimitBytes);
|
||||||
|
}
|
||||||
|
return request;
|
||||||
|
}
|
||||||
|
|
||||||
private static String firstNonBlank(String... values) {
|
private static String firstNonBlank(String... values) {
|
||||||
for (String value : values) {
|
for (String value : values) {
|
||||||
if (value != null && !value.isBlank()) {
|
if (value != null && !value.isBlank()) {
|
||||||
|
|||||||
@@ -0,0 +1,15 @@
|
|||||||
|
package com.nanri.aiimage.config;
|
||||||
|
|
||||||
|
import lombok.Data;
|
||||||
|
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@ConfigurationProperties(prefix = "aiimage.task-image-cache-cleanup")
|
||||||
|
public class TaskImageCacheCleanupProperties {
|
||||||
|
|
||||||
|
private boolean enabled = true;
|
||||||
|
private String cron = "0 30 2 * * *";
|
||||||
|
private int retentionDays = 3;
|
||||||
|
private int batchSize = 5000;
|
||||||
|
private int maxBatchesPerRun = 200;
|
||||||
|
}
|
||||||
@@ -20,6 +20,8 @@ import java.util.regex.Pattern;
|
|||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
public class TaskOperationLockConfig implements WebMvcConfigurer {
|
public class TaskOperationLockConfig implements WebMvcConfigurer {
|
||||||
|
|
||||||
|
private static final String LOCK_ATTRIBUTE = TaskOperationLockConfig.class.getName() + ".LOCK";
|
||||||
|
|
||||||
private final TaskDistributedLockService taskDistributedLockService;
|
private final TaskDistributedLockService taskDistributedLockService;
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -28,6 +30,19 @@ public class TaskOperationLockConfig implements WebMvcConfigurer {
|
|||||||
.addPathPatterns("/api/**");
|
.addPathPatterns("/api/**");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static boolean releaseRequestLock(HttpServletRequest request) {
|
||||||
|
if (request == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
Object lockHandle = request.getAttribute(LOCK_ATTRIBUTE);
|
||||||
|
if (lockHandle instanceof TaskDistributedLockService.LockHandle handle) {
|
||||||
|
request.removeAttribute(LOCK_ATTRIBUTE);
|
||||||
|
handle.close();
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
@Slf4j
|
@Slf4j
|
||||||
private static final class TaskOperationLockInterceptor implements HandlerInterceptor {
|
private static final class TaskOperationLockInterceptor implements HandlerInterceptor {
|
||||||
|
|
||||||
@@ -36,7 +51,6 @@ public class TaskOperationLockConfig implements WebMvcConfigurer {
|
|||||||
private static final Set<String> MUTATING_METHODS = Set.of("POST", "PUT", "PATCH", "DELETE");
|
private static final Set<String> MUTATING_METHODS = Set.of("POST", "PUT", "PATCH", "DELETE");
|
||||||
private static final long RESULT_SUBMIT_WAIT_MILLIS = 30 * 1000L;
|
private static final long RESULT_SUBMIT_WAIT_MILLIS = 30 * 1000L;
|
||||||
private static final long DELETE_WAIT_MILLIS = 60 * 1000L;
|
private static final long DELETE_WAIT_MILLIS = 60 * 1000L;
|
||||||
private static final String LOCK_ATTRIBUTE = TaskOperationLockInterceptor.class.getName() + ".LOCK";
|
|
||||||
|
|
||||||
private final TaskDistributedLockService taskDistributedLockService;
|
private final TaskDistributedLockService taskDistributedLockService;
|
||||||
|
|
||||||
@@ -83,10 +97,7 @@ public class TaskOperationLockConfig implements WebMvcConfigurer {
|
|||||||
HttpServletResponse response,
|
HttpServletResponse response,
|
||||||
Object handler,
|
Object handler,
|
||||||
Exception ex) {
|
Exception ex) {
|
||||||
Object lockHandle = request.getAttribute(LOCK_ATTRIBUTE);
|
releaseRequestLock(request);
|
||||||
if (lockHandle instanceof TaskDistributedLockService.LockHandle handle) {
|
|
||||||
handle.close();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.JsonNode;
|
|||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
import com.nanri.aiimage.config.AppearancePatentProperties;
|
import com.nanri.aiimage.config.AppearancePatentProperties;
|
||||||
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultRowDto;
|
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultRowDto;
|
||||||
|
import com.nanri.aiimage.modules.brand.client.BrandCheckClient;
|
||||||
import com.nanri.aiimage.modules.coze.service.CozeCredentialPoolService;
|
import com.nanri.aiimage.modules.coze.service.CozeCredentialPoolService;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
@@ -29,13 +30,21 @@ public class AppearancePatentCozeClient {
|
|||||||
|
|
||||||
private static final String MODULE_TYPE = "APPEARANCE_PATENT";
|
private static final String MODULE_TYPE = "APPEARANCE_PATENT";
|
||||||
private static final MediaType APPLICATION_JSON_UTF8 = new MediaType("application", "json", StandardCharsets.UTF_8);
|
private static final MediaType APPLICATION_JSON_UTF8 = new MediaType("application", "json", StandardCharsets.UTF_8);
|
||||||
|
private static final String INFRINGEMENT = "侵权";
|
||||||
|
private static final String NO_INFRINGEMENT = "无侵权";
|
||||||
|
private static final String BRAND_QUERY_FAILED = "商标查询失败";
|
||||||
|
|
||||||
private final AppearancePatentProperties properties;
|
private final AppearancePatentProperties properties;
|
||||||
private final ObjectMapper objectMapper;
|
private final ObjectMapper objectMapper;
|
||||||
private final CozeCredentialPoolService cozeCredentialPoolService;
|
private final CozeCredentialPoolService cozeCredentialPoolService;
|
||||||
|
private final BrandCheckClient brandCheckClient;
|
||||||
private final AtomicLong credentialCursor = new AtomicLong();
|
private final AtomicLong credentialCursor = new AtomicLong();
|
||||||
|
|
||||||
public List<AppearancePatentResultRowDto> inspect(List<AppearancePatentResultRowDto> rows, String prompt, String apiKey) {
|
public List<AppearancePatentResultRowDto> inspect(List<AppearancePatentResultRowDto> rows, String prompt, String apiKey) {
|
||||||
|
return inspect(rows, prompt, apiKey, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<AppearancePatentResultRowDto> inspect(List<AppearancePatentResultRowDto> rows, String prompt, String apiKey, String patentToken) {
|
||||||
if (rows == null || rows.isEmpty()) {
|
if (rows == null || rows.isEmpty()) {
|
||||||
return List.of();
|
return List.of();
|
||||||
}
|
}
|
||||||
@@ -44,7 +53,7 @@ public class AppearancePatentCozeClient {
|
|||||||
return rows.stream().map(this::copy).toList();
|
return rows.stream().map(this::copy).toList();
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
return inspectWithFallback(rows, prompt, apiKey);
|
return inspectWithFallback(rows, prompt, apiKey, patentToken);
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
String failureMessage = failureMessage(ex);
|
String failureMessage = failureMessage(ex);
|
||||||
log.warn("[appearance-patent] coze batch failed size={} err={}", rows.size(), failureMessage);
|
log.warn("[appearance-patent] coze batch failed size={} err={}", rows.size(), failureMessage);
|
||||||
@@ -53,15 +62,16 @@ public class AppearancePatentCozeClient {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public CozeSubmitResponse submitWorkflow(List<AppearancePatentResultRowDto> rows, String prompt, String apiKey) throws Exception {
|
public CozeSubmitResponse submitWorkflow(List<AppearancePatentResultRowDto> rows, String prompt, String apiKey) throws Exception {
|
||||||
return submitWorkflow(rows, prompt, apiKey, nextCredential());
|
return submitWorkflow(rows, prompt, apiKey, null, nextCredential());
|
||||||
}
|
}
|
||||||
|
|
||||||
public CozeSubmitResponse submitWorkflow(List<AppearancePatentResultRowDto> rows,
|
public CozeSubmitResponse submitWorkflow(List<AppearancePatentResultRowDto> rows,
|
||||||
String prompt,
|
String prompt,
|
||||||
String apiKey,
|
String apiKey,
|
||||||
|
String patentToken,
|
||||||
CozeCredentialRef credential) throws Exception {
|
CozeCredentialRef credential) throws Exception {
|
||||||
CozeCredentialRef resolvedCredential = resolveCredential(credential);
|
CozeCredentialRef resolvedCredential = resolveCredential(credential);
|
||||||
JsonNode submitRoot = objectMapper.readTree(postWorkflow(rows, prompt, apiKey, resolvedCredential));
|
JsonNode submitRoot = objectMapper.readTree(postWorkflow(rows, prompt, apiKey, patentToken, resolvedCredential));
|
||||||
ensureSuccess(submitRoot);
|
ensureSuccess(submitRoot);
|
||||||
return new CozeSubmitResponse(
|
return new CozeSubmitResponse(
|
||||||
extractExecuteId(submitRoot),
|
extractExecuteId(submitRoot),
|
||||||
@@ -103,12 +113,12 @@ public class AppearancePatentCozeClient {
|
|||||||
return rows.stream().map(this::copy).map(row -> markFailed(row, failureMessage)).toList();
|
return rows.stream().map(this::copy).map(row -> markFailed(row, failureMessage)).toList();
|
||||||
}
|
}
|
||||||
|
|
||||||
private List<AppearancePatentResultRowDto> inspectWithFallback(List<AppearancePatentResultRowDto> rows, String prompt, String apiKey) {
|
private List<AppearancePatentResultRowDto> inspectWithFallback(List<AppearancePatentResultRowDto> rows, String prompt, String apiKey, String patentToken) {
|
||||||
try {
|
try {
|
||||||
if (rows.size() == 1) {
|
if (rows.size() == 1) {
|
||||||
return inspectSingleRowWithRetry(rows, prompt, apiKey);
|
return inspectSingleRowWithRetry(rows, prompt, apiKey, patentToken);
|
||||||
}
|
}
|
||||||
InspectAttempt attempt = inspectOnce(rows, prompt, apiKey);
|
InspectAttempt attempt = inspectOnce(rows, prompt, apiKey, patentToken);
|
||||||
if (attempt.resolvedCount() < rows.size()) {
|
if (attempt.resolvedCount() < rows.size()) {
|
||||||
throw new PartialCozeResultException(attempt.resolvedCount(), rows.size(), attempt.rawResultCount());
|
throw new PartialCozeResultException(attempt.resolvedCount(), rows.size(), attempt.rawResultCount());
|
||||||
}
|
}
|
||||||
@@ -119,17 +129,17 @@ public class AppearancePatentCozeClient {
|
|||||||
log.warn("[appearance-patent] coze batch fallback split size={} left={} right={} err={}",
|
log.warn("[appearance-patent] coze batch fallback split size={} left={} right={} err={}",
|
||||||
rows.size(), middle, rows.size() - middle, failureMessage(ex));
|
rows.size(), middle, rows.size() - middle, failureMessage(ex));
|
||||||
List<AppearancePatentResultRowDto> merged = new ArrayList<>(rows.size());
|
List<AppearancePatentResultRowDto> merged = new ArrayList<>(rows.size());
|
||||||
merged.addAll(inspectPartitionWithFailureFallback(rows.subList(0, middle), prompt, apiKey));
|
merged.addAll(inspectPartitionWithFailureFallback(rows.subList(0, middle), prompt, apiKey, patentToken));
|
||||||
merged.addAll(inspectPartitionWithFailureFallback(rows.subList(middle, rows.size()), prompt, apiKey));
|
merged.addAll(inspectPartitionWithFailureFallback(rows.subList(middle, rows.size()), prompt, apiKey, patentToken));
|
||||||
return merged;
|
return merged;
|
||||||
}
|
}
|
||||||
throw propagate(ex);
|
throw propagate(ex);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private List<AppearancePatentResultRowDto> inspectPartitionWithFailureFallback(List<AppearancePatentResultRowDto> rows, String prompt, String apiKey) {
|
private List<AppearancePatentResultRowDto> inspectPartitionWithFailureFallback(List<AppearancePatentResultRowDto> rows, String prompt, String apiKey, String patentToken) {
|
||||||
try {
|
try {
|
||||||
return inspectWithFallback(rows, prompt, apiKey);
|
return inspectWithFallback(rows, prompt, apiKey, patentToken);
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
String failureMessage = failureMessage(ex);
|
String failureMessage = failureMessage(ex);
|
||||||
log.warn("[appearance-patent] coze partition failed size={} err={}", rows.size(), failureMessage);
|
log.warn("[appearance-patent] coze partition failed size={} err={}", rows.size(), failureMessage);
|
||||||
@@ -137,12 +147,12 @@ public class AppearancePatentCozeClient {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private List<AppearancePatentResultRowDto> inspectSingleRowWithRetry(List<AppearancePatentResultRowDto> rows, String prompt, String apiKey) throws Exception {
|
private List<AppearancePatentResultRowDto> inspectSingleRowWithRetry(List<AppearancePatentResultRowDto> rows, String prompt, String apiKey, String patentToken) throws Exception {
|
||||||
AppearancePatentResultRowDto row = rows.getFirst();
|
AppearancePatentResultRowDto row = rows.getFirst();
|
||||||
PartialCozeResultException lastFailure = null;
|
PartialCozeResultException lastFailure = null;
|
||||||
for (int attemptIndex = 1; attemptIndex <= 3; attemptIndex++) {
|
for (int attemptIndex = 1; attemptIndex <= 3; attemptIndex++) {
|
||||||
try {
|
try {
|
||||||
InspectAttempt attempt = inspectOnce(rows, prompt, apiKey);
|
InspectAttempt attempt = inspectOnce(rows, prompt, apiKey, patentToken);
|
||||||
if (attempt.resolvedCount() == rows.size()) {
|
if (attempt.resolvedCount() == rows.size()) {
|
||||||
return attempt.mergedRows();
|
return attempt.mergedRows();
|
||||||
}
|
}
|
||||||
@@ -173,8 +183,8 @@ public class AppearancePatentCozeClient {
|
|||||||
throw lastFailure == null ? new PartialCozeResultException(0, rows.size(), 0) : lastFailure;
|
throw lastFailure == null ? new PartialCozeResultException(0, rows.size(), 0) : lastFailure;
|
||||||
}
|
}
|
||||||
|
|
||||||
private InspectAttempt inspectOnce(List<AppearancePatentResultRowDto> rows, String prompt, String apiKey) throws Exception {
|
private InspectAttempt inspectOnce(List<AppearancePatentResultRowDto> rows, String prompt, String apiKey, String patentToken) throws Exception {
|
||||||
String raw = runWorkflowAsyncAndWait(rows, prompt, apiKey);
|
String raw = runWorkflowAsyncAndWait(rows, prompt, apiKey, patentToken);
|
||||||
List<CozeResult> results = parseResults(raw);
|
List<CozeResult> results = parseResults(raw);
|
||||||
if (rows.size() > 1 && !results.isEmpty() && results.stream().noneMatch(this::hasIdentity)) {
|
if (rows.size() > 1 && !results.isEmpty() && results.stream().noneMatch(this::hasIdentity)) {
|
||||||
throw new PartialCozeResultException(0, rows.size(), results.size());
|
throw new PartialCozeResultException(0, rows.size(), results.size());
|
||||||
@@ -183,9 +193,9 @@ public class AppearancePatentCozeClient {
|
|||||||
return new InspectAttempt(raw, merged, resolvedCount(merged), results.size());
|
return new InspectAttempt(raw, merged, resolvedCount(merged), results.size());
|
||||||
}
|
}
|
||||||
|
|
||||||
private String runWorkflowAsyncAndWait(List<AppearancePatentResultRowDto> rows, String prompt, String apiKey) throws Exception {
|
private String runWorkflowAsyncAndWait(List<AppearancePatentResultRowDto> rows, String prompt, String apiKey, String patentToken) throws Exception {
|
||||||
CozeCredentialRef credential = nextCredential();
|
CozeCredentialRef credential = nextCredential();
|
||||||
JsonNode submitRoot = objectMapper.readTree(postWorkflow(rows, prompt, apiKey, credential));
|
JsonNode submitRoot = objectMapper.readTree(postWorkflow(rows, prompt, apiKey, patentToken, credential));
|
||||||
ensureSuccess(submitRoot);
|
ensureSuccess(submitRoot);
|
||||||
|
|
||||||
String immediateData = extractResultDataText(submitRoot);
|
String immediateData = extractResultDataText(submitRoot);
|
||||||
@@ -229,8 +239,9 @@ public class AppearancePatentCozeClient {
|
|||||||
private String postWorkflow(List<AppearancePatentResultRowDto> rows,
|
private String postWorkflow(List<AppearancePatentResultRowDto> rows,
|
||||||
String prompt,
|
String prompt,
|
||||||
String apiKey,
|
String apiKey,
|
||||||
|
String patentToken,
|
||||||
CozeCredentialRef credential) {
|
CozeCredentialRef credential) {
|
||||||
Map<String, Object> parameters = buildParameters(rows, prompt, apiKey);
|
Map<String, Object> parameters = buildParameters(rows, prompt, apiKey, patentToken);
|
||||||
Map<String, Object> body = new LinkedHashMap<>();
|
Map<String, Object> body = new LinkedHashMap<>();
|
||||||
body.put("workflow_id", credential.workflowId());
|
body.put("workflow_id", credential.workflowId());
|
||||||
body.put("parameters", parameters);
|
body.put("parameters", parameters);
|
||||||
@@ -350,12 +361,12 @@ public class AppearancePatentCozeClient {
|
|||||||
return credentials;
|
return credentials;
|
||||||
}
|
}
|
||||||
|
|
||||||
private Map<String, Object> buildParameters(List<AppearancePatentResultRowDto> rows, String prompt, String apiKey) {
|
private Map<String, Object> buildParameters(List<AppearancePatentResultRowDto> rows, String prompt, String apiKey, String patentToken) {
|
||||||
List<String> groupKeys = rows.stream().map(row -> nonBlank(row.getGroupKey(), rowKey(row))).toList();
|
List<String> groupKeys = rows.stream().map(row -> nonBlank(row.getGroupKey(), rowKey(row))).toList();
|
||||||
List<String> rowIds = rows.stream().map(row -> nonBlank(row.getId(), "")).toList();
|
List<String> rowIds = rows.stream().map(row -> nonBlank(row.getId(), "")).toList();
|
||||||
List<String> asins = rows.stream().map(row -> nonBlank(row.getAsin(), "")).toList();
|
List<String> asins = rows.stream().map(row -> nonBlank(row.getAsin(), "")).toList();
|
||||||
List<String> countries = rows.stream().map(row -> nonBlank(row.getCountry(), "")).toList();
|
List<String> countries = rows.stream().map(row -> nonBlank(row.getCountry(), "")).toList();
|
||||||
List<String> skus = rows.stream().map(row -> nonBlank(row.getSku(), "")).toList();
|
List<String> skus = rows.stream().map(row -> sanitizeSku(row.getSku())).toList();
|
||||||
List<String> titles = rows.stream().map(row -> nonBlank(row.getTitle(), row.getAsin())).toList();
|
List<String> titles = rows.stream().map(row -> nonBlank(row.getTitle(), row.getAsin())).toList();
|
||||||
List<String> urls = rows.stream().map(row -> nonBlank(row.getUrl(), "")).toList();
|
List<String> urls = rows.stream().map(row -> nonBlank(row.getUrl(), "")).toList();
|
||||||
|
|
||||||
@@ -370,6 +381,9 @@ public class AppearancePatentCozeClient {
|
|||||||
if (apiKey != null && !apiKey.isBlank()) {
|
if (apiKey != null && !apiKey.isBlank()) {
|
||||||
parameters.put("api_key", apiKey.trim());
|
parameters.put("api_key", apiKey.trim());
|
||||||
}
|
}
|
||||||
|
if (patentToken != null && !patentToken.isBlank()) {
|
||||||
|
parameters.put("patent_token", patentToken.trim());
|
||||||
|
}
|
||||||
return parameters;
|
return parameters;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -383,6 +397,10 @@ public class AppearancePatentCozeClient {
|
|||||||
if (apiKey instanceof String apiKeyText && !apiKeyText.isBlank()) {
|
if (apiKey instanceof String apiKeyText && !apiKeyText.isBlank()) {
|
||||||
maskedParameters.put("api_key", maskSecret(apiKeyText));
|
maskedParameters.put("api_key", maskSecret(apiKeyText));
|
||||||
}
|
}
|
||||||
|
Object patentToken = maskedParameters.get("patent_token");
|
||||||
|
if (patentToken instanceof String patentTokenText && !patentTokenText.isBlank()) {
|
||||||
|
maskedParameters.put("patent_token", maskSecret(patentTokenText));
|
||||||
|
}
|
||||||
masked.put("parameters", maskedParameters);
|
masked.put("parameters", maskedParameters);
|
||||||
}
|
}
|
||||||
return masked;
|
return masked;
|
||||||
@@ -567,10 +585,15 @@ public class AppearancePatentCozeClient {
|
|||||||
if (row == null || result == null) {
|
if (row == null || result == null) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
row.setTitleRisk(result.title());
|
if (isNoTitleRisk(result)) {
|
||||||
|
row.setTitleRisk(NO_INFRINGEMENT);
|
||||||
|
} else {
|
||||||
|
BrandCheckClient.BrandCheckBatchResult brandCheck = brandCheckClient.checkTitleText(result.title(), "Terms");
|
||||||
|
row.setTitleRisk(buildTitleRisk(result.title(), brandCheck));
|
||||||
|
}
|
||||||
row.setAppearanceRisk(result.appearance());
|
row.setAppearanceRisk(result.appearance());
|
||||||
row.setPatentRisk(result.patent());
|
row.setPatentRisk(result.patent());
|
||||||
row.setConclusion(result.result());
|
row.setConclusion(buildConclusion(row.getTitleRisk(), row.getAppearanceRisk(), result.result()));
|
||||||
row.setStatus(result.status());
|
row.setStatus(result.status());
|
||||||
row.setTitleReason(result.titleReason());
|
row.setTitleReason(result.titleReason());
|
||||||
row.setAppearanceReason(result.appearanceReason());
|
row.setAppearanceReason(result.appearanceReason());
|
||||||
@@ -578,6 +601,55 @@ public class AppearancePatentCozeClient {
|
|||||||
row.setScore(result.score());
|
row.setScore(result.score());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private boolean isNoTitleRisk(CozeResult result) {
|
||||||
|
return result != null
|
||||||
|
&& "无".equals(normalize(result.title()))
|
||||||
|
&& "无".equals(normalize(result.titleReason()));
|
||||||
|
}
|
||||||
|
|
||||||
|
String buildTitleRisk(String cozeTitle, BrandCheckClient.BrandCheckBatchResult brandCheck) {
|
||||||
|
List<String> brands = brandCheck == null ? List.of() : brandCheck.brands();
|
||||||
|
if (brands == null || brands.isEmpty()) {
|
||||||
|
return firstNonBlank(cozeTitle, "");
|
||||||
|
}
|
||||||
|
if (brandCheck.hasFailedData()) {
|
||||||
|
return INFRINGEMENT;
|
||||||
|
}
|
||||||
|
if (brandCheck.hasQueryFailedData()) {
|
||||||
|
return BRAND_QUERY_FAILED;
|
||||||
|
}
|
||||||
|
return NO_INFRINGEMENT;
|
||||||
|
}
|
||||||
|
|
||||||
|
String buildConclusion(String titleRisk, String appearanceRisk, String fallbackConclusion) {
|
||||||
|
if (isInfringement(titleRisk) || isInfringement(appearanceRisk)) {
|
||||||
|
return INFRINGEMENT;
|
||||||
|
}
|
||||||
|
if (isNoInfringement(titleRisk) && isNoInfringement(appearanceRisk)) {
|
||||||
|
return NO_INFRINGEMENT;
|
||||||
|
}
|
||||||
|
if (isBrandQueryFailed(titleRisk)) {
|
||||||
|
return BRAND_QUERY_FAILED;
|
||||||
|
}
|
||||||
|
return firstNonBlank(fallbackConclusion, "");
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean isInfringement(String value) {
|
||||||
|
String normalized = normalize(value);
|
||||||
|
return normalized.contains("侵权")
|
||||||
|
&& !normalized.contains("无侵权")
|
||||||
|
&& !normalized.contains("未侵权");
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean isNoInfringement(String value) {
|
||||||
|
String normalized = normalize(value);
|
||||||
|
return normalized.contains("无侵权") || normalized.contains("未侵权");
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean isBrandQueryFailed(String value) {
|
||||||
|
return normalize(value).contains(BRAND_QUERY_FAILED);
|
||||||
|
}
|
||||||
|
|
||||||
private RestClient restClient() {
|
private RestClient restClient() {
|
||||||
SimpleClientHttpRequestFactory requestFactory = new SimpleClientHttpRequestFactory();
|
SimpleClientHttpRequestFactory requestFactory = new SimpleClientHttpRequestFactory();
|
||||||
requestFactory.setConnectTimeout(properties.getCozeConnectTimeoutMillis());
|
requestFactory.setConnectTimeout(properties.getCozeConnectTimeoutMillis());
|
||||||
@@ -999,6 +1071,32 @@ public class AppearancePatentCozeClient {
|
|||||||
return value == null || value.isBlank() ? fallback : value;
|
return value == null || value.isBlank() ? fallback : value;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private String sanitizeSku(String value) {
|
||||||
|
String normalized = normalize(value);
|
||||||
|
if (normalized.isBlank() || isProbablyDescriptionSku(normalized)) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean isProbablyDescriptionSku(String value) {
|
||||||
|
String normalized = normalize(value);
|
||||||
|
if (normalized.isBlank()) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
String lower = normalized.toLowerCase(Locale.ROOT);
|
||||||
|
return normalized.length() > 120
|
||||||
|
|| normalized.contains("\n")
|
||||||
|
|| lower.contains(" minimum tensile ")
|
||||||
|
|| lower.contains(" multipurpose:")
|
||||||
|
|| lower.contains(" strong, durable")
|
||||||
|
|| lower.contains(" product includes ")
|
||||||
|
|| lower.contains(" perfect gift")
|
||||||
|
|| lower.contains(" high quality ")
|
||||||
|
|| lower.contains(" easy to ")
|
||||||
|
|| lower.contains(" capacity:");
|
||||||
|
}
|
||||||
|
|
||||||
private String firstNonBlank(String preferred, String fallback) {
|
private String firstNonBlank(String preferred, String fallback) {
|
||||||
return preferred == null || preferred.isBlank() ? fallback : preferred.trim();
|
return preferred == null || preferred.isBlank() ? fallback : preferred.trim();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -32,4 +32,9 @@ public class AppearancePatentParseRequest {
|
|||||||
@Schema(description = "传递给 Coze workflow parameters.api_key 的任务级密钥。")
|
@Schema(description = "传递给 Coze workflow parameters.api_key 的任务级密钥。")
|
||||||
@NotBlank(message = "密钥不能为空")
|
@NotBlank(message = "密钥不能为空")
|
||||||
private String apiKey;
|
private String apiKey;
|
||||||
|
|
||||||
|
@JsonProperty("patent_token")
|
||||||
|
@JsonAlias({"patentToken"})
|
||||||
|
@Schema(description = "传递给 Coze workflow parameters.patent_token 的专利汇令牌。非必填。")
|
||||||
|
private String patentToken;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import java.util.List;
|
|||||||
public class AppearancePatentParsedGroupManifestDto {
|
public class AppearancePatentParsedGroupManifestDto {
|
||||||
private String aiPrompt;
|
private String aiPrompt;
|
||||||
private String apiKey;
|
private String apiKey;
|
||||||
|
private String patentToken;
|
||||||
private Integer pageSize;
|
private Integer pageSize;
|
||||||
private Integer totalGroups;
|
private Integer totalGroups;
|
||||||
private Integer totalRows;
|
private Integer totalRows;
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import java.util.List;
|
|||||||
public class AppearancePatentParsedGroupPageDto {
|
public class AppearancePatentParsedGroupPageDto {
|
||||||
private String aiPrompt;
|
private String aiPrompt;
|
||||||
private String apiKey;
|
private String apiKey;
|
||||||
|
private String patentToken;
|
||||||
private Integer page;
|
private Integer page;
|
||||||
private Integer pageSize;
|
private Integer pageSize;
|
||||||
private Integer totalGroups;
|
private Integer totalGroups;
|
||||||
|
|||||||
@@ -17,6 +17,9 @@ public class AppearancePatentParsedPayloadDto {
|
|||||||
@Schema(description = "传递给 Coze workflow parameters.api_key 的任务级密钥")
|
@Schema(description = "传递给 Coze workflow parameters.api_key 的任务级密钥")
|
||||||
private String apiKey;
|
private String apiKey;
|
||||||
|
|
||||||
|
@Schema(description = "传递给 Coze workflow parameters.patent_token 的专利汇令牌")
|
||||||
|
private String patentToken;
|
||||||
|
|
||||||
@Schema(description = "本次解析的源文件列表")
|
@Schema(description = "本次解析的源文件列表")
|
||||||
private List<AppearancePatentSourceFileDto> sourceFiles = new ArrayList<>();
|
private List<AppearancePatentSourceFileDto> sourceFiles = new ArrayList<>();
|
||||||
|
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
|||||||
import com.fasterxml.jackson.databind.JsonNode;
|
import com.fasterxml.jackson.databind.JsonNode;
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
import com.nanri.aiimage.common.exception.BusinessException;
|
import com.nanri.aiimage.common.exception.BusinessException;
|
||||||
|
import com.nanri.aiimage.common.exception.TaskOwnerMismatchException;
|
||||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||||
import com.nanri.aiimage.common.util.CozeGroupResultPropagator;
|
import com.nanri.aiimage.common.util.CozeGroupResultPropagator;
|
||||||
import com.nanri.aiimage.common.util.FailedStatusRowFilter;
|
import com.nanri.aiimage.common.util.FailedStatusRowFilter;
|
||||||
@@ -108,6 +109,7 @@ public class AppearancePatentTaskService {
|
|||||||
private static final String COZE_STATUS_FAILED = "FAILED";
|
private static final String COZE_STATUS_FAILED = "FAILED";
|
||||||
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
||||||
private static final String CONTENT_TYPE_ZIP = "application/zip";
|
private static final String CONTENT_TYPE_ZIP = "application/zip";
|
||||||
|
private static final int CHUNK_PAYLOAD_MERGE_RETRY_LIMIT = 3;
|
||||||
private static final int RESULT_ROWS_READ_RETRY_LIMIT = 3;
|
private static final int RESULT_ROWS_READ_RETRY_LIMIT = 3;
|
||||||
private static final long RESULT_ROWS_READ_RETRY_DELAY_MS = 500L;
|
private static final long RESULT_ROWS_READ_RETRY_DELAY_MS = 500L;
|
||||||
private static final Duration TASK_LOCK_TTL = Duration.ofMinutes(5);
|
private static final Duration TASK_LOCK_TTL = Duration.ofMinutes(5);
|
||||||
@@ -218,11 +220,11 @@ public class AppearancePatentTaskService {
|
|||||||
|
|
||||||
String aggregateScopeKey = buildAggregateScopeKey(sourceFiles);
|
String aggregateScopeKey = buildAggregateScopeKey(sourceFiles);
|
||||||
String sourceScopeHash = DigestUtil.sha256Hex(aggregateScopeKey);
|
String sourceScopeHash = DigestUtil.sha256Hex(aggregateScopeKey);
|
||||||
String parsedPayload = buildParsedPayloadJson(request.getAiPrompt(), request.getApiKey(), sourceFiles, mergedHeaders, allRows);
|
String parsedPayload = buildParsedPayloadJson(request.getAiPrompt(), request.getApiKey(), request.getPatentToken(), sourceFiles, mergedHeaders, allRows);
|
||||||
long payloadBuiltAt = System.nanoTime();
|
long payloadBuiltAt = System.nanoTime();
|
||||||
String parsedPayloadPointer = storeParsedPayload(task.getId(), sourceScopeHash, parsedPayload);
|
String parsedPayloadPointer = storeParsedPayload(task.getId(), sourceScopeHash, parsedPayload);
|
||||||
long payloadStoredAt = System.nanoTime();
|
long payloadStoredAt = System.nanoTime();
|
||||||
task.setResultJson(buildTaskResultJson(request.getAiPrompt(), request.getApiKey(), sourceFiles, parsedPayloadPointer));
|
task.setResultJson(buildTaskResultJson(request.getAiPrompt(), request.getApiKey(), request.getPatentToken(), sourceFiles, parsedPayloadPointer));
|
||||||
task.setUpdatedAt(LocalDateTime.now());
|
task.setUpdatedAt(LocalDateTime.now());
|
||||||
fileTaskMapper.updateById(task);
|
fileTaskMapper.updateById(task);
|
||||||
|
|
||||||
@@ -572,7 +574,8 @@ public class AppearancePatentTaskService {
|
|||||||
|
|
||||||
public void finalizeStaleTasks() {
|
public void finalizeStaleTasks() {
|
||||||
if (transactionManager != null) {
|
if (transactionManager != null) {
|
||||||
LocalDateTime threshold = LocalDateTime.now().minusMinutes(Math.max(1, properties.getStaleTimeoutMinutes()));
|
LocalDateTime now = LocalDateTime.now();
|
||||||
|
LocalDateTime threshold = now.minusMinutes(Math.max(1, properties.getStaleTimeoutMinutes()));
|
||||||
long thresholdMillis = threshold.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli();
|
long thresholdMillis = threshold.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli();
|
||||||
List<FileTaskEntity> tasks = listStaleFinalizeCandidates(threshold);
|
List<FileTaskEntity> tasks = listStaleFinalizeCandidates(threshold);
|
||||||
for (FileTaskEntity task : tasks) {
|
for (FileTaskEntity task : tasks) {
|
||||||
@@ -596,7 +599,8 @@ public class AppearancePatentTaskService {
|
|||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
LocalDateTime threshold = LocalDateTime.now().minusMinutes(Math.max(1, properties.getStaleTimeoutMinutes()));
|
LocalDateTime now = LocalDateTime.now();
|
||||||
|
LocalDateTime threshold = now.minusMinutes(Math.max(1, properties.getStaleTimeoutMinutes()));
|
||||||
long thresholdMillis = threshold.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli();
|
long thresholdMillis = threshold.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli();
|
||||||
List<FileTaskEntity> tasks = listStaleFinalizeCandidates(threshold);
|
List<FileTaskEntity> tasks = listStaleFinalizeCandidates(threshold);
|
||||||
for (FileTaskEntity task : tasks) {
|
for (FileTaskEntity task : tasks) {
|
||||||
@@ -647,8 +651,8 @@ public class AppearancePatentTaskService {
|
|||||||
return fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
return fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||||
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
|
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
|
||||||
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
|
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
|
||||||
.lt(FileTaskEntity::getCreatedAt, threshold)
|
.lt(FileTaskEntity::getUpdatedAt, threshold)
|
||||||
.orderByAsc(FileTaskEntity::getCreatedAt)
|
.orderByAsc(FileTaskEntity::getUpdatedAt)
|
||||||
.last("limit 200"));
|
.last("limit 200"));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -811,22 +815,6 @@ public class AppearancePatentTaskService {
|
|||||||
long activeAssembleJobs = taskFileJobService.countActiveAssembleJobs(taskId, MODULE_TYPE);
|
long activeAssembleJobs = taskFileJobService.countActiveAssembleJobs(taskId, MODULE_TYPE);
|
||||||
log.info("[appearance-patent] stale recovery probe taskId={} uploadComplete={} pendingCozeStates={} activeAssembleJobs={} persistedRows={}",
|
log.info("[appearance-patent] stale recovery probe taskId={} uploadComplete={} pendingCozeStates={} activeAssembleJobs={} persistedRows={}",
|
||||||
taskId, uploadComplete, pendingCozeStates, activeAssembleJobs, hasPersistedResultRows(taskId));
|
taskId, uploadComplete, pendingCozeStates, activeAssembleJobs, hasPersistedResultRows(taskId));
|
||||||
if (isJavaSideProcessing(taskId)) {
|
|
||||||
// 防止 Coze 永远 pending 时 stale-recovery 永久 defer:
|
|
||||||
// 超过 stale-timeout-minutes × 4 仍未推进的 RUNNING 任务,强制走 finalize 链路。
|
|
||||||
long deferCeilingMinutes = Math.max(1, properties.getStaleTimeoutMinutes()) * 4L;
|
|
||||||
LocalDateTime updatedAt = task.getUpdatedAt();
|
|
||||||
if (updatedAt != null
|
|
||||||
&& Duration.between(updatedAt, LocalDateTime.now()).toMinutes() >= deferCeilingMinutes) {
|
|
||||||
log.warn("[appearance-patent] stale recovery defer ceiling exceeded, forcing finalize taskId={} updatedAt={} ceilingMinutes={} pendingCozeStates={} activeAssembleJobs={}",
|
|
||||||
taskId, updatedAt, deferCeilingMinutes, pendingCozeStates, activeAssembleJobs);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
touchJavaSideTaskActivity(taskId);
|
|
||||||
log.info("[appearance-patent] stale recovery deferred because Java-side processing is still active taskId={} uploadComplete={} pendingCozeStates={} activeAssembleJobs={}",
|
|
||||||
taskId, uploadComplete, pendingCozeStates, activeAssembleJobs);
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
if (!hasPersistedResultRows(taskId)) {
|
if (!hasPersistedResultRows(taskId)) {
|
||||||
log.info("[appearance-patent] stale recovery aborted because no persisted rows taskId={}", taskId);
|
log.info("[appearance-patent] stale recovery aborted because no persisted rows taskId={}", taskId);
|
||||||
return false;
|
return false;
|
||||||
@@ -839,10 +827,10 @@ public class AppearancePatentTaskService {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
int pendingRows = collectPendingCozeCandidates(task, loadSubmittedChunks(taskId)).size();
|
int pendingRows = collectPendingCozeCandidates(task, loadSubmittedChunks(taskId)).size();
|
||||||
log.warn("[appearance-patent] python heartbeat timed out, forcing coze flush taskId={} pendingRows={} pendingCozeStates={} activeAssembleJobs={} forcedScopes={}",
|
log.warn("[appearance-patent] Python 回传心跳超时,已封口上传并继续 Coze/文件收尾 taskId={} pendingRows={} pendingCozeStates={} activeAssembleJobs={} forcedScopes={}",
|
||||||
taskId, pendingRows, pendingCozeStates, activeAssembleJobs, forcedScopes);
|
taskId, pendingRows, pendingCozeStates, activeAssembleJobs, forcedScopes);
|
||||||
} else {
|
} else {
|
||||||
log.warn("[appearance-patent] stale running task resuming coze/file assembly after python timeout taskId={} pendingCozeStates={} activeAssembleJobs={}",
|
log.warn("[appearance-patent] Python 超时恢复继续推进 Coze/文件收尾 taskId={} pendingCozeStates={} activeAssembleJobs={}",
|
||||||
taskId, pendingCozeStates, activeAssembleJobs);
|
taskId, pendingCozeStates, activeAssembleJobs);
|
||||||
}
|
}
|
||||||
submitCozeForSubmittedChunk(new SubmitContext(task, null, null, 0, true, null));
|
submitCozeForSubmittedChunk(new SubmitContext(task, null, null, 0, true, null));
|
||||||
@@ -984,10 +972,11 @@ public class AppearancePatentTaskService {
|
|||||||
}
|
}
|
||||||
String prompt = readAiPrompt(task);
|
String prompt = readAiPrompt(task);
|
||||||
String apiKey = readApiKey(task);
|
String apiKey = readApiKey(task);
|
||||||
|
String patentToken = readPatentToken(task);
|
||||||
int batchSize = Math.max(1, properties.getCozeBatchSize());
|
int batchSize = Math.max(1, properties.getCozeBatchSize());
|
||||||
List<AppearancePatentResultRowDto> result = new ArrayList<>();
|
List<AppearancePatentResultRowDto> result = new ArrayList<>();
|
||||||
for (int i = 0; i < items.size(); i += batchSize) {
|
for (int i = 0; i < items.size(); i += batchSize) {
|
||||||
result.addAll(cozeClient.inspect(items.subList(i, Math.min(i + batchSize, items.size())), prompt, apiKey));
|
result.addAll(cozeClient.inspect(items.subList(i, Math.min(i + batchSize, items.size())), prompt, apiKey, patentToken));
|
||||||
if (progressHook != null) {
|
if (progressHook != null) {
|
||||||
progressHook.run();
|
progressHook.run();
|
||||||
}
|
}
|
||||||
@@ -1028,32 +1017,52 @@ public class AppearancePatentTaskService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private void mergeChunkPayload(Long taskId, String scopeHash, Integer chunkIndex, List<AppearancePatentResultRowDto> rows) {
|
private void mergeChunkPayload(Long taskId, String scopeHash, Integer chunkIndex, List<AppearancePatentResultRowDto> rows) {
|
||||||
TaskChunkEntity chunk = taskChunkMapper.selectOne(new LambdaQueryWrapper<TaskChunkEntity>()
|
if (rows == null || rows.isEmpty()) {
|
||||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
|
||||||
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
|
|
||||||
.eq(TaskChunkEntity::getScopeHash, scopeHash)
|
|
||||||
.eq(TaskChunkEntity::getChunkIndex, chunkIndex)
|
|
||||||
.last("limit 1"));
|
|
||||||
if (chunk == null || rows == null || rows.isEmpty()) {
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
Map<String, AppearancePatentResultRowDto> persistedRows = readChunkRows(chunk);
|
for (int attempt = 1; attempt <= CHUNK_PAYLOAD_MERGE_RETRY_LIMIT; attempt++) {
|
||||||
for (AppearancePatentResultRowDto row : rows) {
|
TaskChunkEntity chunk = taskChunkMapper.selectOne(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||||
persistedRows.put(rowKey(row), row);
|
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||||
}
|
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
|
||||||
String payloadJson = writeJson(rows == null ? List.of() : rows, "结果序列化失败");
|
.eq(TaskChunkEntity::getScopeHash, scopeHash)
|
||||||
payloadJson = writeJson(new ArrayList<>(persistedRows.values()), "appearance patent chunk payload merge failed");
|
.eq(TaskChunkEntity::getChunkIndex, chunkIndex)
|
||||||
String oldPayload = chunk.getPayloadJson();
|
.last("limit 1"));
|
||||||
String storedPayload = storeSharedChunkPayloadVersioned(taskId, scopeHash, chunkIndex, payloadJson);
|
if (chunk == null) {
|
||||||
chunk.setPayloadJson(storedPayload);
|
return;
|
||||||
chunk.setPayloadHash(DigestUtil.sha256Hex(payloadJson));
|
}
|
||||||
chunk.setUpdatedAt(LocalDateTime.now());
|
Map<String, AppearancePatentResultRowDto> persistedRows = readChunkRows(chunk);
|
||||||
int updated = taskChunkMapper.updateById(chunk);
|
for (AppearancePatentResultRowDto row : rows) {
|
||||||
if (updated <= 0) {
|
persistedRows.put(rowKey(row), row);
|
||||||
|
}
|
||||||
|
String payloadJson = writeJson(new ArrayList<>(persistedRows.values()), "appearance patent chunk payload merge failed");
|
||||||
|
String oldPayload = chunk.getPayloadJson();
|
||||||
|
String oldPayloadHash = chunk.getPayloadHash();
|
||||||
|
String newPayloadHash = DigestUtil.sha256Hex(payloadJson);
|
||||||
|
String storedPayload = storeSharedChunkPayloadVersioned(taskId, scopeHash, chunkIndex, payloadJson);
|
||||||
|
LambdaUpdateWrapper<TaskChunkEntity> updateWrapper = new LambdaUpdateWrapper<TaskChunkEntity>()
|
||||||
|
.eq(TaskChunkEntity::getId, chunk.getId())
|
||||||
|
.set(TaskChunkEntity::getPayloadJson, storedPayload)
|
||||||
|
.set(TaskChunkEntity::getPayloadHash, newPayloadHash)
|
||||||
|
.set(TaskChunkEntity::getUpdatedAt, LocalDateTime.now());
|
||||||
|
if (oldPayloadHash == null) {
|
||||||
|
updateWrapper.isNull(TaskChunkEntity::getPayloadHash);
|
||||||
|
} else {
|
||||||
|
updateWrapper.eq(TaskChunkEntity::getPayloadHash, oldPayloadHash);
|
||||||
|
}
|
||||||
|
int updated = taskChunkMapper.update(null, updateWrapper);
|
||||||
|
if (updated > 0) {
|
||||||
|
log.debug("[appearance-patent] chunk payload replaced taskId={} scopeHash={} chunk={} oldPayload={} newPayload={} attempt={}",
|
||||||
|
taskId, scopeHash, chunkIndex, oldPayload, storedPayload, attempt);
|
||||||
|
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(oldPayload, storedPayload);
|
||||||
|
return;
|
||||||
|
}
|
||||||
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
|
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
|
||||||
throw new IllegalStateException("appearance patent chunk payload update failed");
|
if (attempt < CHUNK_PAYLOAD_MERGE_RETRY_LIMIT) {
|
||||||
|
log.warn("[appearance-patent] chunk payload CAS conflict, retry merge taskId={} scopeHash={} chunk={} attempt={}/{}",
|
||||||
|
taskId, scopeHash, chunkIndex, attempt, CHUNK_PAYLOAD_MERGE_RETRY_LIMIT);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(oldPayload, storedPayload);
|
throw new IllegalStateException("appearance patent chunk payload update conflict");
|
||||||
}
|
}
|
||||||
|
|
||||||
private List<AppearancePatentResultRowDto> expandRows(List<AppearancePatentResultRowDto> representatives,
|
private List<AppearancePatentResultRowDto> expandRows(List<AppearancePatentResultRowDto> representatives,
|
||||||
@@ -1287,6 +1296,14 @@ public class AppearancePatentTaskService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private String readPatentToken(FileTaskEntity task) {
|
||||||
|
try {
|
||||||
|
return normalize(readParsedPayload(task).getPatentToken());
|
||||||
|
} catch (Exception ignored) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private void finalizeTask(FileTaskEntity task, String error, int rowCount, boolean assembleWorkbook) {
|
private void finalizeTask(FileTaskEntity task, String error, int rowCount, boolean assembleWorkbook) {
|
||||||
String finalError = error;
|
String finalError = error;
|
||||||
FileResultEntity result = null;
|
FileResultEntity result = null;
|
||||||
@@ -1429,6 +1446,9 @@ public class AppearancePatentTaskService {
|
|||||||
int cozeWorkUnits = countCozeWorkUnits(chunks, Math.max(1, properties.getCozeBatchSize()));
|
int cozeWorkUnits = countCozeWorkUnits(chunks, Math.max(1, properties.getCozeBatchSize()));
|
||||||
int plannedCozeUnits = Math.max(cozeWorkUnits, countAllCozeStates(task.getId()));
|
int plannedCozeUnits = Math.max(cozeWorkUnits, countAllCozeStates(task.getId()));
|
||||||
int totalProgressUnits = Math.max(3, plannedCozeUnits + 3);
|
int totalProgressUnits = Math.max(3, plannedCozeUnits + 3);
|
||||||
|
if (countPendingCozeStates(task.getId()) > 0) {
|
||||||
|
finalizeTimedOutCozeStatesForTask(task.getId());
|
||||||
|
}
|
||||||
if (countPendingCozeStates(task.getId()) > 0) {
|
if (countPendingCozeStates(task.getId()) > 0) {
|
||||||
taskFileJobService.touchRunning(job.getId());
|
taskFileJobService.touchRunning(job.getId());
|
||||||
touchJavaSideTaskActivity(task.getId());
|
touchJavaSideTaskActivity(task.getId());
|
||||||
@@ -1592,9 +1612,20 @@ public class AppearancePatentTaskService {
|
|||||||
}
|
}
|
||||||
String prompt = readAiPrompt(task);
|
String prompt = readAiPrompt(task);
|
||||||
String apiKey = readApiKey(task);
|
String apiKey = readApiKey(task);
|
||||||
|
String patentToken = readPatentToken(task);
|
||||||
int batchSize = Math.max(1, properties.getCozeBatchSize());
|
int batchSize = Math.max(1, properties.getCozeBatchSize());
|
||||||
List<CozeCandidate> candidates = collectPendingCozeCandidates(task, chunks);
|
List<CozeCandidate> candidates = collectPendingCozeCandidates(task, chunks);
|
||||||
boolean flushRemainder = isResultSubmissionComplete(task.getId());
|
boolean flushRemainder = isResultSubmissionComplete(task.getId());
|
||||||
|
if (!flushRemainder && !candidates.isEmpty()) {
|
||||||
|
LocalDateTime jobUpdatedAt = job.getUpdatedAt();
|
||||||
|
long pendingFlushMillis = cozeFlushPendingMillis();
|
||||||
|
if (jobUpdatedAt != null
|
||||||
|
&& Duration.between(jobUpdatedAt, LocalDateTime.now()).toMillis() >= pendingFlushMillis) {
|
||||||
|
flushRemainder = true;
|
||||||
|
log.warn("[appearance-patent] Coze 零头批次等待超时,强制提交 taskId={} jobId={} pendingRows={} batchSize={} jobUpdatedAt={} flushAfterMillis={}",
|
||||||
|
task.getId(), job.getId(), candidates.size(), batchSize, jobUpdatedAt, pendingFlushMillis);
|
||||||
|
}
|
||||||
|
}
|
||||||
int submitLimit = (candidates.size() / batchSize) * batchSize;
|
int submitLimit = (candidates.size() / batchSize) * batchSize;
|
||||||
if (flushRemainder && submitLimit < candidates.size()) {
|
if (flushRemainder && submitLimit < candidates.size()) {
|
||||||
submitLimit = candidates.size();
|
submitLimit = candidates.size();
|
||||||
@@ -1612,7 +1643,7 @@ public class AppearancePatentTaskService {
|
|||||||
List<AppearancePatentResultRowDto> batchRows = batchCandidates.stream()
|
List<AppearancePatentResultRowDto> batchRows = batchCandidates.stream()
|
||||||
.map(CozeCandidate::row)
|
.map(CozeCandidate::row)
|
||||||
.toList();
|
.toList();
|
||||||
pending |= submitCozeBatch(task, result, job, batchRows, batchIndex, batchTotal, prompt, apiKey, allRowsByBaseId);
|
pending |= submitCozeBatch(task, result, job, batchRows, batchIndex, batchTotal, prompt, apiKey, patentToken, allRowsByBaseId);
|
||||||
batchIndex++;
|
batchIndex++;
|
||||||
}
|
}
|
||||||
return pending || countPendingCozeStates(task.getId()) > 0;
|
return pending || countPendingCozeStates(task.getId()) > 0;
|
||||||
@@ -1626,6 +1657,7 @@ public class AppearancePatentTaskService {
|
|||||||
int batchTotal,
|
int batchTotal,
|
||||||
String prompt,
|
String prompt,
|
||||||
String apiKey,
|
String apiKey,
|
||||||
|
String patentToken,
|
||||||
Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId) {
|
Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId) {
|
||||||
if (batchRows == null || batchRows.isEmpty()) {
|
if (batchRows == null || batchRows.isEmpty()) {
|
||||||
return false;
|
return false;
|
||||||
@@ -1645,7 +1677,7 @@ public class AppearancePatentTaskService {
|
|||||||
AppearancePatentCozeClient.CozeCredentialRef credential = cozeClient.nextCredential();
|
AppearancePatentCozeClient.CozeCredentialRef credential = cozeClient.nextCredential();
|
||||||
try {
|
try {
|
||||||
AppearancePatentCozeClient.CozeSubmitResponse submit = submitCozeWorkflowThrottled(
|
AppearancePatentCozeClient.CozeSubmitResponse submit = submitCozeWorkflowThrottled(
|
||||||
batchRows, prompt, apiKey, credential, true);
|
batchRows, prompt, apiKey, patentToken, credential, true);
|
||||||
if (submit.immediateData() != null && !submit.immediateData().isBlank()) {
|
if (submit.immediateData() != null && !submit.immediateData().isBlank()) {
|
||||||
List<AppearancePatentResultRowDto> cozeRows = mergeUsableCozeRows(batchRows, submit.immediateData());
|
List<AppearancePatentResultRowDto> cozeRows = mergeUsableCozeRows(batchRows, submit.immediateData());
|
||||||
mergeCozeRowsIntoSubmittedChunks(task, cozeRows, allRowsByBaseId);
|
mergeCozeRowsIntoSubmittedChunks(task, cozeRows, allRowsByBaseId);
|
||||||
@@ -1858,7 +1890,7 @@ public class AppearancePatentTaskService {
|
|||||||
: "";
|
: "";
|
||||||
if (!poll.hasPayload() && failureMessage.isBlank()) {
|
if (!poll.hasPayload() && failureMessage.isBlank()) {
|
||||||
failureMessage = isCozeStateTimedOut(state)
|
failureMessage = isCozeStateTimedOut(state)
|
||||||
? "Coze async workflow poll timeout"
|
? "Coze 异步工作流轮询超时"
|
||||||
: "Coze async workflow completed without output";
|
: "Coze async workflow completed without output";
|
||||||
}
|
}
|
||||||
List<AppearancePatentResultRowDto> batchRows = readCozeBatchRows(state);
|
List<AppearancePatentResultRowDto> batchRows = readCozeBatchRows(state);
|
||||||
@@ -1929,6 +1961,43 @@ public class AppearancePatentTaskService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void finalizeTimedOutCozeStatesForTask(Long taskId) {
|
||||||
|
if (taskId == null || taskId <= 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||||
|
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
|
||||||
|
.in(TaskScopeStateEntity::getCozeStatus, List.of(COZE_STATUS_SUBMITTED, COZE_STATUS_RUNNING))
|
||||||
|
.orderByAsc(TaskScopeStateEntity::getCozeSubmittedAt)
|
||||||
|
.last("limit 50"));
|
||||||
|
if (states == null || states.isEmpty()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
for (TaskScopeStateEntity state : states) {
|
||||||
|
if (state == null || state.getId() == null || !isCozeStateTimedOut(state)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
CozeBatchContext context = readCozeBatchContext(state);
|
||||||
|
if (context == null || context.resultId() == null) {
|
||||||
|
markCozeStateTerminal(state, COZE_STATUS_FAILED, "Coze 批次上下文缺失");
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
List<AppearancePatentResultRowDto> batchRows = readCozeBatchRows(state);
|
||||||
|
FileTaskEntity task = fileTaskMapper.selectById(taskId);
|
||||||
|
if (task != null) {
|
||||||
|
Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId = loadAllRowsByBaseId(task);
|
||||||
|
mergeCozeRowsIntoSubmittedChunks(task,
|
||||||
|
cozeClient.markRowsFailed(batchRows, "Coze 异步工作流轮询超时"),
|
||||||
|
allRowsByBaseId);
|
||||||
|
}
|
||||||
|
markCozeStateTerminal(state, COZE_STATUS_FAILED, "Coze 异步工作流轮询超时");
|
||||||
|
maybeFinalizeCozeJobLocked(taskId, context);
|
||||||
|
log.warn("[appearance-patent] 文件任务超时兜底已将 Coze pending 批次置为失败 taskId={} stateId={} jobId={}",
|
||||||
|
taskId, state.getId(), context.jobId());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private boolean retryFailedCozeBatchState(TaskScopeStateEntity state,
|
private boolean retryFailedCozeBatchState(TaskScopeStateEntity state,
|
||||||
CozeBatchContext context,
|
CozeBatchContext context,
|
||||||
List<AppearancePatentResultRowDto> batchRows,
|
List<AppearancePatentResultRowDto> batchRows,
|
||||||
@@ -1945,7 +2014,7 @@ public class AppearancePatentTaskService {
|
|||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
AppearancePatentCozeClient.CozeSubmitResponse submit =
|
AppearancePatentCozeClient.CozeSubmitResponse submit =
|
||||||
submitCozeWorkflowThrottled(batchRows, readAiPrompt(task), readApiKey(task),
|
submitCozeWorkflowThrottled(batchRows, readAiPrompt(task), readApiKey(task), readPatentToken(task),
|
||||||
cozeClient.credentialByName(context.credentialName()), false);
|
cozeClient.credentialByName(context.credentialName()), false);
|
||||||
Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId = loadAllRowsByBaseId(task);
|
Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId = loadAllRowsByBaseId(task);
|
||||||
if (submit.immediateData() != null && !submit.immediateData().isBlank()) {
|
if (submit.immediateData() != null && !submit.immediateData().isBlank()) {
|
||||||
@@ -1996,6 +2065,7 @@ public class AppearancePatentTaskService {
|
|||||||
List<AppearancePatentResultRowDto> rows,
|
List<AppearancePatentResultRowDto> rows,
|
||||||
String prompt,
|
String prompt,
|
||||||
String apiKey,
|
String apiKey,
|
||||||
|
String patentToken,
|
||||||
AppearancePatentCozeClient.CozeCredentialRef credential,
|
AppearancePatentCozeClient.CozeCredentialRef credential,
|
||||||
boolean allowCredentialFallback) throws Exception {
|
boolean allowCredentialFallback) throws Exception {
|
||||||
int attempts = allowCredentialFallback ? Math.max(1, cozeClient.configuredCredentialCount()) : 1;
|
int attempts = allowCredentialFallback ? Math.max(1, cozeClient.configuredCredentialCount()) : 1;
|
||||||
@@ -2018,7 +2088,7 @@ public class AppearancePatentTaskService {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
try (lockHandle; borrowedCredential) {
|
try (lockHandle; borrowedCredential) {
|
||||||
return cozeClient.submitWorkflow(rows, prompt, apiKey, currentCredential);
|
return cozeClient.submitWorkflow(rows, prompt, apiKey, patentToken, currentCredential);
|
||||||
} finally {
|
} finally {
|
||||||
sleepQuietly(COZE_SUBMIT_MIN_INTERVAL_MILLIS);
|
sleepQuietly(COZE_SUBMIT_MIN_INTERVAL_MILLIS);
|
||||||
}
|
}
|
||||||
@@ -2088,7 +2158,7 @@ public class AppearancePatentTaskService {
|
|||||||
int partIndex = 1;
|
int partIndex = 1;
|
||||||
for (List<AppearancePatentResultRowDto> partRows : partitions) {
|
for (List<AppearancePatentResultRowDto> partRows : partitions) {
|
||||||
AppearancePatentCozeClient.CozeSubmitResponse submit =
|
AppearancePatentCozeClient.CozeSubmitResponse submit =
|
||||||
submitCozeWorkflowThrottled(partRows, readAiPrompt(task), readApiKey(task),
|
submitCozeWorkflowThrottled(partRows, readAiPrompt(task), readApiKey(task), readPatentToken(task),
|
||||||
cozeClient.credentialByName(context.credentialName()), false);
|
cozeClient.credentialByName(context.credentialName()), false);
|
||||||
if (submit.immediateData() != null && !submit.immediateData().isBlank()) {
|
if (submit.immediateData() != null && !submit.immediateData().isBlank()) {
|
||||||
List<AppearancePatentResultRowDto> cozeRows =
|
List<AppearancePatentResultRowDto> cozeRows =
|
||||||
@@ -2487,6 +2557,10 @@ public class AppearancePatentTaskService {
|
|||||||
return Duration.between(state.getCozeSubmittedAt(), LocalDateTime.now()).toMillis() >= timeoutMillis;
|
return Duration.between(state.getCozeSubmittedAt(), LocalDateTime.now()).toMillis() >= timeoutMillis;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private long cozeFlushPendingMillis() {
|
||||||
|
return Math.max(1, properties.getCozeFlushPendingMinutes()) * 60_000L;
|
||||||
|
}
|
||||||
|
|
||||||
private int cozeAttemptCount(TaskScopeStateEntity state) {
|
private int cozeAttemptCount(TaskScopeStateEntity state) {
|
||||||
return state == null || state.getCozeAttemptCount() == null ? 0 : state.getCozeAttemptCount();
|
return state == null || state.getCozeAttemptCount() == null ? 0 : state.getCozeAttemptCount();
|
||||||
}
|
}
|
||||||
@@ -2537,11 +2611,6 @@ public class AppearancePatentTaskService {
|
|||||||
return count != null && count > 0;
|
return count != null && count > 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
private boolean isJavaSideProcessing(Long taskId) {
|
|
||||||
return countPendingCozeStates(taskId) > 0
|
|
||||||
|| taskFileJobService.countActiveAssembleJobs(taskId, MODULE_TYPE) > 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void touchJavaSideTaskActivity(Long taskId) {
|
private void touchJavaSideTaskActivity(Long taskId) {
|
||||||
if (taskId == null || taskId <= 0) {
|
if (taskId == null || taskId <= 0) {
|
||||||
return;
|
return;
|
||||||
@@ -2615,7 +2684,7 @@ public class AppearancePatentTaskService {
|
|||||||
try {
|
try {
|
||||||
taskFileJobService.touchRunning(context.jobId());
|
taskFileJobService.touchRunning(context.jobId());
|
||||||
AppearancePatentCozeClient.CozeSubmitResponse submit =
|
AppearancePatentCozeClient.CozeSubmitResponse submit =
|
||||||
submitCozeWorkflowThrottled(batchRows, readAiPrompt(task), readApiKey(task),
|
submitCozeWorkflowThrottled(batchRows, readAiPrompt(task), readApiKey(task), readPatentToken(task),
|
||||||
cozeClient.credentialByName(context.credentialName()), false);
|
cozeClient.credentialByName(context.credentialName()), false);
|
||||||
Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId = loadAllRowsByBaseId(task);
|
Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId = loadAllRowsByBaseId(task);
|
||||||
if (submit.immediateData() != null && !submit.immediateData().isBlank()) {
|
if (submit.immediateData() != null && !submit.immediateData().isBlank()) {
|
||||||
@@ -2652,13 +2721,27 @@ public class AppearancePatentTaskService {
|
|||||||
log.warn("[appearance-patent] coze pending submit retry failed taskId={} stateId={} jobId={} rows={} err={}",
|
log.warn("[appearance-patent] coze pending submit retry failed taskId={} stateId={} jobId={} rows={} err={}",
|
||||||
state.getTaskId(), state.getId(), context.jobId(), batchRows.size(), message);
|
state.getTaskId(), state.getId(), context.jobId(), batchRows.size(), message);
|
||||||
if (isCozeThrottleLockTimeout(message)) {
|
if (isCozeThrottleLockTimeout(message)) {
|
||||||
|
if (isCozeStateTimedOut(state)) {
|
||||||
|
Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId = loadAllRowsByBaseId(task);
|
||||||
|
String finalMessage = "等待 Coze 提交凭证超时:" + message;
|
||||||
|
mergeCozeRowsIntoSubmittedChunks(task,
|
||||||
|
cozeClient.markRowsFailed(batchRows, finalMessage),
|
||||||
|
allRowsByBaseId);
|
||||||
|
markCozeStateTerminal(state, COZE_STATUS_FAILED, finalMessage);
|
||||||
|
maybeFinalizeCozeJobLocked(state.getTaskId(), context);
|
||||||
|
log.warn("[appearance-patent] Coze pending 提交等待超时,已将批次置为失败 taskId={} stateId={} jobId={} rows={}",
|
||||||
|
state.getTaskId(), state.getId(), context.jobId(), batchRows.size());
|
||||||
|
return;
|
||||||
|
}
|
||||||
keepPendingCozeSubmitState(state, message);
|
keepPendingCozeSubmitState(state, message);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
int nextAttemptCount = cozeAttemptCount(state) + 1;
|
int nextAttemptCount = cozeAttemptCount(state) + 1;
|
||||||
if (nextAttemptCount >= MAX_COZE_SUBMIT_RETRY_COUNT) {
|
if (nextAttemptCount >= MAX_COZE_SUBMIT_RETRY_COUNT || isCozeStateTimedOut(state)) {
|
||||||
Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId = loadAllRowsByBaseId(task);
|
Map<String, List<AppearancePatentParsedRowVo>> allRowsByBaseId = loadAllRowsByBaseId(task);
|
||||||
String finalMessage = message + " after " + nextAttemptCount + " submit attempts";
|
String finalMessage = isCozeStateTimedOut(state)
|
||||||
|
? message + "(超过 " + properties.getCozePollTimeoutMillis() + "ms)"
|
||||||
|
: message + ",已重试提交 " + nextAttemptCount + " 次";
|
||||||
mergeCozeRowsIntoSubmittedChunks(task,
|
mergeCozeRowsIntoSubmittedChunks(task,
|
||||||
cozeClient.markRowsFailed(batchRows, finalMessage),
|
cozeClient.markRowsFailed(batchRows, finalMessage),
|
||||||
allRowsByBaseId);
|
allRowsByBaseId);
|
||||||
@@ -2699,7 +2782,11 @@ public class AppearancePatentTaskService {
|
|||||||
}
|
}
|
||||||
log.warn("[appearance-patent] reject task operation because owner is another instance taskId={} operation={} owner={} current={}",
|
log.warn("[appearance-patent] reject task operation because owner is another instance taskId={} operation={} owner={} current={}",
|
||||||
task == null ? null : task.getId(), operation, owner, currentInstanceId());
|
task == null ? null : task.getId(), operation, owner, currentInstanceId());
|
||||||
throw new BusinessException(40903, "该任务已绑定到另一台服务实例处理,请通过原实例继续处理");
|
throw new TaskOwnerMismatchException(
|
||||||
|
task == null ? null : task.getId(),
|
||||||
|
operation,
|
||||||
|
owner,
|
||||||
|
currentInstanceId());
|
||||||
}
|
}
|
||||||
|
|
||||||
private boolean isCozeStateOwnedByCurrentInstance(TaskScopeStateEntity state) {
|
private boolean isCozeStateOwnedByCurrentInstance(TaskScopeStateEntity state) {
|
||||||
@@ -3814,10 +3901,11 @@ public class AppearancePatentTaskService {
|
|||||||
return val == null ? "" : val.replace(String.valueOf((char) 0xFEFF), "").replace((char) 0x3000, ' ').trim().replaceAll("\\s+", " ");
|
return val == null ? "" : val.replace(String.valueOf((char) 0xFEFF), "").replace((char) 0x3000, ' ').trim().replaceAll("\\s+", " ");
|
||||||
}
|
}
|
||||||
|
|
||||||
private String buildParsedPayloadJson(String aiPrompt, String apiKey, List<AppearancePatentSourceFileDto> sourceFiles, List<String> headers, List<AppearancePatentParsedRowVo> allRows) {
|
private String buildParsedPayloadJson(String aiPrompt, String apiKey, String patentToken, List<AppearancePatentSourceFileDto> sourceFiles, List<String> headers, List<AppearancePatentParsedRowVo> allRows) {
|
||||||
AppearancePatentParsedPayloadDto payload = new AppearancePatentParsedPayloadDto();
|
AppearancePatentParsedPayloadDto payload = new AppearancePatentParsedPayloadDto();
|
||||||
payload.setAiPrompt(normalize(aiPrompt));
|
payload.setAiPrompt(normalize(aiPrompt));
|
||||||
payload.setApiKey(normalize(apiKey));
|
payload.setApiKey(normalize(apiKey));
|
||||||
|
payload.setPatentToken(normalize(patentToken));
|
||||||
payload.setSourceFiles(sourceFiles == null ? List.of() : sourceFiles);
|
payload.setSourceFiles(sourceFiles == null ? List.of() : sourceFiles);
|
||||||
payload.setHeaders(headers == null ? List.of() : headers);
|
payload.setHeaders(headers == null ? List.of() : headers);
|
||||||
payload.setItems(List.of());
|
payload.setItems(List.of());
|
||||||
@@ -3826,10 +3914,11 @@ public class AppearancePatentTaskService {
|
|||||||
return writeJson(payload, "保存解析结果失败");
|
return writeJson(payload, "保存解析结果失败");
|
||||||
}
|
}
|
||||||
|
|
||||||
private String buildTaskResultJson(String aiPrompt, String apiKey, List<AppearancePatentSourceFileDto> sourceFiles, String parsedPayloadPointer) {
|
private String buildTaskResultJson(String aiPrompt, String apiKey, String patentToken, List<AppearancePatentSourceFileDto> sourceFiles, String parsedPayloadPointer) {
|
||||||
Map<String, Object> payload = new LinkedHashMap<>();
|
Map<String, Object> payload = new LinkedHashMap<>();
|
||||||
payload.put("aiPrompt", normalize(aiPrompt));
|
payload.put("aiPrompt", normalize(aiPrompt));
|
||||||
payload.put("apiKey", normalize(apiKey));
|
payload.put("apiKey", normalize(apiKey));
|
||||||
|
payload.put("patentToken", normalize(patentToken));
|
||||||
payload.put("sourceFileKeys", sourceFiles == null ? List.of() : sourceFiles.stream()
|
payload.put("sourceFileKeys", sourceFiles == null ? List.of() : sourceFiles.stream()
|
||||||
.map(AppearancePatentSourceFileDto::getFileKey)
|
.map(AppearancePatentSourceFileDto::getFileKey)
|
||||||
.filter(Objects::nonNull)
|
.filter(Objects::nonNull)
|
||||||
@@ -3895,6 +3984,7 @@ public class AppearancePatentTaskService {
|
|||||||
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || userId != null && !Objects.equals(userId, task.getUserId())) {
|
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || userId != null && !Objects.equals(userId, task.getUserId())) {
|
||||||
throw new BusinessException("任务不存在");
|
throw new BusinessException("任务不存在");
|
||||||
}
|
}
|
||||||
|
ensureTaskOwnedByCurrentInstance(task, "parsed payload");
|
||||||
return readParsedPayload(task);
|
return readParsedPayload(task);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -3903,10 +3993,12 @@ public class AppearancePatentTaskService {
|
|||||||
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || userId != null && !Objects.equals(userId, task.getUserId())) {
|
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || userId != null && !Objects.equals(userId, task.getUserId())) {
|
||||||
throw new BusinessException("任务不存在");
|
throw new BusinessException("任务不存在");
|
||||||
}
|
}
|
||||||
|
ensureTaskOwnedByCurrentInstance(task, "queue payload");
|
||||||
AppearancePatentParsedPayloadDto payload = readParsedPayload(task);
|
AppearancePatentParsedPayloadDto payload = readParsedPayload(task);
|
||||||
AppearancePatentParsedPayloadDto queuePayload = new AppearancePatentParsedPayloadDto();
|
AppearancePatentParsedPayloadDto queuePayload = new AppearancePatentParsedPayloadDto();
|
||||||
queuePayload.setAiPrompt(payload.getAiPrompt());
|
queuePayload.setAiPrompt(payload.getAiPrompt());
|
||||||
queuePayload.setApiKey(payload.getApiKey());
|
queuePayload.setApiKey(payload.getApiKey());
|
||||||
|
queuePayload.setPatentToken(payload.getPatentToken());
|
||||||
queuePayload.setGroups(payload.getGroups() == null ? List.of() : payload.getGroups());
|
queuePayload.setGroups(payload.getGroups() == null ? List.of() : payload.getGroups());
|
||||||
queuePayload.setItems(List.of());
|
queuePayload.setItems(List.of());
|
||||||
queuePayload.setAllItems(List.of());
|
queuePayload.setAllItems(List.of());
|
||||||
@@ -3932,6 +4024,7 @@ public class AppearancePatentTaskService {
|
|||||||
AppearancePatentParsedGroupPageDto vo = new AppearancePatentParsedGroupPageDto();
|
AppearancePatentParsedGroupPageDto vo = new AppearancePatentParsedGroupPageDto();
|
||||||
vo.setAiPrompt(payload.getAiPrompt());
|
vo.setAiPrompt(payload.getAiPrompt());
|
||||||
vo.setApiKey(payload.getApiKey());
|
vo.setApiKey(payload.getApiKey());
|
||||||
|
vo.setPatentToken(payload.getPatentToken());
|
||||||
vo.setPage(safePage);
|
vo.setPage(safePage);
|
||||||
vo.setPageSize(safePageSize);
|
vo.setPageSize(safePageSize);
|
||||||
vo.setTotalGroups(totalGroups);
|
vo.setTotalGroups(totalGroups);
|
||||||
@@ -3988,6 +4081,8 @@ public class AppearancePatentTaskService {
|
|||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
log.warn("[appearance-patent] read chunk payload failed taskId={} chunk={} err={}",
|
log.warn("[appearance-patent] read chunk payload failed taskId={} chunk={} err={}",
|
||||||
chunk.getTaskId(), chunk.getChunkIndex(), ex.getMessage());
|
chunk.getTaskId(), chunk.getChunkIndex(), ex.getMessage());
|
||||||
|
throw new BusinessException("appearance patent chunk payload read failed chunk="
|
||||||
|
+ chunk.getChunkIndex() + ": " + ex.getMessage(), ex);
|
||||||
}
|
}
|
||||||
return rows;
|
return rows;
|
||||||
}
|
}
|
||||||
@@ -4081,13 +4176,6 @@ public class AppearancePatentTaskService {
|
|||||||
return firstNonBlank(row.getConclusion(), "");
|
return firstNonBlank(row.getConclusion(), "");
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* 导出最终 xlsx 时的状态展示:
|
|
||||||
* - 如果 status 是 markFailed 合成的(failureSyntheticStatus=true),
|
|
||||||
* 且 row 实际并没有任何风险维度结果,导出层展示错误信息(有则放入,无则留空),
|
|
||||||
* 避免用户原样重新上传时该行被反复识别为失败行重新触发 Coze。
|
|
||||||
* - 真正业务侧失败 / Python 真实回传 FAILED 仍按原值显示。
|
|
||||||
*/
|
|
||||||
private String userFacingStatus(AppearancePatentResultRowDto row) {
|
private String userFacingStatus(AppearancePatentResultRowDto row) {
|
||||||
if (row == null) {
|
if (row == null) {
|
||||||
return "";
|
return "";
|
||||||
@@ -4096,7 +4184,6 @@ public class AppearancePatentTaskService {
|
|||||||
if (!row.isFailureSyntheticStatus()) {
|
if (!row.isFailureSyntheticStatus()) {
|
||||||
return original;
|
return original;
|
||||||
}
|
}
|
||||||
// 合成状态:有错误信息则放入错误信息,没有则留空
|
|
||||||
return firstNonBlank(row.getError(), "");
|
return firstNonBlank(row.getError(), "");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,200 @@
|
|||||||
|
package com.nanri.aiimage.modules.brand.client;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.annotation.JsonAlias;
|
||||||
|
import com.nanri.aiimage.config.BrandCheckProperties;
|
||||||
|
import lombok.Data;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.http.HttpHeaders;
|
||||||
|
import org.springframework.http.MediaType;
|
||||||
|
import org.springframework.http.client.SimpleClientHttpRequestFactory;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
import org.springframework.web.client.RestClient;
|
||||||
|
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
@Component
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
@Slf4j
|
||||||
|
public class BrandCheckClient {
|
||||||
|
|
||||||
|
private static final MediaType APPLICATION_JSON_UTF8 = new MediaType("application", "json", StandardCharsets.UTF_8);
|
||||||
|
|
||||||
|
private final BrandCheckProperties properties;
|
||||||
|
private volatile RestClient sharedRestClient;
|
||||||
|
|
||||||
|
public BrandCheckResponse check(String brand) {
|
||||||
|
return check(brand, properties.getDefaultStrategy());
|
||||||
|
}
|
||||||
|
|
||||||
|
public BrandCheckResponse check(String brand, String strategy) {
|
||||||
|
String normalizedBrand = normalize(brand);
|
||||||
|
if (normalizedBrand.isBlank()) {
|
||||||
|
return BrandCheckResponse.empty();
|
||||||
|
}
|
||||||
|
Map<String, Object> body = new LinkedHashMap<>();
|
||||||
|
body.put("brand", normalizedBrand);
|
||||||
|
body.put("strategy", normalize(strategy).isBlank() ? "Terms" : normalize(strategy));
|
||||||
|
return restClient().post()
|
||||||
|
.uri(joinUrl(properties.getBaseUrl(), properties.getPath()))
|
||||||
|
.headers(headers -> {
|
||||||
|
headers.set(HttpHeaders.ACCEPT, MediaType.APPLICATION_JSON_VALUE);
|
||||||
|
headers.setContentType(APPLICATION_JSON_UTF8);
|
||||||
|
if (properties.getToken() != null && !properties.getToken().isBlank()) {
|
||||||
|
headers.set("X-Token", properties.getToken().trim());
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.body(body)
|
||||||
|
.retrieve()
|
||||||
|
.body(BrandCheckResponse.class);
|
||||||
|
}
|
||||||
|
|
||||||
|
public BrandCheckBatchResult checkAll(List<String> brands, String strategy) {
|
||||||
|
List<String> distinctBrands = distinctNonBlank(brands);
|
||||||
|
List<Object> failedData = new ArrayList<>();
|
||||||
|
List<Object> queryFailedData = new ArrayList<>();
|
||||||
|
for (String brand : distinctBrands) {
|
||||||
|
try {
|
||||||
|
BrandCheckResponse response = check(brand, strategy);
|
||||||
|
if (response == null) {
|
||||||
|
queryFailedData.add(brand);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
failedData.addAll(nullToEmpty(response.getFaildData()));
|
||||||
|
queryFailedData.addAll(nullToEmpty(response.getQueryFaildData()));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[brand-check] request failed brand={} strategy={} err={}", brand, strategy, ex.getMessage());
|
||||||
|
queryFailedData.add(brand);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return new BrandCheckBatchResult(distinctBrands, failedData, queryFailedData);
|
||||||
|
}
|
||||||
|
|
||||||
|
public BrandCheckBatchResult checkTitleText(String titleText) {
|
||||||
|
return checkTitleText(titleText, properties.getDefaultStrategy());
|
||||||
|
}
|
||||||
|
|
||||||
|
public BrandCheckBatchResult checkTitleText(String titleText, String strategy) {
|
||||||
|
return checkAll(splitTitleText(titleText), strategy);
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<String> splitTitleText(String titleText) {
|
||||||
|
String normalized = normalize(titleText)
|
||||||
|
.replace(',', ',')
|
||||||
|
.replace('、', ',')
|
||||||
|
.replace(';', ',')
|
||||||
|
.replace(';', ',')
|
||||||
|
.replace('\n', ',')
|
||||||
|
.replace('\r', ',')
|
||||||
|
.replace('\t', ',');
|
||||||
|
if (normalized.isBlank()) {
|
||||||
|
return List.of();
|
||||||
|
}
|
||||||
|
List<String> result = new ArrayList<>();
|
||||||
|
for (String item : normalized.split(",")) {
|
||||||
|
String brand = cleanTitleToken(item);
|
||||||
|
if (!brand.isBlank() && !result.contains(brand)) {
|
||||||
|
result.add(brand);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
private RestClient restClient() {
|
||||||
|
RestClient client = sharedRestClient;
|
||||||
|
if (client != null) {
|
||||||
|
return client;
|
||||||
|
}
|
||||||
|
synchronized (this) {
|
||||||
|
if (sharedRestClient == null) {
|
||||||
|
SimpleClientHttpRequestFactory requestFactory = new SimpleClientHttpRequestFactory();
|
||||||
|
requestFactory.setConnectTimeout(properties.getConnectTimeoutMillis());
|
||||||
|
requestFactory.setReadTimeout(properties.getReadTimeoutMillis());
|
||||||
|
sharedRestClient = RestClient.builder().requestFactory(requestFactory).build();
|
||||||
|
}
|
||||||
|
return sharedRestClient;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<Object> nullToEmpty(List<Object> values) {
|
||||||
|
return values == null ? List.of() : values;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String cleanTitleToken(String value) {
|
||||||
|
String normalized = normalize(value);
|
||||||
|
while (normalized.startsWith("'")
|
||||||
|
|| normalized.startsWith("\"")
|
||||||
|
|| normalized.startsWith("“")
|
||||||
|
|| normalized.startsWith("‘")) {
|
||||||
|
normalized = normalized.substring(1).trim();
|
||||||
|
}
|
||||||
|
while (normalized.endsWith("'")
|
||||||
|
|| normalized.endsWith("\"")
|
||||||
|
|| normalized.endsWith("”")
|
||||||
|
|| normalized.endsWith("’")) {
|
||||||
|
normalized = normalized.substring(0, normalized.length() - 1).trim();
|
||||||
|
}
|
||||||
|
return normalized;
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<String> distinctNonBlank(List<String> values) {
|
||||||
|
List<String> result = new ArrayList<>();
|
||||||
|
if (values == null) {
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
for (String value : values) {
|
||||||
|
String normalized = normalize(value);
|
||||||
|
if (!normalized.isBlank() && !result.contains(normalized)) {
|
||||||
|
result.add(normalized);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String joinUrl(String baseUrl, String path) {
|
||||||
|
String base = baseUrl == null ? "" : baseUrl.trim();
|
||||||
|
String suffix = path == null ? "" : path.trim();
|
||||||
|
if (base.endsWith("/") && suffix.startsWith("/")) {
|
||||||
|
return base + suffix.substring(1);
|
||||||
|
}
|
||||||
|
if (!base.endsWith("/") && !suffix.startsWith("/")) {
|
||||||
|
return base + "/" + suffix;
|
||||||
|
}
|
||||||
|
return base + suffix;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String normalize(String value) {
|
||||||
|
return value == null ? "" : value.trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Data
|
||||||
|
public static class BrandCheckResponse {
|
||||||
|
@JsonAlias({"faild_data", "failed_data"})
|
||||||
|
private List<Object> faildData = new ArrayList<>();
|
||||||
|
|
||||||
|
@JsonAlias({"query_faild_data", "query_failed_data"})
|
||||||
|
private List<Object> queryFaildData = new ArrayList<>();
|
||||||
|
|
||||||
|
private static BrandCheckResponse empty() {
|
||||||
|
return new BrandCheckResponse();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public record BrandCheckBatchResult(
|
||||||
|
List<String> brands,
|
||||||
|
List<Object> faildData,
|
||||||
|
List<Object> queryFaildData
|
||||||
|
) {
|
||||||
|
public boolean hasFailedData() {
|
||||||
|
return faildData != null && !faildData.isEmpty();
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean hasQueryFailedData() {
|
||||||
|
return queryFaildData != null && !queryFaildData.isEmpty();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -80,6 +80,29 @@ public class BrandTaskProgressCacheService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void touchHeartbeat(Long taskId, String phase, Integer current, Integer total) {
|
||||||
|
String key = buildKey(taskId);
|
||||||
|
String now = String.valueOf(Instant.now().toEpochMilli());
|
||||||
|
Map<String, String> values = new LinkedHashMap<>();
|
||||||
|
values.put("updated_at", now);
|
||||||
|
values.put("last_heartbeat_at", now);
|
||||||
|
if (phase != null && !phase.isBlank()) {
|
||||||
|
values.put("phase", normalizePhase(phase));
|
||||||
|
}
|
||||||
|
if (current != null) {
|
||||||
|
values.put("current_line", String.valueOf(Math.max(current, 0)));
|
||||||
|
}
|
||||||
|
if (total != null) {
|
||||||
|
values.put("total_lines", String.valueOf(Math.max(total, 0)));
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
stringRedisTemplate.opsForHash().putAll(key, values);
|
||||||
|
stringRedisTemplate.expire(key, ttl());
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[brand-progress-cache] touch heartbeat degraded taskId={} msg={}", taskId, ex.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public void markFailed(Long taskId, String message) {
|
public void markFailed(Long taskId, String message) {
|
||||||
String key = buildKey(taskId);
|
String key = buildKey(taskId);
|
||||||
String now = String.valueOf(Instant.now().toEpochMilli());
|
String now = String.valueOf(Instant.now().toEpochMilli());
|
||||||
|
|||||||
@@ -154,12 +154,10 @@ public class DeleteBrandStaleTaskService {
|
|||||||
|
|
||||||
private void failStaleDeleteBrandTasks() {
|
private void failStaleDeleteBrandTasks() {
|
||||||
long minutes = Math.max(1L, deleteBrandProgressProperties.getHeartbeatTimeoutMinutes());
|
long minutes = Math.max(1L, deleteBrandProgressProperties.getHeartbeatTimeoutMinutes());
|
||||||
long initialMinutes = Math.max(minutes, deleteBrandProgressProperties.getDeleteBrandInitialTimeoutMinutes());
|
|
||||||
long staleTimeoutMillis = Duration.ofMinutes(minutes).toMillis();
|
long staleTimeoutMillis = Duration.ofMinutes(minutes).toMillis();
|
||||||
long nowMillis = System.currentTimeMillis();
|
long nowMillis = System.currentTimeMillis();
|
||||||
LocalDateTime now = LocalDateTime.now();
|
LocalDateTime now = LocalDateTime.now();
|
||||||
LocalDateTime threshold = now.minusMinutes(minutes);
|
LocalDateTime threshold = now.minusMinutes(minutes);
|
||||||
LocalDateTime initialThreshold = now.minusMinutes(initialMinutes);
|
|
||||||
|
|
||||||
java.util.List<FileTaskEntity> runningTasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
java.util.List<FileTaskEntity> runningTasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||||
.eq(FileTaskEntity::getModuleType, MODULE_TYPE_DELETE_BRAND)
|
.eq(FileTaskEntity::getModuleType, MODULE_TYPE_DELETE_BRAND)
|
||||||
@@ -187,9 +185,6 @@ public class DeleteBrandStaleTaskService {
|
|||||||
if (lastHeartbeatAt > 0L && nowMillis - lastHeartbeatAt < staleTimeoutMillis) {
|
if (lastHeartbeatAt > 0L && nowMillis - lastHeartbeatAt < staleTimeoutMillis) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if (!hasStartedProgress && isWithinInitialGrace(task, initialThreshold)) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
TaskDistributedLockService.LockHandle taskLockHandle = acquireTaskLock(MODULE_TYPE_DELETE_BRAND, task.getId());
|
TaskDistributedLockService.LockHandle taskLockHandle = acquireTaskLock(MODULE_TYPE_DELETE_BRAND, task.getId());
|
||||||
if (taskLockHandle == null) {
|
if (taskLockHandle == null) {
|
||||||
@@ -209,16 +204,14 @@ public class DeleteBrandStaleTaskService {
|
|||||||
log.warn("[stale-check] delete-brand finalize threw taskId={} msg={}", task.getId(), ex.getMessage());
|
log.warn("[stale-check] delete-brand finalize threw taskId={} msg={}", task.getId(), ex.getMessage());
|
||||||
}
|
}
|
||||||
|
|
||||||
log.warn("[stale-check] delete-brand failing stale task -> taskId={} updatedAt={} createdAt={} lastHeartbeatAt={} timeoutMinutes={} initialTimeoutMinutes={} completedScopes={} hasStartedProgress={} initialThreshold={}",
|
log.warn("[stale-check] delete-brand failing stale task -> taskId={} updatedAt={} createdAt={} lastHeartbeatAt={} timeoutMinutes={} completedScopes={} hasStartedProgress={}",
|
||||||
task.getId(),
|
task.getId(),
|
||||||
task.getUpdatedAt(),
|
task.getUpdatedAt(),
|
||||||
task.getCreatedAt(),
|
task.getCreatedAt(),
|
||||||
lastHeartbeatAt,
|
lastHeartbeatAt,
|
||||||
minutes,
|
minutes,
|
||||||
initialMinutes,
|
|
||||||
completedScopeCount,
|
completedScopeCount,
|
||||||
hasStartedProgress,
|
hasStartedProgress);
|
||||||
initialThreshold);
|
|
||||||
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||||
.eq(FileTaskEntity::getId, task.getId())
|
.eq(FileTaskEntity::getId, task.getId())
|
||||||
.eq(FileTaskEntity::getModuleType, MODULE_TYPE_DELETE_BRAND)
|
.eq(FileTaskEntity::getModuleType, MODULE_TYPE_DELETE_BRAND)
|
||||||
@@ -239,12 +232,10 @@ public class DeleteBrandStaleTaskService {
|
|||||||
private ProductRiskStaleCheckStats failStaleProductRiskResolveTasks() {
|
private ProductRiskStaleCheckStats failStaleProductRiskResolveTasks() {
|
||||||
ProductRiskStaleCheckStats stats = new ProductRiskStaleCheckStats();
|
ProductRiskStaleCheckStats stats = new ProductRiskStaleCheckStats();
|
||||||
long minutes = Math.max(1L, deleteBrandProgressProperties.getProductRiskStaleTimeoutMinutes());
|
long minutes = Math.max(1L, deleteBrandProgressProperties.getProductRiskStaleTimeoutMinutes());
|
||||||
long initialMinutes = Math.max(minutes, deleteBrandProgressProperties.getProductRiskInitialTimeoutMinutes());
|
|
||||||
long staleTimeoutMillis = Duration.ofMinutes(minutes).toMillis();
|
long staleTimeoutMillis = Duration.ofMinutes(minutes).toMillis();
|
||||||
long nowMillis = System.currentTimeMillis();
|
long nowMillis = System.currentTimeMillis();
|
||||||
LocalDateTime now = LocalDateTime.now();
|
LocalDateTime now = LocalDateTime.now();
|
||||||
LocalDateTime threshold = now.minusMinutes(minutes);
|
LocalDateTime threshold = now.minusMinutes(minutes);
|
||||||
LocalDateTime initialThreshold = now.minusMinutes(initialMinutes);
|
|
||||||
java.util.List<FileTaskEntity> runningTasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
java.util.List<FileTaskEntity> runningTasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||||
.eq(FileTaskEntity::getModuleType, MODULE_TYPE_PRODUCT_RISK)
|
.eq(FileTaskEntity::getModuleType, MODULE_TYPE_PRODUCT_RISK)
|
||||||
.eq(FileTaskEntity::getStatus, "RUNNING")
|
.eq(FileTaskEntity::getStatus, "RUNNING")
|
||||||
@@ -256,8 +247,8 @@ public class DeleteBrandStaleTaskService {
|
|||||||
}
|
}
|
||||||
Map<Long, Long> heartbeatByTaskId = productRiskTaskCacheService.getTaskHeartbeatMillisBatch(
|
Map<Long, Long> heartbeatByTaskId = productRiskTaskCacheService.getTaskHeartbeatMillisBatch(
|
||||||
runningTasks.stream().map(FileTaskEntity::getId).toList());
|
runningTasks.stream().map(FileTaskEntity::getId).toList());
|
||||||
log.info("[stale-check] product-risk candidates={} threshold={} timeoutMinutes={} initialThreshold={} initialTimeoutMinutes={}",
|
log.info("[stale-check] product-risk candidates={} threshold={} timeoutMinutes={}",
|
||||||
runningTasks.size(), threshold, minutes, initialThreshold, initialMinutes);
|
runningTasks.size(), threshold, minutes);
|
||||||
for (FileTaskEntity task : runningTasks) {
|
for (FileTaskEntity task : runningTasks) {
|
||||||
long lastPayloadHeartbeatMillis = heartbeatByTaskId.getOrDefault(task.getId(), 0L);
|
long lastPayloadHeartbeatMillis = heartbeatByTaskId.getOrDefault(task.getId(), 0L);
|
||||||
boolean hasFinishedRows = (task.getSuccessFileCount() != null && task.getSuccessFileCount() > 0)
|
boolean hasFinishedRows = (task.getSuccessFileCount() != null && task.getSuccessFileCount() > 0)
|
||||||
@@ -280,12 +271,6 @@ public class DeleteBrandStaleTaskService {
|
|||||||
taskFileJobService.countActiveAssembleJobs(task.getId(), MODULE_TYPE_PRODUCT_RISK));
|
taskFileJobService.countActiveAssembleJobs(task.getId(), MODULE_TYPE_PRODUCT_RISK));
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if (!hasStartedProgress && isWithinInitialGrace(task, initialThreshold)) {
|
|
||||||
stats.skippedTaskCount++;
|
|
||||||
log.info("[stale-check] product-risk skip initial-grace taskId={} createdAt={} initialThreshold={}",
|
|
||||||
task.getId(), task.getCreatedAt(), initialThreshold);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
TaskDistributedLockService.LockHandle taskLockHandle = acquireTaskLock(MODULE_TYPE_PRODUCT_RISK, task.getId());
|
TaskDistributedLockService.LockHandle taskLockHandle = acquireTaskLock(MODULE_TYPE_PRODUCT_RISK, task.getId());
|
||||||
if (taskLockHandle == null) {
|
if (taskLockHandle == null) {
|
||||||
stats.skippedTaskCount++;
|
stats.skippedTaskCount++;
|
||||||
@@ -323,12 +308,10 @@ public class DeleteBrandStaleTaskService {
|
|||||||
private PriceTrackStaleCheckStats failStalePriceTrackTasks() {
|
private PriceTrackStaleCheckStats failStalePriceTrackTasks() {
|
||||||
PriceTrackStaleCheckStats stats = new PriceTrackStaleCheckStats();
|
PriceTrackStaleCheckStats stats = new PriceTrackStaleCheckStats();
|
||||||
long minutes = Math.max(1L, deleteBrandProgressProperties.getPriceTrackStaleTimeoutMinutes());
|
long minutes = Math.max(1L, deleteBrandProgressProperties.getPriceTrackStaleTimeoutMinutes());
|
||||||
long initialMinutes = Math.max(minutes, deleteBrandProgressProperties.getPriceTrackInitialTimeoutMinutes());
|
|
||||||
long staleTimeoutMillis = Duration.ofMinutes(minutes).toMillis();
|
long staleTimeoutMillis = Duration.ofMinutes(minutes).toMillis();
|
||||||
long nowMillis = System.currentTimeMillis();
|
long nowMillis = System.currentTimeMillis();
|
||||||
LocalDateTime now = LocalDateTime.now();
|
LocalDateTime now = LocalDateTime.now();
|
||||||
LocalDateTime threshold = now.minusMinutes(minutes);
|
LocalDateTime threshold = now.minusMinutes(minutes);
|
||||||
LocalDateTime initialThreshold = now.minusMinutes(initialMinutes);
|
|
||||||
java.util.List<FileTaskEntity> runningTasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
java.util.List<FileTaskEntity> runningTasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||||
.eq(FileTaskEntity::getModuleType, MODULE_TYPE_PRICE_TRACK)
|
.eq(FileTaskEntity::getModuleType, MODULE_TYPE_PRICE_TRACK)
|
||||||
.eq(FileTaskEntity::getStatus, "RUNNING")
|
.eq(FileTaskEntity::getStatus, "RUNNING")
|
||||||
@@ -340,8 +323,8 @@ public class DeleteBrandStaleTaskService {
|
|||||||
}
|
}
|
||||||
Map<Long, Long> heartbeatByTaskId = priceTrackTaskCacheService.getTaskHeartbeatMillisBatch(
|
Map<Long, Long> heartbeatByTaskId = priceTrackTaskCacheService.getTaskHeartbeatMillisBatch(
|
||||||
runningTasks.stream().map(FileTaskEntity::getId).toList());
|
runningTasks.stream().map(FileTaskEntity::getId).toList());
|
||||||
log.info("[stale-check] price-track candidates={} threshold={} timeoutMinutes={} initialThreshold={} initialTimeoutMinutes={}",
|
log.info("[stale-check] price-track candidates={} threshold={} timeoutMinutes={}",
|
||||||
runningTasks.size(), threshold, minutes, initialThreshold, initialMinutes);
|
runningTasks.size(), threshold, minutes);
|
||||||
for (FileTaskEntity task : runningTasks) {
|
for (FileTaskEntity task : runningTasks) {
|
||||||
long lastHeartbeatMillis = heartbeatByTaskId.getOrDefault(task.getId(), 0L);
|
long lastHeartbeatMillis = heartbeatByTaskId.getOrDefault(task.getId(), 0L);
|
||||||
boolean hasFinishedRows = (task.getSuccessFileCount() != null && task.getSuccessFileCount() > 0)
|
boolean hasFinishedRows = (task.getSuccessFileCount() != null && task.getSuccessFileCount() > 0)
|
||||||
@@ -362,10 +345,6 @@ public class DeleteBrandStaleTaskService {
|
|||||||
taskFileJobService.countActiveAssembleJobs(task.getId(), MODULE_TYPE_PRICE_TRACK));
|
taskFileJobService.countActiveAssembleJobs(task.getId(), MODULE_TYPE_PRICE_TRACK));
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if (!hasStartedProgress && isWithinInitialGrace(task, initialThreshold)) {
|
|
||||||
stats.skippedTaskCount++;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
TaskDistributedLockService.LockHandle taskLockHandle = acquireTaskLock(MODULE_TYPE_PRICE_TRACK, task.getId());
|
TaskDistributedLockService.LockHandle taskLockHandle = acquireTaskLock(MODULE_TYPE_PRICE_TRACK, task.getId());
|
||||||
if (taskLockHandle == null) {
|
if (taskLockHandle == null) {
|
||||||
stats.skippedTaskCount++;
|
stats.skippedTaskCount++;
|
||||||
@@ -404,12 +383,10 @@ public class DeleteBrandStaleTaskService {
|
|||||||
private ShopMatchStaleCheckStats failStaleShopMatchTasks() {
|
private ShopMatchStaleCheckStats failStaleShopMatchTasks() {
|
||||||
ShopMatchStaleCheckStats stats = new ShopMatchStaleCheckStats();
|
ShopMatchStaleCheckStats stats = new ShopMatchStaleCheckStats();
|
||||||
long minutes = Math.max(1L, deleteBrandProgressProperties.getShopMatchStaleTimeoutMinutes());
|
long minutes = Math.max(1L, deleteBrandProgressProperties.getShopMatchStaleTimeoutMinutes());
|
||||||
long initialMinutes = Math.max(minutes, deleteBrandProgressProperties.getShopMatchInitialTimeoutMinutes());
|
|
||||||
long staleTimeoutMillis = Duration.ofMinutes(minutes).toMillis();
|
long staleTimeoutMillis = Duration.ofMinutes(minutes).toMillis();
|
||||||
long nowMillis = System.currentTimeMillis();
|
long nowMillis = System.currentTimeMillis();
|
||||||
LocalDateTime now = LocalDateTime.now();
|
LocalDateTime now = LocalDateTime.now();
|
||||||
LocalDateTime threshold = now.minusMinutes(minutes);
|
LocalDateTime threshold = now.minusMinutes(minutes);
|
||||||
LocalDateTime initialThreshold = now.minusMinutes(initialMinutes);
|
|
||||||
java.util.List<FileTaskEntity> runningTasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
java.util.List<FileTaskEntity> runningTasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||||
.eq(FileTaskEntity::getModuleType, MODULE_TYPE_SHOP_MATCH)
|
.eq(FileTaskEntity::getModuleType, MODULE_TYPE_SHOP_MATCH)
|
||||||
.eq(FileTaskEntity::getStatus, "RUNNING")
|
.eq(FileTaskEntity::getStatus, "RUNNING")
|
||||||
@@ -421,8 +398,8 @@ public class DeleteBrandStaleTaskService {
|
|||||||
}
|
}
|
||||||
Map<Long, Long> heartbeatByTaskId = shopMatchTaskCacheService.getTaskHeartbeatMillisBatch(
|
Map<Long, Long> heartbeatByTaskId = shopMatchTaskCacheService.getTaskHeartbeatMillisBatch(
|
||||||
runningTasks.stream().map(FileTaskEntity::getId).toList());
|
runningTasks.stream().map(FileTaskEntity::getId).toList());
|
||||||
log.info("[stale-check] shop-match candidates={} threshold={} timeoutMinutes={} initialThreshold={} initialTimeoutMinutes={}",
|
log.info("[stale-check] shop-match candidates={} threshold={} timeoutMinutes={}",
|
||||||
runningTasks.size(), threshold, minutes, initialThreshold, initialMinutes);
|
runningTasks.size(), threshold, minutes);
|
||||||
for (FileTaskEntity task : runningTasks) {
|
for (FileTaskEntity task : runningTasks) {
|
||||||
long lastHeartbeatMillis = heartbeatByTaskId.getOrDefault(task.getId(), 0L);
|
long lastHeartbeatMillis = heartbeatByTaskId.getOrDefault(task.getId(), 0L);
|
||||||
boolean hasFinishedRows = (task.getSuccessFileCount() != null && task.getSuccessFileCount() > 0)
|
boolean hasFinishedRows = (task.getSuccessFileCount() != null && task.getSuccessFileCount() > 0)
|
||||||
@@ -445,12 +422,6 @@ public class DeleteBrandStaleTaskService {
|
|||||||
taskFileJobService.countActiveAssembleJobs(task.getId(), MODULE_TYPE_SHOP_MATCH));
|
taskFileJobService.countActiveAssembleJobs(task.getId(), MODULE_TYPE_SHOP_MATCH));
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if (!hasStartedProgress && isWithinInitialGrace(task, initialThreshold)) {
|
|
||||||
stats.skippedTaskCount++;
|
|
||||||
log.info("[stale-check] shop-match skip initial-grace taskId={} createdAt={} initialThreshold={}",
|
|
||||||
task.getId(), task.getCreatedAt(), initialThreshold);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
TaskDistributedLockService.LockHandle taskLockHandle = acquireTaskLock(MODULE_TYPE_SHOP_MATCH, task.getId());
|
TaskDistributedLockService.LockHandle taskLockHandle = acquireTaskLock(MODULE_TYPE_SHOP_MATCH, task.getId());
|
||||||
if (taskLockHandle == null) {
|
if (taskLockHandle == null) {
|
||||||
stats.skippedTaskCount++;
|
stats.skippedTaskCount++;
|
||||||
@@ -489,12 +460,10 @@ public class DeleteBrandStaleTaskService {
|
|||||||
private ShopMatchStaleCheckStats failStalePatrolDeleteTasks() {
|
private ShopMatchStaleCheckStats failStalePatrolDeleteTasks() {
|
||||||
ShopMatchStaleCheckStats stats = new ShopMatchStaleCheckStats();
|
ShopMatchStaleCheckStats stats = new ShopMatchStaleCheckStats();
|
||||||
long minutes = Math.max(1L, deleteBrandProgressProperties.getPatrolDeleteStaleTimeoutMinutes());
|
long minutes = Math.max(1L, deleteBrandProgressProperties.getPatrolDeleteStaleTimeoutMinutes());
|
||||||
long initialMinutes = Math.max(minutes, deleteBrandProgressProperties.getPatrolDeleteInitialTimeoutMinutes());
|
|
||||||
long staleTimeoutMillis = Duration.ofMinutes(minutes).toMillis();
|
long staleTimeoutMillis = Duration.ofMinutes(minutes).toMillis();
|
||||||
long nowMillis = System.currentTimeMillis();
|
long nowMillis = System.currentTimeMillis();
|
||||||
LocalDateTime now = LocalDateTime.now();
|
LocalDateTime now = LocalDateTime.now();
|
||||||
LocalDateTime threshold = now.minusMinutes(minutes);
|
LocalDateTime threshold = now.minusMinutes(minutes);
|
||||||
LocalDateTime initialThreshold = now.minusMinutes(initialMinutes);
|
|
||||||
java.util.List<FileTaskEntity> runningTasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
java.util.List<FileTaskEntity> runningTasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||||
.eq(FileTaskEntity::getModuleType, MODULE_TYPE_PATROL_DELETE)
|
.eq(FileTaskEntity::getModuleType, MODULE_TYPE_PATROL_DELETE)
|
||||||
.eq(FileTaskEntity::getStatus, "RUNNING")
|
.eq(FileTaskEntity::getStatus, "RUNNING")
|
||||||
@@ -526,10 +495,6 @@ public class DeleteBrandStaleTaskService {
|
|||||||
taskFileJobService.countActiveAssembleJobs(task.getId(), MODULE_TYPE_PATROL_DELETE));
|
taskFileJobService.countActiveAssembleJobs(task.getId(), MODULE_TYPE_PATROL_DELETE));
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if (!hasStartedProgress && isWithinInitialGrace(task, initialThreshold)) {
|
|
||||||
stats.skippedTaskCount++;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
TaskDistributedLockService.LockHandle taskLockHandle = acquireTaskLock(MODULE_TYPE_PATROL_DELETE, task.getId());
|
TaskDistributedLockService.LockHandle taskLockHandle = acquireTaskLock(MODULE_TYPE_PATROL_DELETE, task.getId());
|
||||||
if (taskLockHandle == null) {
|
if (taskLockHandle == null) {
|
||||||
stats.skippedTaskCount++;
|
stats.skippedTaskCount++;
|
||||||
@@ -566,13 +531,11 @@ public class DeleteBrandStaleTaskService {
|
|||||||
|
|
||||||
private ShopMatchStaleCheckStats failStaleQueryAsinTasks() {
|
private ShopMatchStaleCheckStats failStaleQueryAsinTasks() {
|
||||||
ShopMatchStaleCheckStats stats = new ShopMatchStaleCheckStats();
|
ShopMatchStaleCheckStats stats = new ShopMatchStaleCheckStats();
|
||||||
long minutes = Math.max(1L, deleteBrandProgressProperties.getPatrolDeleteStaleTimeoutMinutes());
|
long minutes = Math.max(1L, deleteBrandProgressProperties.getQueryAsinStaleTimeoutMinutes());
|
||||||
long initialMinutes = Math.max(minutes, deleteBrandProgressProperties.getPatrolDeleteInitialTimeoutMinutes());
|
|
||||||
long staleTimeoutMillis = Duration.ofMinutes(minutes).toMillis();
|
long staleTimeoutMillis = Duration.ofMinutes(minutes).toMillis();
|
||||||
long nowMillis = System.currentTimeMillis();
|
long nowMillis = System.currentTimeMillis();
|
||||||
LocalDateTime now = LocalDateTime.now();
|
LocalDateTime now = LocalDateTime.now();
|
||||||
LocalDateTime threshold = now.minusMinutes(minutes);
|
LocalDateTime threshold = now.minusMinutes(minutes);
|
||||||
LocalDateTime initialThreshold = now.minusMinutes(initialMinutes);
|
|
||||||
java.util.List<FileTaskEntity> runningTasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
java.util.List<FileTaskEntity> runningTasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||||
.eq(FileTaskEntity::getModuleType, MODULE_TYPE_QUERY_ASIN)
|
.eq(FileTaskEntity::getModuleType, MODULE_TYPE_QUERY_ASIN)
|
||||||
.eq(FileTaskEntity::getStatus, "RUNNING")
|
.eq(FileTaskEntity::getStatus, "RUNNING")
|
||||||
@@ -604,10 +567,6 @@ public class DeleteBrandStaleTaskService {
|
|||||||
taskFileJobService.countActiveAssembleJobs(task.getId(), MODULE_TYPE_QUERY_ASIN));
|
taskFileJobService.countActiveAssembleJobs(task.getId(), MODULE_TYPE_QUERY_ASIN));
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if (!hasStartedProgress && isWithinInitialGrace(task, initialThreshold)) {
|
|
||||||
stats.skippedTaskCount++;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
TaskDistributedLockService.LockHandle taskLockHandle = acquireTaskLock(MODULE_TYPE_QUERY_ASIN, task.getId());
|
TaskDistributedLockService.LockHandle taskLockHandle = acquireTaskLock(MODULE_TYPE_QUERY_ASIN, task.getId());
|
||||||
if (taskLockHandle == null) {
|
if (taskLockHandle == null) {
|
||||||
stats.skippedTaskCount++;
|
stats.skippedTaskCount++;
|
||||||
@@ -643,14 +602,6 @@ public class DeleteBrandStaleTaskService {
|
|||||||
return stats;
|
return stats;
|
||||||
}
|
}
|
||||||
|
|
||||||
private boolean isWithinInitialGrace(FileTaskEntity task, LocalDateTime initialThreshold) {
|
|
||||||
if (task == null || initialThreshold == null) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
LocalDateTime baseline = task.getUpdatedAt() != null ? task.getUpdatedAt() : task.getCreatedAt();
|
|
||||||
return baseline != null && baseline.isAfter(initialThreshold);
|
|
||||||
}
|
|
||||||
|
|
||||||
private TaskDistributedLockService.LockHandle acquireTaskLock(String moduleType, Long taskId) {
|
private TaskDistributedLockService.LockHandle acquireTaskLock(String moduleType, Long taskId) {
|
||||||
TaskDistributedLockService.LockHandle lockHandle = taskDistributedLockService.acquire(moduleType, taskId, 0L);
|
TaskDistributedLockService.LockHandle lockHandle = taskDistributedLockService.acquire(moduleType, taskId, 0L);
|
||||||
if (lockHandle == null) {
|
if (lockHandle == null) {
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ import org.springframework.stereotype.Component;
|
|||||||
import org.springframework.util.StreamUtils;
|
import org.springframework.util.StreamUtils;
|
||||||
import org.springframework.web.client.RestClient;
|
import org.springframework.web.client.RestClient;
|
||||||
|
|
||||||
|
import java.math.BigDecimal;
|
||||||
import java.nio.charset.StandardCharsets;
|
import java.nio.charset.StandardCharsets;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.LinkedHashMap;
|
import java.util.LinkedHashMap;
|
||||||
@@ -370,8 +371,9 @@ public class SimilarAsinCozeClient {
|
|||||||
List<String> skus = rows.stream().map(row -> safeText(row.getSku())).toList();
|
List<String> skus = rows.stream().map(row -> safeText(row.getSku())).toList();
|
||||||
List<String> urls = rows.stream().map(this::primaryImageUrl).toList();
|
List<String> urls = rows.stream().map(this::primaryImageUrl).toList();
|
||||||
List<List<String>> urlLists = rows.stream().map(this::imageUrls).toList();
|
List<List<String>> urlLists = rows.stream().map(this::imageUrls).toList();
|
||||||
|
List<List<Map<String, Object>>> alibabaLists = rows.stream().map(this::alibabaItems).toList();
|
||||||
|
|
||||||
List<Map<String, Object>> items = buildItemObjects(asins, titles, skus, urls, urlLists);
|
List<Map<String, Object>> items = buildItemObjects(asins, titles, skus, urls, urlLists, alibabaLists);
|
||||||
logCozeItemsDiff(rows, items);
|
logCozeItemsDiff(rows, items);
|
||||||
Map<String, Object> parameters = new LinkedHashMap<>();
|
Map<String, Object> parameters = new LinkedHashMap<>();
|
||||||
parameters.put("items", items);
|
parameters.put("items", items);
|
||||||
@@ -408,22 +410,26 @@ public class SimilarAsinCozeClient {
|
|||||||
List<String> rowUrls = safeUrls(row.getUrls());
|
List<String> rowUrls = safeUrls(row.getUrls());
|
||||||
String rowUrl = safeText(row.getUrl());
|
String rowUrl = safeText(row.getUrl());
|
||||||
Object outUrl = item.get("url");
|
Object outUrl = item.get("url");
|
||||||
|
Object outAlibaba = item.get("alibaba");
|
||||||
Object outTargetUrls = item.get("target_urls");
|
Object outTargetUrls = item.get("target_urls");
|
||||||
int outTargetSize = (outTargetUrls instanceof List<?> list) ? list.size() : 0;
|
int outTargetSize = (outTargetUrls instanceof List<?> list) ? list.size() : 0;
|
||||||
|
int outAlibabaSize = (outAlibaba instanceof List<?> list) ? list.size() : 0;
|
||||||
boolean urlMatch = String.valueOf(outUrl == null ? "" : outUrl).equals(rowUrl);
|
boolean urlMatch = String.valueOf(outUrl == null ? "" : outUrl).equals(rowUrl);
|
||||||
boolean targetMatch = (outTargetUrls instanceof List<?> outList)
|
boolean targetMatch = (outTargetUrls instanceof List<?> outList)
|
||||||
&& outList.size() == rowUrls.size()
|
&& outList.size() == rowUrls.size()
|
||||||
&& outList.equals(rowUrls);
|
&& outList.equals(rowUrls);
|
||||||
log.info("[similar-asin] coze item idx={} asin={} title={} sku={} rowUrl={} rowUrlsSize={} rowUrlsHead={} rowUrlsTail={} outUrl={} outTargetSize={} outTargetHead={} outTargetTail={} urlMatch={} targetUrlsMatch={}",
|
log.info("[similar-asin] coze item idx={} asin={} title={} sku={} price={} rowUrl={} rowUrlsSize={} rowUrlsHead={} rowUrlsTail={} outUrl={} outAlibabaSize={} outTargetSize={} outTargetHead={} outTargetTail={} urlMatch={} targetUrlsMatch={}",
|
||||||
i,
|
i,
|
||||||
safeText(row.getAsin()),
|
safeText(row.getAsin()),
|
||||||
abbreviate(safeText(row.getTitle()), 80),
|
abbreviate(safeText(row.getTitle()), 80),
|
||||||
safeText(row.getSku()),
|
safeText(row.getSku()),
|
||||||
|
safeText(row.getPrice()),
|
||||||
abbreviate(rowUrl, 200),
|
abbreviate(rowUrl, 200),
|
||||||
rowUrls.size(),
|
rowUrls.size(),
|
||||||
rowUrls.isEmpty() ? "" : abbreviate(rowUrls.get(0), 200),
|
rowUrls.isEmpty() ? "" : abbreviate(rowUrls.get(0), 200),
|
||||||
rowUrls.size() <= 1 ? "" : abbreviate(rowUrls.get(rowUrls.size() - 1), 200),
|
rowUrls.size() <= 1 ? "" : abbreviate(rowUrls.get(rowUrls.size() - 1), 200),
|
||||||
abbreviate(String.valueOf(outUrl == null ? "" : outUrl), 200),
|
abbreviate(String.valueOf(outUrl == null ? "" : outUrl), 200),
|
||||||
|
outAlibabaSize,
|
||||||
outTargetSize,
|
outTargetSize,
|
||||||
(outTargetUrls instanceof List<?> headList && !headList.isEmpty())
|
(outTargetUrls instanceof List<?> headList && !headList.isEmpty())
|
||||||
? abbreviate(String.valueOf(headList.get(0)), 200) : "",
|
? abbreviate(String.valueOf(headList.get(0)), 200) : "",
|
||||||
@@ -468,7 +474,8 @@ public class SimilarAsinCozeClient {
|
|||||||
List<String> titles,
|
List<String> titles,
|
||||||
List<String> skus,
|
List<String> skus,
|
||||||
List<String> urls,
|
List<String> urls,
|
||||||
List<List<String>> urlLists) {
|
List<List<String>> urlLists,
|
||||||
|
List<List<Map<String, Object>>> alibabaLists) {
|
||||||
// legacy flag:保留切回旧字段顺序 {asin, sku, url, target_urls, title} 的开关,
|
// legacy flag:保留切回旧字段顺序 {asin, sku, url, target_urls, title} 的开关,
|
||||||
// 默认 false 使用当前顺序 {asin, url, target_urls, title, sku}。
|
// 默认 false 使用当前顺序 {asin, url, target_urls, title, sku}。
|
||||||
boolean useLegacyOrder = properties.isCozeUseLegacyItemFieldOrder();
|
boolean useLegacyOrder = properties.isCozeUseLegacyItemFieldOrder();
|
||||||
@@ -476,16 +483,19 @@ public class SimilarAsinCozeClient {
|
|||||||
for (int i = 0; i < asins.size(); i++) {
|
for (int i = 0; i < asins.size(); i++) {
|
||||||
String url = urls.get(i);
|
String url = urls.get(i);
|
||||||
List<String> imageUrls = urlLists.get(i);
|
List<String> imageUrls = urlLists.get(i);
|
||||||
|
List<Map<String, Object>> alibabaItems = alibabaLists.get(i);
|
||||||
Map<String, Object> item = new LinkedHashMap<>();
|
Map<String, Object> item = new LinkedHashMap<>();
|
||||||
if (useLegacyOrder) {
|
if (useLegacyOrder) {
|
||||||
item.put("asin", asins.get(i));
|
item.put("asin", asins.get(i));
|
||||||
item.put("sku", skus.get(i));
|
item.put("sku", skus.get(i));
|
||||||
item.put("url", url);
|
item.put("url", url);
|
||||||
|
item.put("alibaba", alibabaItems);
|
||||||
item.put("target_urls", imageUrls);
|
item.put("target_urls", imageUrls);
|
||||||
item.put("title", titles.get(i));
|
item.put("title", titles.get(i));
|
||||||
} else {
|
} else {
|
||||||
item.put("asin", asins.get(i));
|
item.put("asin", asins.get(i));
|
||||||
item.put("url", url);
|
item.put("url", url);
|
||||||
|
item.put("alibaba", alibabaItems);
|
||||||
item.put("target_urls", imageUrls);
|
item.put("target_urls", imageUrls);
|
||||||
item.put("title", titles.get(i));
|
item.put("title", titles.get(i));
|
||||||
item.put("sku", skus.get(i));
|
item.put("sku", skus.get(i));
|
||||||
@@ -726,6 +736,7 @@ public class SimilarAsinCozeClient {
|
|||||||
row.setSku(source.getSku());
|
row.setSku(source.getSku());
|
||||||
row.setPrice(source.getPrice());
|
row.setPrice(source.getPrice());
|
||||||
row.setUrls(source.getUrls());
|
row.setUrls(source.getUrls());
|
||||||
|
row.setAlibaba(source.getAlibaba());
|
||||||
row.setTitle(source.getTitle());
|
row.setTitle(source.getTitle());
|
||||||
row.setError(source.getError());
|
row.setError(source.getError());
|
||||||
row.setDone(source.getDone());
|
row.setDone(source.getDone());
|
||||||
@@ -1213,6 +1224,69 @@ public class SimilarAsinCozeClient {
|
|||||||
return primaryUrl.isBlank() ? List.of() : List.of(primaryUrl);
|
return primaryUrl.isBlank() ? List.of() : List.of(primaryUrl);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private List<Map<String, Object>> alibabaItems(SimilarAsinResultRowDto row) {
|
||||||
|
if (row == null) {
|
||||||
|
return List.of();
|
||||||
|
}
|
||||||
|
List<SimilarAsinResultRowDto.AlibabaItem> explicitItems = row.getAlibaba();
|
||||||
|
if (explicitItems != null && !explicitItems.isEmpty()) {
|
||||||
|
List<Map<String, Object>> result = new ArrayList<>(explicitItems.size());
|
||||||
|
for (SimilarAsinResultRowDto.AlibabaItem source : explicitItems) {
|
||||||
|
if (source == null) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
String sourceUrl = safeText(source.getUrl());
|
||||||
|
Object sourcePrice = cozePrice(source.getRawPrice());
|
||||||
|
if (sourceUrl.isBlank() && isBlankPrice(sourcePrice)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
Map<String, Object> item = new LinkedHashMap<>();
|
||||||
|
item.put("url", sourceUrl);
|
||||||
|
item.put("price", sourcePrice);
|
||||||
|
result.add(item);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
List<String> imageUrls = imageUrls(row);
|
||||||
|
if (imageUrls.isEmpty()) {
|
||||||
|
return List.of();
|
||||||
|
}
|
||||||
|
Object rowPrice = cozePrice(row.getPrice());
|
||||||
|
List<Map<String, Object>> result = new ArrayList<>(imageUrls.size());
|
||||||
|
for (String imageUrl : imageUrls) {
|
||||||
|
Map<String, Object> item = new LinkedHashMap<>();
|
||||||
|
item.put("url", imageUrl);
|
||||||
|
item.put("price", rowPrice);
|
||||||
|
result.add(item);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
private Object cozePrice(Object value) {
|
||||||
|
if (value == null) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
if (value instanceof BigDecimal decimal) {
|
||||||
|
return decimal.stripTrailingZeros();
|
||||||
|
}
|
||||||
|
if (value instanceof Number number) {
|
||||||
|
return new BigDecimal(number.toString()).stripTrailingZeros();
|
||||||
|
}
|
||||||
|
String text = safeText(String.valueOf(value));
|
||||||
|
if (text.isBlank()) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return new BigDecimal(text).stripTrailingZeros();
|
||||||
|
} catch (NumberFormatException ex) {
|
||||||
|
return text;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean isBlankPrice(Object value) {
|
||||||
|
return value == null || value instanceof String text && text.isBlank();
|
||||||
|
}
|
||||||
|
|
||||||
private String cozeGroupKey(SimilarAsinResultRowDto row) {
|
private String cozeGroupKey(SimilarAsinResultRowDto row) {
|
||||||
if (row == null) {
|
if (row == null) {
|
||||||
return "";
|
return "";
|
||||||
|
|||||||
@@ -44,6 +44,10 @@ public class SimilarAsinResultRowDto {
|
|||||||
@Schema(description = "商品价格。来自 Excel 解析或 Python 回传。", example = "12.29")
|
@Schema(description = "商品价格。来自 Excel 解析或 Python 回传。", example = "12.29")
|
||||||
private String price;
|
private String price;
|
||||||
|
|
||||||
|
@JsonAlias({"alibaba", "alibaba_items", "alibabaItems", "alibaba_products", "alibabaProducts"})
|
||||||
|
@Schema(description = "Alibaba candidates returned by Python, each item contains url and price")
|
||||||
|
private List<AlibabaItem> alibaba = new ArrayList<>();
|
||||||
|
|
||||||
@Schema(description = "商品主图 URL。Python 端解析的代表图。允许传字符串或数组(数组取首个非空),与 urls 互不影响。", example = "https://webstatic.aiproxy.vip/output/20260425/103322/demo.jpg")
|
@Schema(description = "商品主图 URL。Python 端解析的代表图。允许传字符串或数组(数组取首个非空),与 urls 互不影响。", example = "https://webstatic.aiproxy.vip/output/20260425/103322/demo.jpg")
|
||||||
private String url;
|
private String url;
|
||||||
|
|
||||||
@@ -150,7 +154,11 @@ public class SimilarAsinResultRowDto {
|
|||||||
|
|
||||||
public List<String> getUrls() {
|
public List<String> getUrls() {
|
||||||
// 直接返回 Python 回传的 urls,不再把 url 强推到首位、不去重、不截断。
|
// 直接返回 Python 回传的 urls,不再把 url 强推到首位、不去重、不截断。
|
||||||
return cleanUrls(urls);
|
List<String> cleaned = cleanUrls(urls);
|
||||||
|
if (!cleaned.isEmpty()) {
|
||||||
|
return cleaned;
|
||||||
|
}
|
||||||
|
return alibabaUrls();
|
||||||
}
|
}
|
||||||
|
|
||||||
@JsonSetter("urls")
|
@JsonSetter("urls")
|
||||||
@@ -169,12 +177,30 @@ public class SimilarAsinResultRowDto {
|
|||||||
this.urls = normalized;
|
this.urls = normalized;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public List<AlibabaItem> getAlibaba() {
|
||||||
|
if (alibaba == null || alibaba.isEmpty()) {
|
||||||
|
return new ArrayList<>();
|
||||||
|
}
|
||||||
|
List<AlibabaItem> result = new ArrayList<>(alibaba.size());
|
||||||
|
for (AlibabaItem item : alibaba) {
|
||||||
|
if (item == null || item.getUrl().isBlank() && item.getPrice().isBlank()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
result.add(item);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setAlibaba(List<AlibabaItem> alibaba) {
|
||||||
|
this.alibaba = alibaba == null ? new ArrayList<>() : new ArrayList<>(alibaba);
|
||||||
|
}
|
||||||
|
|
||||||
public boolean hasImageUrl() {
|
public boolean hasImageUrl() {
|
||||||
// url(主图)和 urls(同类商品图)任一存在即可作为可送 Coze 的素材。
|
// url(主图)和 urls(同类商品图)任一存在即可作为可送 Coze 的素材。
|
||||||
if (url != null && !url.isBlank()) {
|
if (url != null && !url.isBlank()) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
return !cleanUrls(urls).isEmpty();
|
return !getUrls().isEmpty();
|
||||||
}
|
}
|
||||||
|
|
||||||
private static void appendUrls(List<String> result, Object value) {
|
private static void appendUrls(List<String> result, Object value) {
|
||||||
@@ -227,4 +253,44 @@ public class SimilarAsinResultRowDto {
|
|||||||
}
|
}
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private List<String> alibabaUrls() {
|
||||||
|
List<AlibabaItem> items = getAlibaba();
|
||||||
|
if (items.isEmpty()) {
|
||||||
|
return new ArrayList<>();
|
||||||
|
}
|
||||||
|
List<String> result = new ArrayList<>(items.size());
|
||||||
|
for (AlibabaItem item : items) {
|
||||||
|
String itemUrl = item.getUrl();
|
||||||
|
if (!itemUrl.isBlank()) {
|
||||||
|
result.add(itemUrl);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@Schema(description = "Alibaba candidate")
|
||||||
|
public static class AlibabaItem {
|
||||||
|
|
||||||
|
@JsonAlias({"url", "image_url", "imageUrl", "img_url", "imgUrl", "link"})
|
||||||
|
@Schema(description = "Alibaba image or product URL")
|
||||||
|
private String url;
|
||||||
|
|
||||||
|
@JsonAlias({"price", "浠锋牸"})
|
||||||
|
@Schema(description = "Alibaba candidate price")
|
||||||
|
private Object price;
|
||||||
|
|
||||||
|
public String getUrl() {
|
||||||
|
return url == null ? "" : url.trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getPrice() {
|
||||||
|
return price == null ? "" : String.valueOf(price).trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
public Object getRawPrice() {
|
||||||
|
return price;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
|
|||||||
import com.fasterxml.jackson.databind.node.ArrayNode;
|
import com.fasterxml.jackson.databind.node.ArrayNode;
|
||||||
import com.fasterxml.jackson.databind.node.ObjectNode;
|
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||||
import com.nanri.aiimage.common.exception.BusinessException;
|
import com.nanri.aiimage.common.exception.BusinessException;
|
||||||
|
import com.nanri.aiimage.common.exception.TaskOwnerMismatchException;
|
||||||
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||||
import com.nanri.aiimage.common.util.CozeGroupResultPropagator;
|
import com.nanri.aiimage.common.util.CozeGroupResultPropagator;
|
||||||
import com.nanri.aiimage.common.util.FailedStatusRowFilter;
|
import com.nanri.aiimage.common.util.FailedStatusRowFilter;
|
||||||
@@ -205,6 +206,8 @@ public class SimilarAsinTaskService {
|
|||||||
"id",
|
"id",
|
||||||
"asin",
|
"asin",
|
||||||
"国家",
|
"国家",
|
||||||
|
"卖家名称",
|
||||||
|
"品牌",
|
||||||
"是否有货",
|
"是否有货",
|
||||||
"相似度",
|
"相似度",
|
||||||
"是否符合类目",
|
"是否符合类目",
|
||||||
@@ -216,9 +219,9 @@ public class SimilarAsinTaskService {
|
|||||||
"阿里巴巴图片2"
|
"阿里巴巴图片2"
|
||||||
);
|
);
|
||||||
|
|
||||||
private static final int IMG_COL_MAIN = 9;
|
private static final int IMG_COL_MAIN = 11;
|
||||||
private static final int IMG_COL_PUZZLE1 = 10;
|
private static final int IMG_COL_PUZZLE1 = 12;
|
||||||
private static final int IMG_COL_PUZZLE2 = 11;
|
private static final int IMG_COL_PUZZLE2 = 13;
|
||||||
|
|
||||||
private final LocalFileStorageService localFileStorageService;
|
private final LocalFileStorageService localFileStorageService;
|
||||||
private final OssStorageService ossStorageService;
|
private final OssStorageService ossStorageService;
|
||||||
@@ -471,6 +474,7 @@ public class SimilarAsinTaskService {
|
|||||||
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || userId != null && !Objects.equals(userId, task.getUserId())) {
|
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || userId != null && !Objects.equals(userId, task.getUserId())) {
|
||||||
throw new BusinessException("任务不存在");
|
throw new BusinessException("任务不存在");
|
||||||
}
|
}
|
||||||
|
ensureTaskOwnedByCurrentInstance(task, "activate");
|
||||||
if (STATUS_SUCCESS.equals(task.getStatus()) || STATUS_FAILED.equals(task.getStatus())) {
|
if (STATUS_SUCCESS.equals(task.getStatus()) || STATUS_FAILED.equals(task.getStatus())) {
|
||||||
throw new BusinessException("任务已结束");
|
throw new BusinessException("任务已结束");
|
||||||
}
|
}
|
||||||
@@ -483,8 +487,11 @@ public class SimilarAsinTaskService {
|
|||||||
|
|
||||||
public SimilarAsinDashboardVo dashboard(Long userId) {
|
public SimilarAsinDashboardVo dashboard(Long userId) {
|
||||||
SimilarAsinDashboardVo vo = new SimilarAsinDashboardVo();
|
SimilarAsinDashboardVo vo = new SimilarAsinDashboardVo();
|
||||||
vo.setPendingTaskCount(countTask(userId, STATUS_RUNNING));
|
long completedWithFile = countCompletedTasksWithResultFile(userId);
|
||||||
vo.setSuccessTaskCount(countTask(userId, STATUS_SUCCESS));
|
long rawRunning = countTask(userId, STATUS_RUNNING);
|
||||||
|
long rawSuccess = countTask(userId, STATUS_SUCCESS);
|
||||||
|
vo.setPendingTaskCount(Math.max(0L, rawRunning - completedWithFile));
|
||||||
|
vo.setSuccessTaskCount(rawSuccess + completedWithFile);
|
||||||
vo.setFailedTaskCount(countTask(userId, STATUS_FAILED));
|
vo.setFailedTaskCount(countTask(userId, STATUS_FAILED));
|
||||||
vo.setProcessedTaskCount(vo.getSuccessTaskCount() + vo.getFailedTaskCount());
|
vo.setProcessedTaskCount(vo.getSuccessTaskCount() + vo.getFailedTaskCount());
|
||||||
return vo;
|
return vo;
|
||||||
@@ -802,7 +809,8 @@ public class SimilarAsinTaskService {
|
|||||||
|
|
||||||
public void finalizeStaleTasks() {
|
public void finalizeStaleTasks() {
|
||||||
if (transactionManager != null) {
|
if (transactionManager != null) {
|
||||||
LocalDateTime threshold = LocalDateTime.now().minusMinutes(Math.max(1, properties.getStaleTimeoutMinutes()));
|
LocalDateTime now = LocalDateTime.now();
|
||||||
|
LocalDateTime threshold = now.minusMinutes(Math.max(1, properties.getStaleTimeoutMinutes()));
|
||||||
long thresholdMillis = threshold.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli();
|
long thresholdMillis = threshold.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli();
|
||||||
List<FileTaskEntity> tasks = listStaleFinalizeCandidates(threshold);
|
List<FileTaskEntity> tasks = listStaleFinalizeCandidates(threshold);
|
||||||
for (FileTaskEntity task : tasks) {
|
for (FileTaskEntity task : tasks) {
|
||||||
@@ -826,7 +834,8 @@ public class SimilarAsinTaskService {
|
|||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
LocalDateTime threshold = LocalDateTime.now().minusMinutes(Math.max(1, properties.getStaleTimeoutMinutes()));
|
LocalDateTime now = LocalDateTime.now();
|
||||||
|
LocalDateTime threshold = now.minusMinutes(Math.max(1, properties.getStaleTimeoutMinutes()));
|
||||||
long thresholdMillis = threshold.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli();
|
long thresholdMillis = threshold.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli();
|
||||||
List<FileTaskEntity> tasks = listStaleFinalizeCandidates(threshold);
|
List<FileTaskEntity> tasks = listStaleFinalizeCandidates(threshold);
|
||||||
for (FileTaskEntity task : tasks) {
|
for (FileTaskEntity task : tasks) {
|
||||||
@@ -877,8 +886,8 @@ public class SimilarAsinTaskService {
|
|||||||
return fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
return fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||||
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
|
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
|
||||||
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
|
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
|
||||||
.lt(FileTaskEntity::getCreatedAt, threshold)
|
.lt(FileTaskEntity::getUpdatedAt, threshold)
|
||||||
.orderByAsc(FileTaskEntity::getCreatedAt)
|
.orderByAsc(FileTaskEntity::getUpdatedAt)
|
||||||
.last("limit 200"));
|
.last("limit 200"));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1068,22 +1077,6 @@ public class SimilarAsinTaskService {
|
|||||||
boolean uploadComplete = isResultSubmissionComplete(taskId);
|
boolean uploadComplete = isResultSubmissionComplete(taskId);
|
||||||
long pendingCozeStates = countPendingCozeStates(taskId);
|
long pendingCozeStates = countPendingCozeStates(taskId);
|
||||||
long activeAssembleJobs = taskFileJobService.countActiveAssembleJobs(taskId, MODULE_TYPE);
|
long activeAssembleJobs = taskFileJobService.countActiveAssembleJobs(taskId, MODULE_TYPE);
|
||||||
if (isJavaSideProcessing(taskId)) {
|
|
||||||
// 防止 Coze 永远 pending 时 stale-recovery 永久 defer:
|
|
||||||
// 超过 stale-timeout-minutes × 4 仍未推进的 RUNNING 任务,强制走 finalize 链路。
|
|
||||||
long deferCeilingMinutes = Math.max(1, properties.getStaleTimeoutMinutes()) * 4L;
|
|
||||||
LocalDateTime updatedAt = task.getUpdatedAt();
|
|
||||||
if (updatedAt != null
|
|
||||||
&& Duration.between(updatedAt, LocalDateTime.now()).toMinutes() >= deferCeilingMinutes) {
|
|
||||||
log.warn("[similar-asin] stale recovery defer ceiling exceeded, forcing finalize taskId={} updatedAt={} ceilingMinutes={} pendingCozeStates={} activeAssembleJobs={}",
|
|
||||||
taskId, updatedAt, deferCeilingMinutes, pendingCozeStates, activeAssembleJobs);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
touchJavaSideTaskActivity(taskId);
|
|
||||||
log.info("[similar-asin] stale recovery deferred because Java-side processing is still active taskId={} uploadComplete={} pendingCozeStates={} activeAssembleJobs={}",
|
|
||||||
taskId, uploadComplete, pendingCozeStates, activeAssembleJobs);
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
if (!hasPersistedResultRows(taskId)) {
|
if (!hasPersistedResultRows(taskId)) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
@@ -1092,10 +1085,10 @@ public class SimilarAsinTaskService {
|
|||||||
if (forcedScopes <= 0) {
|
if (forcedScopes <= 0) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
log.warn("[similar-asin] python heartbeat timed out, forcing finalize pipeline taskId={} pendingCozeStates={} activeAssembleJobs={} forcedScopes={}",
|
log.warn("[similar-asin] Python 回传心跳超时,已封口上传并继续 Coze/文件收尾 taskId={} pendingCozeStates={} activeAssembleJobs={} forcedScopes={}",
|
||||||
taskId, pendingCozeStates, activeAssembleJobs, forcedScopes);
|
taskId, pendingCozeStates, activeAssembleJobs, forcedScopes);
|
||||||
} else {
|
} else {
|
||||||
log.warn("[similar-asin] stale running task resuming coze/file assembly after python timeout taskId={} pendingCozeStates={} activeAssembleJobs={}",
|
log.warn("[similar-asin] Python 超时恢复继续推进 Coze/文件收尾 taskId={} pendingCozeStates={} activeAssembleJobs={}",
|
||||||
taskId, pendingCozeStates, activeAssembleJobs);
|
taskId, pendingCozeStates, activeAssembleJobs);
|
||||||
}
|
}
|
||||||
submitCozeForSubmittedChunk(new SubmitContext(task, null, null, null, true, null));
|
submitCozeForSubmittedChunk(new SubmitContext(task, null, null, null, true, null));
|
||||||
@@ -1269,32 +1262,48 @@ public class SimilarAsinTaskService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private void mergeChunkPayload(Long taskId, String scopeHash, Integer chunkIndex, List<SimilarAsinResultRowDto> rows) {
|
private void mergeChunkPayload(Long taskId, String scopeHash, Integer chunkIndex, List<SimilarAsinResultRowDto> rows) {
|
||||||
TaskChunkEntity chunk = taskChunkMapper.selectOne(new LambdaQueryWrapper<TaskChunkEntity>()
|
if (rows == null || rows.isEmpty()) {
|
||||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
|
||||||
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
|
|
||||||
.eq(TaskChunkEntity::getScopeHash, scopeHash)
|
|
||||||
.eq(TaskChunkEntity::getChunkIndex, chunkIndex)
|
|
||||||
.last("limit 1"));
|
|
||||||
if (chunk == null || rows == null || rows.isEmpty()) {
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
Map<String, SimilarAsinResultRowDto> persistedRows = readChunkRows(chunk);
|
int maxAttempts = 3;
|
||||||
for (SimilarAsinResultRowDto row : rows) {
|
for (int attempt = 1; attempt <= maxAttempts; attempt++) {
|
||||||
persistedRows.put(rowKey(row), row);
|
TaskChunkEntity chunk = taskChunkMapper.selectOne(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||||
}
|
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||||
String payloadJson = writeJson(rows == null ? List.of() : rows, "结果序列化失败");
|
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE)
|
||||||
payloadJson = writeJson(new ArrayList<>(persistedRows.values()), "相似ASIN分片载荷合并失败");
|
.eq(TaskChunkEntity::getScopeHash, scopeHash)
|
||||||
String oldPayload = chunk.getPayloadJson();
|
.eq(TaskChunkEntity::getChunkIndex, chunkIndex)
|
||||||
String storedPayload = transientPayloadStorageService.storeChunkPayloadVersioned(MODULE_TYPE, taskId, scopeHash, chunkIndex, payloadJson);
|
.last("limit 1"));
|
||||||
chunk.setPayloadJson(storedPayload);
|
if (chunk == null) {
|
||||||
chunk.setPayloadHash(DigestUtil.sha256Hex(payloadJson));
|
return;
|
||||||
chunk.setUpdatedAt(LocalDateTime.now());
|
}
|
||||||
int updated = taskChunkMapper.updateById(chunk);
|
Map<String, SimilarAsinResultRowDto> persistedRows = readChunkRows(chunk);
|
||||||
if (updated <= 0) {
|
for (SimilarAsinResultRowDto row : rows) {
|
||||||
|
persistedRows.put(rowKey(row), row);
|
||||||
|
}
|
||||||
|
String payloadJson = writeJson(new ArrayList<>(persistedRows.values()), "相似ASIN分片载荷合并失败");
|
||||||
|
String oldPayload = chunk.getPayloadJson();
|
||||||
|
String oldPayloadHash = chunk.getPayloadHash();
|
||||||
|
String newPayloadHash = DigestUtil.sha256Hex(payloadJson);
|
||||||
|
String storedPayload = transientPayloadStorageService.storeChunkPayloadVersioned(MODULE_TYPE, taskId, scopeHash, chunkIndex, payloadJson);
|
||||||
|
int updated = taskChunkMapper.update(null, new LambdaUpdateWrapper<TaskChunkEntity>()
|
||||||
|
.eq(TaskChunkEntity::getId, chunk.getId())
|
||||||
|
.eq(TaskChunkEntity::getPayloadHash, oldPayloadHash)
|
||||||
|
.set(TaskChunkEntity::getPayloadJson, storedPayload)
|
||||||
|
.set(TaskChunkEntity::getPayloadHash, newPayloadHash)
|
||||||
|
.set(TaskChunkEntity::getUpdatedAt, LocalDateTime.now()));
|
||||||
|
if (updated > 0) {
|
||||||
|
log.debug("[similar-asin] chunk payload replaced taskId={} scopeHash={} chunk={} oldPayload={} newPayload={} attempt={}",
|
||||||
|
taskId, scopeHash, chunkIndex, oldPayload, storedPayload, attempt);
|
||||||
|
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(oldPayload, storedPayload);
|
||||||
|
return;
|
||||||
|
}
|
||||||
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
|
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
|
||||||
throw new IllegalStateException("相似ASIN分片载荷更新失败");
|
if (attempt < maxAttempts) {
|
||||||
|
log.warn("[similar-asin] chunk payload CAS conflict, retry merge taskId={} scopeHash={} chunk={} attempt={}/{}",
|
||||||
|
taskId, scopeHash, chunkIndex, attempt, maxAttempts);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(oldPayload, storedPayload);
|
throw new IllegalStateException("相似ASIN分片载荷更新失败");
|
||||||
}
|
}
|
||||||
|
|
||||||
private List<SimilarAsinResultRowDto> expandRows(List<SimilarAsinResultRowDto> rows,
|
private List<SimilarAsinResultRowDto> expandRows(List<SimilarAsinResultRowDto> rows,
|
||||||
@@ -1479,7 +1488,8 @@ public class SimilarAsinTaskService {
|
|||||||
}
|
}
|
||||||
String url = row.getUrl();
|
String url = row.getUrl();
|
||||||
List<String> urls = row.getUrls();
|
List<String> urls = row.getUrls();
|
||||||
log.info("[similar-asin] python inbound idx={} groupKey={} rowToken={} id={} asin={} country={} title={} sku={} url={} urlsSize={} urlsHead={} urlsTail={}",
|
List<SimilarAsinResultRowDto.AlibabaItem> alibaba = row.getAlibaba();
|
||||||
|
log.info("[similar-asin] python inbound idx={} groupKey={} rowToken={} id={} asin={} country={} title={} sku={} price={} url={} urlsSize={} alibabaSize={} urlsHead={} urlsTail={}",
|
||||||
i,
|
i,
|
||||||
normalize(row.getGroupKey()),
|
normalize(row.getGroupKey()),
|
||||||
normalize(row.getRowToken()),
|
normalize(row.getRowToken()),
|
||||||
@@ -1488,8 +1498,10 @@ public class SimilarAsinTaskService {
|
|||||||
normalize(row.getCountry()),
|
normalize(row.getCountry()),
|
||||||
abbreviateForLog(row.getTitle(), 80),
|
abbreviateForLog(row.getTitle(), 80),
|
||||||
normalize(row.getSku()),
|
normalize(row.getSku()),
|
||||||
|
normalize(row.getPrice()),
|
||||||
abbreviateForLog(url, 200),
|
abbreviateForLog(url, 200),
|
||||||
urls == null ? 0 : urls.size(),
|
urls == null ? 0 : urls.size(),
|
||||||
|
alibaba == null ? 0 : alibaba.size(),
|
||||||
urls == null || urls.isEmpty() ? "" : abbreviateForLog(urls.get(0), 200),
|
urls == null || urls.isEmpty() ? "" : abbreviateForLog(urls.get(0), 200),
|
||||||
urls == null || urls.size() <= 1 ? "" : abbreviateForLog(urls.get(urls.size() - 1), 200));
|
urls == null || urls.size() <= 1 ? "" : abbreviateForLog(urls.get(urls.size() - 1), 200));
|
||||||
}
|
}
|
||||||
@@ -1695,6 +1707,9 @@ public class SimilarAsinTaskService {
|
|||||||
int batchSize = resolveCozeBatchSize(readImgSwitch(task));
|
int batchSize = resolveCozeBatchSize(readImgSwitch(task));
|
||||||
int cozeWorkUnits = countCozeWorkUnits(chunks, batchSize);
|
int cozeWorkUnits = countCozeWorkUnits(chunks, batchSize);
|
||||||
int totalProgressUnits = Math.max(3, cozeWorkUnits + 3);
|
int totalProgressUnits = Math.max(3, cozeWorkUnits + 3);
|
||||||
|
if (countPendingCozeStates(task.getId()) > 0) {
|
||||||
|
finalizeTimedOutCozeStatesForTask(task.getId());
|
||||||
|
}
|
||||||
if (countPendingCozeStates(task.getId()) > 0) {
|
if (countPendingCozeStates(task.getId()) > 0) {
|
||||||
taskFileJobService.touchRunning(job.getId());
|
taskFileJobService.touchRunning(job.getId());
|
||||||
touchJavaSideTaskActivity(task.getId());
|
touchJavaSideTaskActivity(task.getId());
|
||||||
@@ -2394,7 +2409,16 @@ public class SimilarAsinTaskService {
|
|||||||
FileTaskEntity task = taskForPoll(state.getTaskId());
|
FileTaskEntity task = taskForPoll(state.getTaskId());
|
||||||
if (task != null) {
|
if (task != null) {
|
||||||
Map<String, List<SimilarAsinParsedRowVo>> allRowsByBaseId = allRowsByBaseIdForPoll(task);
|
Map<String, List<SimilarAsinParsedRowVo>> allRowsByBaseId = allRowsByBaseIdForPoll(task);
|
||||||
mergeCozeRowsIntoChunk(task, context.chunkScopeHash(), context.chunkIndex(), cozeRows, allRowsByBaseId);
|
try {
|
||||||
|
mergeCozeRowsIntoChunk(task, context.chunkScopeHash(), context.chunkIndex(), cozeRows, allRowsByBaseId);
|
||||||
|
} catch (Exception mergeEx) {
|
||||||
|
if (failureMessage.isBlank()) {
|
||||||
|
throw mergeEx;
|
||||||
|
}
|
||||||
|
log.warn("[similar-asin] coze failed result merge failed, mark state terminal anyway taskId={} stateId={} executeId={} err={}",
|
||||||
|
state.getTaskId(), state.getId(), state.getCozeExecuteId(),
|
||||||
|
firstNonBlank(mergeEx.getMessage(), mergeEx.getClass().getSimpleName()));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
markCozeStateTerminal(state,
|
markCozeStateTerminal(state,
|
||||||
@@ -2408,11 +2432,17 @@ public class SimilarAsinTaskService {
|
|||||||
FileTaskEntity task = taskForPoll(state.getTaskId());
|
FileTaskEntity task = taskForPoll(state.getTaskId());
|
||||||
if (task != null) {
|
if (task != null) {
|
||||||
Map<String, List<SimilarAsinParsedRowVo>> allRowsByBaseId = allRowsByBaseIdForPoll(task);
|
Map<String, List<SimilarAsinParsedRowVo>> allRowsByBaseId = allRowsByBaseIdForPoll(task);
|
||||||
mergeCozeRowsIntoChunk(task,
|
try {
|
||||||
context.chunkScopeHash(),
|
mergeCozeRowsIntoChunk(task,
|
||||||
context.chunkIndex(),
|
context.chunkScopeHash(),
|
||||||
cozeClient.markRowsFailed(batchRows, message),
|
context.chunkIndex(),
|
||||||
allRowsByBaseId);
|
cozeClient.markRowsFailed(batchRows, message),
|
||||||
|
allRowsByBaseId);
|
||||||
|
} catch (Exception mergeEx) {
|
||||||
|
log.warn("[similar-asin] coze timeout fallback merge failed, mark state terminal anyway taskId={} stateId={} executeId={} err={}",
|
||||||
|
state.getTaskId(), state.getId(), state.getCozeExecuteId(),
|
||||||
|
firstNonBlank(mergeEx.getMessage(), mergeEx.getClass().getSimpleName()));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
markCozeStateTerminal(state, COZE_STATUS_FAILED, message);
|
markCozeStateTerminal(state, COZE_STATUS_FAILED, message);
|
||||||
maybeFinalizeCozeJob(state.getTaskId(), context);
|
maybeFinalizeCozeJob(state.getTaskId(), context);
|
||||||
@@ -2424,6 +2454,45 @@ public class SimilarAsinTaskService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void finalizeTimedOutCozeStatesForTask(Long taskId) {
|
||||||
|
if (taskId == null || taskId <= 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||||
|
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
|
||||||
|
.in(TaskScopeStateEntity::getCozeStatus, List.of(COZE_STATUS_SUBMITTED, COZE_STATUS_RUNNING))
|
||||||
|
.orderByAsc(TaskScopeStateEntity::getCozeSubmittedAt)
|
||||||
|
.last("limit 50"));
|
||||||
|
if (states == null || states.isEmpty()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
for (TaskScopeStateEntity state : states) {
|
||||||
|
if (state == null || state.getId() == null || !isCozeStateTimedOut(state)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
CozeBatchContext context = readCozeBatchContext(state);
|
||||||
|
if (context == null || context.resultId() == null) {
|
||||||
|
markCozeStateTerminal(state, COZE_STATUS_FAILED, "Coze 批次上下文缺失");
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
List<SimilarAsinResultRowDto> batchRows = readCozeBatchRows(state);
|
||||||
|
FileTaskEntity task = taskForPoll(taskId);
|
||||||
|
if (task != null) {
|
||||||
|
Map<String, List<SimilarAsinParsedRowVo>> allRowsByBaseId = allRowsByBaseIdForPoll(task);
|
||||||
|
mergeCozeRowsIntoChunk(task,
|
||||||
|
context.chunkScopeHash(),
|
||||||
|
context.chunkIndex(),
|
||||||
|
cozeClient.markRowsFailed(batchRows, "Coze 异步工作流轮询超时"),
|
||||||
|
allRowsByBaseId);
|
||||||
|
}
|
||||||
|
markCozeStateTerminal(state, COZE_STATUS_FAILED, "Coze 异步工作流轮询超时");
|
||||||
|
maybeFinalizeCozeJob(taskId, context);
|
||||||
|
log.warn("[similar-asin] 文件任务超时兜底已将 Coze pending 批次置为失败 taskId={} stateId={} jobId={}",
|
||||||
|
taskId, state.getId(), context.jobId());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private boolean retryFailedCozeBatchState(TaskScopeStateEntity state,
|
private boolean retryFailedCozeBatchState(TaskScopeStateEntity state,
|
||||||
CozeBatchContext context,
|
CozeBatchContext context,
|
||||||
List<SimilarAsinResultRowDto> batchRows,
|
List<SimilarAsinResultRowDto> batchRows,
|
||||||
@@ -2910,7 +2979,7 @@ public class SimilarAsinTaskService {
|
|||||||
if (CozeFailureClassifier.isThrottleLockTimeout(message)) {
|
if (CozeFailureClassifier.isThrottleLockTimeout(message)) {
|
||||||
if (isCozeStateTimedOut(state)) {
|
if (isCozeStateTimedOut(state)) {
|
||||||
Map<String, List<SimilarAsinParsedRowVo>> allRowsByBaseId = allRowsByBaseIdForPoll(task);
|
Map<String, List<SimilarAsinParsedRowVo>> allRowsByBaseId = allRowsByBaseIdForPoll(task);
|
||||||
String finalMessage = "Coze pending submit timeout after waiting credentials: " + message;
|
String finalMessage = "等待 Coze 提交凭证超时:" + message;
|
||||||
mergeCozeRowsIntoChunk(task,
|
mergeCozeRowsIntoChunk(task,
|
||||||
null,
|
null,
|
||||||
null,
|
null,
|
||||||
@@ -2930,7 +2999,7 @@ public class SimilarAsinTaskService {
|
|||||||
Map<String, List<SimilarAsinParsedRowVo>> allRowsByBaseId = allRowsByBaseIdForPoll(task);
|
Map<String, List<SimilarAsinParsedRowVo>> allRowsByBaseId = allRowsByBaseIdForPoll(task);
|
||||||
String finalMessage = isCozeStateTimedOut(state)
|
String finalMessage = isCozeStateTimedOut(state)
|
||||||
? message + " (timeout after " + properties.getCozePollTimeoutMillis() + "ms)"
|
? message + " (timeout after " + properties.getCozePollTimeoutMillis() + "ms)"
|
||||||
: message + " after " + nextAttemptCount + " submit attempts";
|
: message + ",已重试提交 " + nextAttemptCount + " 次";
|
||||||
mergeCozeRowsIntoChunk(task,
|
mergeCozeRowsIntoChunk(task,
|
||||||
null,
|
null,
|
||||||
null,
|
null,
|
||||||
@@ -3731,11 +3800,6 @@ public class SimilarAsinTaskService {
|
|||||||
return hasCompletedScope;
|
return hasCompletedScope;
|
||||||
}
|
}
|
||||||
|
|
||||||
private boolean isJavaSideProcessing(Long taskId) {
|
|
||||||
return countPendingCozeStates(taskId) > 0
|
|
||||||
|| taskFileJobService.countActiveAssembleJobs(taskId, MODULE_TYPE) > 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void touchJavaSideTaskActivity(Long taskId) {
|
private void touchJavaSideTaskActivity(Long taskId) {
|
||||||
if (taskId == null || taskId <= 0) {
|
if (taskId == null || taskId <= 0) {
|
||||||
return;
|
return;
|
||||||
@@ -3792,7 +3856,11 @@ public class SimilarAsinTaskService {
|
|||||||
}
|
}
|
||||||
log.warn("[similar-asin] reject task operation because owner is another instance taskId={} operation={} owner={} current={}",
|
log.warn("[similar-asin] reject task operation because owner is another instance taskId={} operation={} owner={} current={}",
|
||||||
task == null ? null : task.getId(), operation, owner, currentInstanceId());
|
task == null ? null : task.getId(), operation, owner, currentInstanceId());
|
||||||
throw new BusinessException(40903, "This task is bound to another service instance");
|
throw new TaskOwnerMismatchException(
|
||||||
|
task == null ? null : task.getId(),
|
||||||
|
operation,
|
||||||
|
owner,
|
||||||
|
currentInstanceId());
|
||||||
}
|
}
|
||||||
|
|
||||||
private boolean isCozeStateOwnedByCurrentInstance(TaskScopeStateEntity state) {
|
private boolean isCozeStateOwnedByCurrentInstance(TaskScopeStateEntity state) {
|
||||||
@@ -4458,6 +4526,8 @@ public class SimilarAsinTaskService {
|
|||||||
row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getDisplayId(), parsedRow.getSourceId()));
|
row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getDisplayId(), parsedRow.getSourceId()));
|
||||||
row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getAsin(), ""));
|
row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getAsin(), ""));
|
||||||
row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getCountry(), ""));
|
row.createCell(col++).setCellValue(firstNonBlank(parsedRow.getCountry(), ""));
|
||||||
|
row.createCell(col++).setCellValue(readValueByHeader(parsedRow, "卖家名称", "卖家名", "卖家", "店铺名称", "店铺名", "seller name", "seller_name", "seller-name", "sellername", "store name", "shop name"));
|
||||||
|
row.createCell(col++).setCellValue(readValueByHeader(parsedRow, "品牌", "brand"));
|
||||||
row.createCell(col++).setCellValue(resultRow == null ? "" : userFacingCozeCellValue(resultRow, resultRow.getIsStock()));
|
row.createCell(col++).setCellValue(resultRow == null ? "" : userFacingCozeCellValue(resultRow, resultRow.getIsStock()));
|
||||||
row.createCell(col++).setCellValue(resultRow == null ? "" : userFacingCozeCellValue(resultRow, resultRow.getSimilarity()));
|
row.createCell(col++).setCellValue(resultRow == null ? "" : userFacingCozeCellValue(resultRow, resultRow.getSimilarity()));
|
||||||
row.createCell(col++).setCellValue(resultRow == null ? "" : userFacingCozeCellValue(resultRow, resultRow.getIsConform()));
|
row.createCell(col++).setCellValue(resultRow == null ? "" : userFacingCozeCellValue(resultRow, resultRow.getIsConform()));
|
||||||
@@ -4819,6 +4889,22 @@ public class SimilarAsinTaskService {
|
|||||||
return count == null ? 0L : count;
|
return count == null ? 0L : count;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private long countCompletedTasksWithResultFile(Long userId) {
|
||||||
|
if (userId == null || userId <= 0) {
|
||||||
|
return 0L;
|
||||||
|
}
|
||||||
|
Long count = fileTaskMapper.selectCount(new LambdaQueryWrapper<FileTaskEntity>()
|
||||||
|
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
|
||||||
|
.eq(FileTaskEntity::getUserId, userId)
|
||||||
|
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
|
||||||
|
.inSql(FileTaskEntity::getId,
|
||||||
|
"select distinct task_id from file_result " +
|
||||||
|
"where module_type = '" + MODULE_TYPE + "' " +
|
||||||
|
"and user_id = " + userId + " " +
|
||||||
|
"and result_file_url is not null and result_file_url <> ''"));
|
||||||
|
return count == null ? 0L : count;
|
||||||
|
}
|
||||||
|
|
||||||
private int countChunks(Long taskId, String scopeHash) {
|
private int countChunks(Long taskId, String scopeHash) {
|
||||||
Long count = taskChunkMapper.selectCount(new LambdaQueryWrapper<TaskChunkEntity>()
|
Long count = taskChunkMapper.selectCount(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||||
.eq(TaskChunkEntity::getTaskId, taskId)
|
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||||
@@ -4861,7 +4947,7 @@ public class SimilarAsinTaskService {
|
|||||||
SimilarAsinTaskItemVo vo = new SimilarAsinTaskItemVo();
|
SimilarAsinTaskItemVo vo = new SimilarAsinTaskItemVo();
|
||||||
vo.setId(task.getId());
|
vo.setId(task.getId());
|
||||||
vo.setTaskNo(task.getTaskNo());
|
vo.setTaskNo(task.getTaskNo());
|
||||||
vo.setStatus(task.getStatus());
|
vo.setStatus(resolveDisplayTaskStatus(task));
|
||||||
vo.setErrorMessage(task.getErrorMessage());
|
vo.setErrorMessage(task.getErrorMessage());
|
||||||
vo.setCreatedAt(fmt(task.getCreatedAt()));
|
vo.setCreatedAt(fmt(task.getCreatedAt()));
|
||||||
vo.setUpdatedAt(fmt(task.getUpdatedAt()));
|
vo.setUpdatedAt(fmt(task.getUpdatedAt()));
|
||||||
@@ -4876,7 +4962,7 @@ public class SimilarAsinTaskService {
|
|||||||
vo.setSourceFilename(row.getSourceFilename());
|
vo.setSourceFilename(row.getSourceFilename());
|
||||||
vo.setResultFilename(row.getResultFilename());
|
vo.setResultFilename(row.getResultFilename());
|
||||||
vo.setDownloadUrl(buildFreshDownloadUrl(row));
|
vo.setDownloadUrl(buildFreshDownloadUrl(row));
|
||||||
vo.setTaskStatus(task == null ? null : task.getStatus());
|
vo.setTaskStatus(resolveDisplayTaskStatus(task, row));
|
||||||
attachFileJobState(vo, row, job, task);
|
attachFileJobState(vo, row, job, task);
|
||||||
vo.setSuccess(row.getSuccess() != null && row.getSuccess() == 1);
|
vo.setSuccess(row.getSuccess() != null && row.getSuccess() == 1);
|
||||||
vo.setError(row.getErrorMessage());
|
vo.setError(row.getErrorMessage());
|
||||||
@@ -4931,6 +5017,13 @@ public class SimilarAsinTaskService {
|
|||||||
int cozePending = countPendingCozeStates(taskId);
|
int cozePending = countPendingCozeStates(taskId);
|
||||||
boolean uploadComplete = isResultSubmissionComplete(taskId);
|
boolean uploadComplete = isResultSubmissionComplete(taskId);
|
||||||
if (!uploadComplete) {
|
if (!uploadComplete) {
|
||||||
|
if (canTreatTaskAsCompleted(task, row, job)) {
|
||||||
|
vo.setFileProgressCurrent(1);
|
||||||
|
vo.setFileProgressTotal(1);
|
||||||
|
vo.setFileProgressPercent(100);
|
||||||
|
vo.setFileProgressMessage("结果文件已生成");
|
||||||
|
return;
|
||||||
|
}
|
||||||
if (task != null && !STATUS_RUNNING.equals(task.getStatus())) {
|
if (task != null && !STATUS_RUNNING.equals(task.getStatus())) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -5088,6 +5181,42 @@ public class SimilarAsinTaskService {
|
|||||||
return "等待 Python 回传,已回传 " + Math.max(0, current) + "/" + Math.max(1, total) + " " + safeUnit;
|
return "等待 Python 回传,已回传 " + Math.max(0, current) + "/" + Math.max(1, total) + " " + safeUnit;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private String resolveDisplayTaskStatus(FileTaskEntity task) {
|
||||||
|
return resolveDisplayTaskStatus(task, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String resolveDisplayTaskStatus(FileTaskEntity task, FileResultEntity row) {
|
||||||
|
if (task == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (canTreatTaskAsCompleted(task, row, null)) {
|
||||||
|
return STATUS_SUCCESS;
|
||||||
|
}
|
||||||
|
return task.getStatus();
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean canTreatTaskAsCompleted(FileTaskEntity task, FileResultEntity row, TaskFileJobEntity job) {
|
||||||
|
if (task == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (STATUS_SUCCESS.equals(task.getStatus())) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (!STATUS_RUNNING.equals(task.getStatus())) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (row == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (row.getResultFileUrl() == null || row.getResultFileUrl().isBlank()) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (job != null && "FAILED".equals(job.getStatus())) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
private int calculateCozeDisplayTotal(TaskProgressSnapshotEntity snapshot, int observedCozeStates) {
|
private int calculateCozeDisplayTotal(TaskProgressSnapshotEntity snapshot, int observedCozeStates) {
|
||||||
int observed = Math.max(0, observedCozeStates);
|
int observed = Math.max(0, observedCozeStates);
|
||||||
// 小任务 totalCount <= 3 时直接使用 observed,避免 totalCount - 3 → 0 引起进度条 0/0。
|
// 小任务 totalCount <= 3 时直接使用 observed,避免 totalCount - 3 → 0 引起进度条 0/0。
|
||||||
@@ -5135,6 +5264,7 @@ public class SimilarAsinTaskService {
|
|||||||
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || userId != null && !Objects.equals(userId, task.getUserId())) {
|
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || userId != null && !Objects.equals(userId, task.getUserId())) {
|
||||||
throw new BusinessException("任务不存在");
|
throw new BusinessException("任务不存在");
|
||||||
}
|
}
|
||||||
|
ensureTaskOwnedByCurrentInstance(task, "parsed payload");
|
||||||
return readParsedPayload(task);
|
return readParsedPayload(task);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -5243,6 +5373,8 @@ public class SimilarAsinTaskService {
|
|||||||
log.warn("[similar-asin] read chunk payload failed taskId={} chunk={} crossInstance={} err={}",
|
log.warn("[similar-asin] read chunk payload failed taskId={} chunk={} crossInstance={} err={}",
|
||||||
chunk.getTaskId(), chunk.getChunkIndex(), crossInstance, msg);
|
chunk.getTaskId(), chunk.getChunkIndex(), crossInstance, msg);
|
||||||
recordChunkReadFailure(chunk, crossInstance, msg);
|
recordChunkReadFailure(chunk, crossInstance, msg);
|
||||||
|
throw new BusinessException("similar ASIN chunk payload read failed chunk="
|
||||||
|
+ chunk.getChunkIndex() + ": " + msg, ex);
|
||||||
}
|
}
|
||||||
return rows;
|
return rows;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -92,6 +92,10 @@ public class SimilarAsinImageEmbedder {
|
|||||||
|
|
||||||
private static final String UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
private static final String UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||||
+ "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36";
|
+ "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36";
|
||||||
|
private static final String IMAGE_ACCEPT = "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8";
|
||||||
|
private static final String DOWNLOAD_ACCEPT = "*/*";
|
||||||
|
private static final String ACCEPT_LANGUAGE = "en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7";
|
||||||
|
private static final String AMAZON_REFERER = "https://www.amazon.com/";
|
||||||
|
|
||||||
private final int downloadTimeoutSeconds;
|
private final int downloadTimeoutSeconds;
|
||||||
private final int downloadPoolSize;
|
private final int downloadPoolSize;
|
||||||
@@ -108,9 +112,9 @@ public class SimilarAsinImageEmbedder {
|
|||||||
.readTimeout(Duration.ofSeconds(downloadTimeoutSeconds))
|
.readTimeout(Duration.ofSeconds(downloadTimeoutSeconds))
|
||||||
.writeTimeout(Duration.ofSeconds(downloadTimeoutSeconds))
|
.writeTimeout(Duration.ofSeconds(downloadTimeoutSeconds))
|
||||||
.callTimeout(Duration.ofSeconds(downloadTimeoutSeconds * 2L))
|
.callTimeout(Duration.ofSeconds(downloadTimeoutSeconds * 2L))
|
||||||
.retryOnConnectionFailure(false)
|
.retryOnConnectionFailure(true)
|
||||||
.followRedirects(false)
|
.followRedirects(true)
|
||||||
.followSslRedirects(false)
|
.followSslRedirects(true)
|
||||||
.dns(new SafeDns(Dns.SYSTEM))
|
.dns(new SafeDns(Dns.SYSTEM))
|
||||||
.build();
|
.build();
|
||||||
this.downloadPool = Executors.newFixedThreadPool(downloadPoolSize, namedFactory("similar-asin-image-dl"));
|
this.downloadPool = Executors.newFixedThreadPool(downloadPoolSize, namedFactory("similar-asin-image-dl"));
|
||||||
@@ -129,6 +133,10 @@ public class SimilarAsinImageEmbedder {
|
|||||||
return downloadPoolSize;
|
return downloadPoolSize;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
OkHttpClient httpClient() {
|
||||||
|
return httpClient;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* P2-8 prefetch:assemble 阶段第一遍扫所有 url,并行预下载到 taskImageCache。
|
* P2-8 prefetch:assemble 阶段第一遍扫所有 url,并行预下载到 taskImageCache。
|
||||||
* embed() 第二遍写入时若 cache 命中直接复用,否则回退到现在的串行下载逻辑(保持兜底)。
|
* embed() 第二遍写入时若 cache 命中直接复用,否则回退到现在的串行下载逻辑(保持兜底)。
|
||||||
@@ -174,7 +182,7 @@ public class SimilarAsinImageEmbedder {
|
|||||||
taskImageCache.putIfAbsent(url, thumb);
|
taskImageCache.putIfAbsent(url, thumb);
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
// 预下载失败不抛出:embed() 时同 url 会再次尝试并走原有兜底链路。
|
// 预下载失败不抛出:embed() 时同 url 会再次尝试并走原有兜底链路。
|
||||||
log.debug("[similar-asin][image] prefetch-fail url={} err={}", url, ex.getMessage());
|
log.debug("[similar-asin][image] prefetch-fail url={} err={}", url, errorSummary(ex));
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
futures.add(completion.submit(task, null));
|
futures.add(completion.submit(task, null));
|
||||||
@@ -229,7 +237,7 @@ public class SimilarAsinImageEmbedder {
|
|||||||
byte[] raw = downloadWithRetry(trimmed);
|
byte[] raw = downloadWithRetry(trimmed);
|
||||||
return resizeImage(trimmed, raw);
|
return resizeImage(trimmed, raw);
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
log.debug("[similar-asin][image] prefetch-cache-fail url={} err={}", trimmed, ex.getMessage());
|
log.debug("[similar-asin][image] prefetch-cache-fail url={} err={}", trimmed, errorSummary(ex));
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -273,6 +281,7 @@ public class SimilarAsinImageEmbedder {
|
|||||||
int colIdx,
|
int colIdx,
|
||||||
Row row,
|
Row row,
|
||||||
Map<String, ResizedImage> taskImageCache) {
|
Map<String, ResizedImage> taskImageCache) {
|
||||||
|
long startedNanos = System.nanoTime();
|
||||||
try {
|
try {
|
||||||
ResizedImage thumb = taskImageCache.get(trimmedUrl);
|
ResizedImage thumb = taskImageCache.get(trimmedUrl);
|
||||||
boolean cacheHit = thumb != null;
|
boolean cacheHit = thumb != null;
|
||||||
@@ -294,11 +303,14 @@ public class SimilarAsinImageEmbedder {
|
|||||||
} catch (ResizeOversizeException ex) {
|
} catch (ResizeOversizeException ex) {
|
||||||
log.warn("[similar-asin][image] resize-oversize url={} bytes={}", ex.url(), ex.size());
|
log.warn("[similar-asin][image] resize-oversize url={} bytes={}", ex.url(), ex.size());
|
||||||
} catch (ResizeException ex) {
|
} catch (ResizeException ex) {
|
||||||
log.warn("[similar-asin][image] resize-fail url={} err={}", trimmedUrl, ex.getMessage());
|
log.warn("[similar-asin][image] resize-fail url={} elapsedMs={} err={}",
|
||||||
|
trimmedUrl, elapsedMs(startedNanos), errorSummary(ex));
|
||||||
} catch (IOException | TimeoutException ex) {
|
} catch (IOException | TimeoutException ex) {
|
||||||
log.warn("[similar-asin][image] download-fail url={} err={}", trimmedUrl, ex.getMessage());
|
log.warn("[similar-asin][image] download-fail url={} elapsedMs={} err={}",
|
||||||
|
trimmedUrl, elapsedMs(startedNanos), errorSummary(ex));
|
||||||
} catch (RuntimeException ex) {
|
} catch (RuntimeException ex) {
|
||||||
log.warn("[similar-asin][image] embed-fail url={} err={}", trimmedUrl, ex.getMessage());
|
log.warn("[similar-asin][image] embed-fail url={} elapsedMs={} err={}",
|
||||||
|
trimmedUrl, elapsedMs(startedNanos), errorSummary(ex));
|
||||||
}
|
}
|
||||||
row.createCell(colIdx).setCellValue(trimmedUrl);
|
row.createCell(colIdx).setCellValue(trimmedUrl);
|
||||||
return null;
|
return null;
|
||||||
@@ -334,10 +346,12 @@ public class SimilarAsinImageEmbedder {
|
|||||||
private byte[] downloadWithRetry(String url) throws IOException, TimeoutException {
|
private byte[] downloadWithRetry(String url) throws IOException, TimeoutException {
|
||||||
validateHttpsUrl(url);
|
validateHttpsUrl(url);
|
||||||
IOException last = null;
|
IOException last = null;
|
||||||
|
TimeoutException lastTimeout = null;
|
||||||
|
long waitSeconds = downloadTimeoutSeconds * 2L;
|
||||||
for (int attempt = 0; attempt <= DOWNLOAD_MAX_RETRY; attempt++) {
|
for (int attempt = 0; attempt <= DOWNLOAD_MAX_RETRY; attempt++) {
|
||||||
Future<byte[]> future = downloadPool.submit(() -> doFetch(url));
|
Future<byte[]> future = downloadPool.submit(() -> doFetch(url));
|
||||||
try {
|
try {
|
||||||
return future.get(downloadTimeoutSeconds * 2L, TimeUnit.SECONDS);
|
return future.get(waitSeconds, TimeUnit.SECONDS);
|
||||||
} catch (java.util.concurrent.ExecutionException ee) {
|
} catch (java.util.concurrent.ExecutionException ee) {
|
||||||
Throwable cause = ee.getCause();
|
Throwable cause = ee.getCause();
|
||||||
if (cause instanceof DownloadOversizeException doe) {
|
if (cause instanceof DownloadOversizeException doe) {
|
||||||
@@ -345,6 +359,8 @@ public class SimilarAsinImageEmbedder {
|
|||||||
}
|
}
|
||||||
if (cause instanceof IOException io) {
|
if (cause instanceof IOException io) {
|
||||||
last = io;
|
last = io;
|
||||||
|
lastTimeout = null;
|
||||||
|
logRetry(url, attempt, io);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if (cause instanceof RuntimeException re) {
|
if (cause instanceof RuntimeException re) {
|
||||||
@@ -353,15 +369,31 @@ public class SimilarAsinImageEmbedder {
|
|||||||
throw new IOException("image download failed: " + cause.getMessage(), cause);
|
throw new IOException("image download failed: " + cause.getMessage(), cause);
|
||||||
} catch (java.util.concurrent.TimeoutException te) {
|
} catch (java.util.concurrent.TimeoutException te) {
|
||||||
future.cancel(true);
|
future.cancel(true);
|
||||||
throw te;
|
TimeoutException wrapped = new TimeoutException("image download timeout after attempt "
|
||||||
|
+ (attempt + 1) + "/" + (DOWNLOAD_MAX_RETRY + 1)
|
||||||
|
+ ", waitSeconds=" + waitSeconds);
|
||||||
|
wrapped.initCause(te);
|
||||||
|
lastTimeout = wrapped;
|
||||||
|
last = null;
|
||||||
|
logRetry(url, attempt, wrapped);
|
||||||
} catch (InterruptedException ie) {
|
} catch (InterruptedException ie) {
|
||||||
Thread.currentThread().interrupt();
|
Thread.currentThread().interrupt();
|
||||||
throw new IOException("image download interrupted", ie);
|
throw new IOException("image download interrupted", ie);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if (lastTimeout != null) {
|
||||||
|
throw lastTimeout;
|
||||||
|
}
|
||||||
throw last != null ? last : new IOException("image download failed without cause");
|
throw last != null ? last : new IOException("image download failed without cause");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void logRetry(String url, int attempt, Exception ex) {
|
||||||
|
if (attempt < DOWNLOAD_MAX_RETRY) {
|
||||||
|
log.debug("[similar-asin][image] download-retry url={} nextAttempt={}/{} err={}",
|
||||||
|
url, attempt + 2, DOWNLOAD_MAX_RETRY + 1, errorSummary(ex));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
static void validateHttpsUrl(String url) {
|
static void validateHttpsUrl(String url) {
|
||||||
URI uri;
|
URI uri;
|
||||||
try {
|
try {
|
||||||
@@ -407,11 +439,7 @@ public class SimilarAsinImageEmbedder {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private byte[] doFetch(String url) throws IOException {
|
private byte[] doFetch(String url) throws IOException {
|
||||||
Request req = new Request.Builder()
|
Request req = buildImageRequest(url);
|
||||||
.url(url)
|
|
||||||
.header("User-Agent", UA)
|
|
||||||
.get()
|
|
||||||
.build();
|
|
||||||
try (Response resp = httpClient.newCall(req).execute()) {
|
try (Response resp = httpClient.newCall(req).execute()) {
|
||||||
if (!resp.isSuccessful()) {
|
if (!resp.isSuccessful()) {
|
||||||
throw new IOException("http " + resp.code() + " for " + url);
|
throw new IOException("http " + resp.code() + " for " + url);
|
||||||
@@ -441,6 +469,57 @@ public class SimilarAsinImageEmbedder {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static Request buildImageRequest(String url) {
|
||||||
|
Request.Builder builder = new Request.Builder()
|
||||||
|
.url(url)
|
||||||
|
.header("User-Agent", UA)
|
||||||
|
.header("Accept-Language", ACCEPT_LANGUAGE)
|
||||||
|
.header("Cache-Control", "no-cache")
|
||||||
|
.get();
|
||||||
|
if (isCozeSignedImageUrl(url)) {
|
||||||
|
// Coze/TOS signed image links behave like direct file downloads.
|
||||||
|
// A foreign Referer can be rejected, so keep this close to a browser address-bar download.
|
||||||
|
builder.header("Accept", DOWNLOAD_ACCEPT);
|
||||||
|
} else {
|
||||||
|
builder.header("Accept", IMAGE_ACCEPT)
|
||||||
|
.header("Referer", AMAZON_REFERER);
|
||||||
|
}
|
||||||
|
return builder.build();
|
||||||
|
}
|
||||||
|
|
||||||
|
static boolean isCozeSignedImageUrl(String url) {
|
||||||
|
try {
|
||||||
|
URI uri = new URI(url);
|
||||||
|
String host = uri.getHost();
|
||||||
|
return host != null && host.toLowerCase(Locale.ROOT).endsWith("-bot-platform-tos-sign.coze.cn");
|
||||||
|
} catch (URISyntaxException ex) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static long elapsedMs(long startedNanos) {
|
||||||
|
return TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startedNanos);
|
||||||
|
}
|
||||||
|
|
||||||
|
static String errorSummary(Throwable ex) {
|
||||||
|
if (ex == null) {
|
||||||
|
return "unknown";
|
||||||
|
}
|
||||||
|
StringBuilder sb = new StringBuilder(ex.getClass().getSimpleName());
|
||||||
|
String message = ex.getMessage();
|
||||||
|
sb.append(": ");
|
||||||
|
sb.append(message == null || message.isBlank() ? "<empty>" : message);
|
||||||
|
Throwable cause = ex.getCause();
|
||||||
|
if (cause != null) {
|
||||||
|
sb.append("; cause=").append(cause.getClass().getSimpleName());
|
||||||
|
String causeMessage = cause.getMessage();
|
||||||
|
if (causeMessage != null && !causeMessage.isBlank()) {
|
||||||
|
sb.append(": ").append(causeMessage);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return sb.toString();
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 等比缩放到长边 TARGET_LONG_EDGE_PX,JPEG q=0.75 输出;返回字节 + 实际像素,供调用方按图自适应单元格尺寸。
|
* 等比缩放到长边 TARGET_LONG_EDGE_PX,JPEG q=0.75 输出;返回字节 + 实际像素,供调用方按图自适应单元格尺寸。
|
||||||
* 出口校验字节数 ≤ MAX_THUMB_SIZE_BYTES,超限按 MAX_THUMB_SIZE → 长边 → 质量的顺序迭代降级,
|
* 出口校验字节数 ≤ MAX_THUMB_SIZE_BYTES,超限按 MAX_THUMB_SIZE → 长边 → 质量的顺序迭代降级,
|
||||||
|
|||||||
@@ -0,0 +1,36 @@
|
|||||||
|
package com.nanri.aiimage.modules.task.controller;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.common.api.ApiResponse;
|
||||||
|
import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest;
|
||||||
|
import com.nanri.aiimage.modules.task.model.vo.TaskHeartbeatVo;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskHeartbeatService;
|
||||||
|
import io.swagger.v3.oas.annotations.Operation;
|
||||||
|
import io.swagger.v3.oas.annotations.Parameter;
|
||||||
|
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||||
|
import jakarta.validation.Valid;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import org.springframework.web.bind.annotation.PathVariable;
|
||||||
|
import org.springframework.web.bind.annotation.PostMapping;
|
||||||
|
import org.springframework.web.bind.annotation.RequestBody;
|
||||||
|
import org.springframework.web.bind.annotation.RequestMapping;
|
||||||
|
import org.springframework.web.bind.annotation.RestController;
|
||||||
|
|
||||||
|
@RestController
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
@RequestMapping("/api/tasks")
|
||||||
|
@Tag(name = "Task heartbeat", description = "Unified heartbeat API for Python task workers")
|
||||||
|
public class TaskHeartbeatController {
|
||||||
|
|
||||||
|
private final TaskHeartbeatService taskHeartbeatService;
|
||||||
|
|
||||||
|
@PostMapping("/{taskId}/heartbeat")
|
||||||
|
@Operation(
|
||||||
|
summary = "Task heartbeat",
|
||||||
|
description = "Python calls this during execution. The backend resolves biz_file_task or brand task by taskId. user_id is optional.")
|
||||||
|
public ApiResponse<TaskHeartbeatVo> heartbeat(
|
||||||
|
@Parameter(description = "Task ID", required = true, example = "200")
|
||||||
|
@PathVariable Long taskId,
|
||||||
|
@Valid @RequestBody(required = false) TaskHeartbeatRequest request) {
|
||||||
|
return ApiResponse.success(taskHeartbeatService.heartbeat(taskId, request));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -2,11 +2,14 @@ package com.nanri.aiimage.modules.task.mapper;
|
|||||||
|
|
||||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||||
import com.nanri.aiimage.modules.task.model.entity.TaskImageCacheEntity;
|
import com.nanri.aiimage.modules.task.model.entity.TaskImageCacheEntity;
|
||||||
|
import org.apache.ibatis.annotations.Delete;
|
||||||
import org.apache.ibatis.annotations.Mapper;
|
import org.apache.ibatis.annotations.Mapper;
|
||||||
import org.apache.ibatis.annotations.Param;
|
import org.apache.ibatis.annotations.Param;
|
||||||
import org.apache.ibatis.annotations.Select;
|
import org.apache.ibatis.annotations.Select;
|
||||||
import org.apache.ibatis.annotations.Update;
|
import org.apache.ibatis.annotations.Update;
|
||||||
|
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* P2-12:图片缩略图缓存 mapper。提供 LRU 命中刷新、按 url_hash 直读字节两个轻量入口,
|
* P2-12:图片缩略图缓存 mapper。提供 LRU 命中刷新、按 url_hash 直读字节两个轻量入口,
|
||||||
* 其余 CRUD 走 {@link BaseMapper} 默认实现。
|
* 其余 CRUD 走 {@link BaseMapper} 默认实现。
|
||||||
@@ -15,7 +18,7 @@ import org.apache.ibatis.annotations.Update;
|
|||||||
public interface TaskImageCacheMapper extends BaseMapper<TaskImageCacheEntity> {
|
public interface TaskImageCacheMapper extends BaseMapper<TaskImageCacheEntity> {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 命中时同步 bumping last_used_at,便于后续按 LRU 清理 7 天前未用记录。
|
* 命中时同步 bumping last_used_at,便于后续按 LRU 清理过期未用记录。
|
||||||
*/
|
*/
|
||||||
@Update("UPDATE biz_task_image_cache SET last_used_at = NOW(3) WHERE url_hash = #{urlHash}")
|
@Update("UPDATE biz_task_image_cache SET last_used_at = NOW(3) WHERE url_hash = #{urlHash}")
|
||||||
int touchLastUsed(@Param("urlHash") String urlHash);
|
int touchLastUsed(@Param("urlHash") String urlHash);
|
||||||
@@ -26,4 +29,7 @@ public interface TaskImageCacheMapper extends BaseMapper<TaskImageCacheEntity> {
|
|||||||
*/
|
*/
|
||||||
@Select("SELECT image_bytes FROM biz_task_image_cache WHERE url_hash = #{urlHash} LIMIT 1")
|
@Select("SELECT image_bytes FROM biz_task_image_cache WHERE url_hash = #{urlHash} LIMIT 1")
|
||||||
byte[] selectBytesByUrlHash(@Param("urlHash") String urlHash);
|
byte[] selectBytesByUrlHash(@Param("urlHash") String urlHash);
|
||||||
|
|
||||||
|
@Delete("DELETE FROM biz_task_image_cache WHERE last_used_at < #{cutoff} ORDER BY last_used_at LIMIT #{limit}")
|
||||||
|
int deleteExpiredBatch(@Param("cutoff") LocalDateTime cutoff, @Param("limit") int limit);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,23 @@
|
|||||||
|
package com.nanri.aiimage.modules.task.model.dto;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||||
|
import io.swagger.v3.oas.annotations.media.Schema;
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@Schema(description = "Task heartbeat request")
|
||||||
|
public class TaskHeartbeatRequest {
|
||||||
|
|
||||||
|
@JsonProperty("user_id")
|
||||||
|
@Schema(description = "Optional task owner user ID. Python workers can omit it.", example = "453")
|
||||||
|
private Long userId;
|
||||||
|
|
||||||
|
@Schema(description = "Optional phase label", example = "crawling")
|
||||||
|
private String phase;
|
||||||
|
|
||||||
|
@Schema(description = "Optional current progress", example = "42")
|
||||||
|
private Integer current;
|
||||||
|
|
||||||
|
@Schema(description = "Optional total progress", example = "1100")
|
||||||
|
private Integer total;
|
||||||
|
}
|
||||||
@@ -16,7 +16,7 @@ import java.time.LocalDateTime;
|
|||||||
* <li>{@link #urlHash}:sha256 lowercase hex,作为唯一键避免 1024 长 URL 命中索引长度限制;</li>
|
* <li>{@link #urlHash}:sha256 lowercase hex,作为唯一键避免 1024 长 URL 命中索引长度限制;</li>
|
||||||
* <li>{@link #imageBytes}:已 resize 后的 JPEG 缩略图字节,最大约 300KB(保持与
|
* <li>{@link #imageBytes}:已 resize 后的 JPEG 缩略图字节,最大约 300KB(保持与
|
||||||
* {@code SimilarAsinImageEmbedder.MAX_THUMB_SIZE_BYTES} 对齐);</li>
|
* {@code SimilarAsinImageEmbedder.MAX_THUMB_SIZE_BYTES} 对齐);</li>
|
||||||
* <li>{@link #lastUsedAt}:用于后续 LRU 清理(>7 天未用)。</li>
|
* <li>{@link #lastUsedAt}:用于后续 LRU 清理(默认 >3 天未用)。</li>
|
||||||
* </ul>
|
* </ul>
|
||||||
*/
|
*/
|
||||||
@Data
|
@Data
|
||||||
|
|||||||
@@ -0,0 +1,39 @@
|
|||||||
|
package com.nanri.aiimage.modules.task.model.vo;
|
||||||
|
|
||||||
|
import io.swagger.v3.oas.annotations.media.Schema;
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@Schema(description = "Task heartbeat response")
|
||||||
|
public class TaskHeartbeatVo {
|
||||||
|
|
||||||
|
@Schema(description = "Whether the task can continue", example = "true")
|
||||||
|
private boolean alive;
|
||||||
|
|
||||||
|
@Schema(description = "Task status", example = "RUNNING")
|
||||||
|
private String status;
|
||||||
|
|
||||||
|
@Schema(description = "Resolved task module", example = "PRODUCT_RISK_RESOLVE")
|
||||||
|
private String moduleType;
|
||||||
|
|
||||||
|
@Schema(description = "Message", example = "ok")
|
||||||
|
private String message;
|
||||||
|
|
||||||
|
public static TaskHeartbeatVo alive(String moduleType, String status) {
|
||||||
|
TaskHeartbeatVo vo = new TaskHeartbeatVo();
|
||||||
|
vo.setAlive(true);
|
||||||
|
vo.setModuleType(moduleType);
|
||||||
|
vo.setStatus(status);
|
||||||
|
vo.setMessage("ok");
|
||||||
|
return vo;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static TaskHeartbeatVo notAlive(String moduleType, String status, String message) {
|
||||||
|
TaskHeartbeatVo vo = new TaskHeartbeatVo();
|
||||||
|
vo.setAlive(false);
|
||||||
|
vo.setModuleType(moduleType);
|
||||||
|
vo.setStatus(status);
|
||||||
|
vo.setMessage(message);
|
||||||
|
return vo;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,211 @@
|
|||||||
|
package com.nanri.aiimage.modules.task.service;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||||
|
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||||
|
import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskCacheService;
|
||||||
|
import com.nanri.aiimage.modules.brand.mapper.BrandCrawlTaskMapper;
|
||||||
|
import com.nanri.aiimage.modules.brand.model.entity.BrandCrawlTaskEntity;
|
||||||
|
import com.nanri.aiimage.modules.brand.service.BrandTaskProgressCacheService;
|
||||||
|
import com.nanri.aiimage.modules.deletebrand.service.DeleteBrandTaskCacheService;
|
||||||
|
import com.nanri.aiimage.modules.patroldelete.service.PatrolDeleteTaskCacheService;
|
||||||
|
import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskCacheService;
|
||||||
|
import com.nanri.aiimage.modules.productrisk.service.ProductRiskTaskCacheService;
|
||||||
|
import com.nanri.aiimage.modules.queryasin.service.QueryAsinTaskCacheService;
|
||||||
|
import com.nanri.aiimage.modules.shopmatch.service.ShopMatchTaskCacheService;
|
||||||
|
import com.nanri.aiimage.modules.similarasin.service.SimilarAsinTaskCacheService;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.model.dto.TaskHeartbeatRequest;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.vo.TaskHeartbeatVo;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
@Service
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
public class TaskHeartbeatService {
|
||||||
|
|
||||||
|
private static final String STATUS_RUNNING = "RUNNING";
|
||||||
|
private static final String BRAND_STATUS_RUNNING = "running";
|
||||||
|
|
||||||
|
private static final String MODULE_DELETE_BRAND = "DELETE_BRAND";
|
||||||
|
private static final String MODULE_PRODUCT_RISK = "PRODUCT_RISK_RESOLVE";
|
||||||
|
private static final String MODULE_PRICE_TRACK = "PRICE_TRACK";
|
||||||
|
private static final String MODULE_SHOP_MATCH = "SHOP_MATCH";
|
||||||
|
private static final String MODULE_PATROL_DELETE = "PATROL_DELETE";
|
||||||
|
private static final String MODULE_QUERY_ASIN = "QUERY_ASIN";
|
||||||
|
private static final String MODULE_APPEARANCE_PATENT = "APPEARANCE_PATENT";
|
||||||
|
private static final String MODULE_SIMILAR_ASIN = "SIMILAR_ASIN";
|
||||||
|
private static final String MODULE_BRAND = "BRAND";
|
||||||
|
|
||||||
|
private final FileTaskMapper fileTaskMapper;
|
||||||
|
private final BrandCrawlTaskMapper brandCrawlTaskMapper;
|
||||||
|
private final ProductRiskTaskCacheService productRiskTaskCacheService;
|
||||||
|
private final PriceTrackTaskCacheService priceTrackTaskCacheService;
|
||||||
|
private final ShopMatchTaskCacheService shopMatchTaskCacheService;
|
||||||
|
private final PatrolDeleteTaskCacheService patrolDeleteTaskCacheService;
|
||||||
|
private final QueryAsinTaskCacheService queryAsinTaskCacheService;
|
||||||
|
private final AppearancePatentTaskCacheService appearancePatentTaskCacheService;
|
||||||
|
private final SimilarAsinTaskCacheService similarAsinTaskCacheService;
|
||||||
|
private final DeleteBrandTaskCacheService deleteBrandTaskCacheService;
|
||||||
|
private final BrandTaskProgressCacheService brandTaskProgressCacheService;
|
||||||
|
|
||||||
|
public TaskHeartbeatVo heartbeat(Long taskId, TaskHeartbeatRequest request) {
|
||||||
|
if (taskId == null || taskId <= 0) {
|
||||||
|
return TaskHeartbeatVo.notAlive(null, null, "invalid taskId");
|
||||||
|
}
|
||||||
|
Long userId = request == null ? null : request.getUserId();
|
||||||
|
|
||||||
|
LambdaQueryWrapper<FileTaskEntity> fileQuery = new LambdaQueryWrapper<FileTaskEntity>()
|
||||||
|
.eq(FileTaskEntity::getId, taskId)
|
||||||
|
.last("limit 1");
|
||||||
|
if (userId != null && userId > 0) {
|
||||||
|
fileQuery.eq(FileTaskEntity::getUserId, userId);
|
||||||
|
}
|
||||||
|
FileTaskEntity fileTask = fileTaskMapper.selectOne(fileQuery);
|
||||||
|
|
||||||
|
LambdaQueryWrapper<BrandCrawlTaskEntity> brandQuery = new LambdaQueryWrapper<BrandCrawlTaskEntity>()
|
||||||
|
.eq(BrandCrawlTaskEntity::getId, taskId)
|
||||||
|
.last("limit 1");
|
||||||
|
if (userId != null && userId > 0) {
|
||||||
|
brandQuery.eq(BrandCrawlTaskEntity::getUserId, userId);
|
||||||
|
}
|
||||||
|
BrandCrawlTaskEntity brandTask = brandCrawlTaskMapper.selectOne(brandQuery);
|
||||||
|
|
||||||
|
TaskHeartbeatVo fileResult = touchFileTaskIfRunning(fileTask, request);
|
||||||
|
TaskHeartbeatVo brandResult = touchBrandTaskIfRunning(brandTask, request);
|
||||||
|
|
||||||
|
if (fileResult != null && fileResult.isAlive() && brandResult != null && brandResult.isAlive()) {
|
||||||
|
return TaskHeartbeatVo.alive(fileResult.getModuleType() + "," + MODULE_BRAND, STATUS_RUNNING);
|
||||||
|
}
|
||||||
|
if (fileResult != null && fileResult.isAlive()) {
|
||||||
|
return fileResult;
|
||||||
|
}
|
||||||
|
if (brandResult != null && brandResult.isAlive()) {
|
||||||
|
return brandResult;
|
||||||
|
}
|
||||||
|
if (fileResult != null) {
|
||||||
|
return fileResult;
|
||||||
|
}
|
||||||
|
if (brandResult != null) {
|
||||||
|
return brandResult;
|
||||||
|
}
|
||||||
|
return TaskHeartbeatVo.notAlive(null, null, "task not found");
|
||||||
|
}
|
||||||
|
|
||||||
|
private TaskHeartbeatVo touchFileTaskIfRunning(FileTaskEntity task, TaskHeartbeatRequest request) {
|
||||||
|
if (task == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
String moduleType = task.getModuleType() == null ? "" : task.getModuleType();
|
||||||
|
String status = task.getStatus();
|
||||||
|
if (!STATUS_RUNNING.equals(status)) {
|
||||||
|
return TaskHeartbeatVo.notAlive(moduleType, status, "task is not running");
|
||||||
|
}
|
||||||
|
LocalDateTime now = LocalDateTime.now();
|
||||||
|
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||||
|
.eq(FileTaskEntity::getId, task.getId())
|
||||||
|
.eq(FileTaskEntity::getUserId, task.getUserId())
|
||||||
|
.eq(FileTaskEntity::getStatus, STATUS_RUNNING)
|
||||||
|
.set(FileTaskEntity::getUpdatedAt, now));
|
||||||
|
if (updated <= 0) {
|
||||||
|
FileTaskEntity latest = fileTaskMapper.selectById(task.getId());
|
||||||
|
return TaskHeartbeatVo.notAlive(moduleType, latest == null ? status : latest.getStatus(), "task is not running");
|
||||||
|
}
|
||||||
|
touchModuleHeartbeat(moduleType, task.getId(), request);
|
||||||
|
task.setUpdatedAt(now);
|
||||||
|
saveFileTaskCache(moduleType, task);
|
||||||
|
return TaskHeartbeatVo.alive(moduleType, STATUS_RUNNING);
|
||||||
|
}
|
||||||
|
|
||||||
|
private TaskHeartbeatVo touchBrandTaskIfRunning(BrandCrawlTaskEntity task, TaskHeartbeatRequest request) {
|
||||||
|
if (task == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
String status = task.getStatus();
|
||||||
|
if (!BRAND_STATUS_RUNNING.equals(status)) {
|
||||||
|
return TaskHeartbeatVo.notAlive(MODULE_BRAND, status, "task is not running");
|
||||||
|
}
|
||||||
|
int updated = brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper<BrandCrawlTaskEntity>()
|
||||||
|
.eq(BrandCrawlTaskEntity::getId, task.getId())
|
||||||
|
.eq(BrandCrawlTaskEntity::getUserId, task.getUserId())
|
||||||
|
.eq(BrandCrawlTaskEntity::getStatus, BRAND_STATUS_RUNNING)
|
||||||
|
.set(BrandCrawlTaskEntity::getUpdatedAt, LocalDateTime.now()));
|
||||||
|
if (updated <= 0) {
|
||||||
|
BrandCrawlTaskEntity latest = brandCrawlTaskMapper.selectById(task.getId());
|
||||||
|
return TaskHeartbeatVo.notAlive(MODULE_BRAND, latest == null ? status : latest.getStatus(), "task is not running");
|
||||||
|
}
|
||||||
|
brandTaskProgressCacheService.touchHeartbeat(
|
||||||
|
task.getId(),
|
||||||
|
request == null ? null : request.getPhase(),
|
||||||
|
request == null ? null : request.getCurrent(),
|
||||||
|
request == null ? null : request.getTotal());
|
||||||
|
return TaskHeartbeatVo.alive(MODULE_BRAND, BRAND_STATUS_RUNNING);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void touchModuleHeartbeat(String moduleType, Long taskId, TaskHeartbeatRequest request) {
|
||||||
|
switch (moduleType) {
|
||||||
|
case MODULE_PRODUCT_RISK -> {
|
||||||
|
productRiskTaskCacheService.touchTaskHeartbeat(taskId);
|
||||||
|
}
|
||||||
|
case MODULE_PRICE_TRACK -> {
|
||||||
|
priceTrackTaskCacheService.touchTaskHeartbeat(taskId);
|
||||||
|
}
|
||||||
|
case MODULE_SHOP_MATCH -> {
|
||||||
|
shopMatchTaskCacheService.touchTaskHeartbeat(taskId);
|
||||||
|
}
|
||||||
|
case MODULE_PATROL_DELETE -> {
|
||||||
|
patrolDeleteTaskCacheService.touchTaskHeartbeat(taskId);
|
||||||
|
}
|
||||||
|
case MODULE_QUERY_ASIN -> {
|
||||||
|
queryAsinTaskCacheService.touchTaskHeartbeat(taskId);
|
||||||
|
}
|
||||||
|
case MODULE_APPEARANCE_PATENT -> {
|
||||||
|
appearancePatentTaskCacheService.touchTaskHeartbeat(taskId);
|
||||||
|
}
|
||||||
|
case MODULE_SIMILAR_ASIN -> {
|
||||||
|
similarAsinTaskCacheService.touchTaskHeartbeat(taskId);
|
||||||
|
}
|
||||||
|
case MODULE_DELETE_BRAND -> {
|
||||||
|
deleteBrandTaskCacheService.saveProgress(taskId, buildDeleteBrandHeartbeatProgress(request), true);
|
||||||
|
}
|
||||||
|
default -> {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private Map<String, String> buildDeleteBrandHeartbeatProgress(TaskHeartbeatRequest request) {
|
||||||
|
String now = String.valueOf(System.currentTimeMillis());
|
||||||
|
Map<String, String> values = new LinkedHashMap<>();
|
||||||
|
values.put("last_heartbeat_at", now);
|
||||||
|
values.put("updated_at", now);
|
||||||
|
if (request != null) {
|
||||||
|
putIfPresent(values, "phase", request.getPhase());
|
||||||
|
putIfPresent(values, "current", request.getCurrent());
|
||||||
|
putIfPresent(values, "total", request.getTotal());
|
||||||
|
}
|
||||||
|
return values;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void saveFileTaskCache(String moduleType, FileTaskEntity task) {
|
||||||
|
switch (moduleType) {
|
||||||
|
case MODULE_PRODUCT_RISK -> productRiskTaskCacheService.saveTaskCache(task);
|
||||||
|
case MODULE_PRICE_TRACK -> priceTrackTaskCacheService.saveTaskCache(task);
|
||||||
|
case MODULE_SHOP_MATCH -> shopMatchTaskCacheService.saveTaskCache(task);
|
||||||
|
case MODULE_PATROL_DELETE -> patrolDeleteTaskCacheService.saveTaskCache(task);
|
||||||
|
case MODULE_QUERY_ASIN -> queryAsinTaskCacheService.saveTaskCache(task);
|
||||||
|
case MODULE_DELETE_BRAND -> deleteBrandTaskCacheService.saveTaskCache(task);
|
||||||
|
default -> {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void putIfPresent(Map<String, String> values, String key, Object value) {
|
||||||
|
if (value != null) {
|
||||||
|
values.put(key, String.valueOf(value));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,69 @@
|
|||||||
|
package com.nanri.aiimage.modules.task.service;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.common.service.DistributedJobLockService;
|
||||||
|
import com.nanri.aiimage.config.TaskImageCacheCleanupProperties;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.TaskImageCacheMapper;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.scheduling.annotation.Scheduled;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
|
import java.time.Duration;
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
|
||||||
|
@Service
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
@Slf4j
|
||||||
|
public class TaskImageCacheCleanupService {
|
||||||
|
|
||||||
|
private static final Duration CLEANUP_LOCK_TTL = Duration.ofMinutes(30);
|
||||||
|
|
||||||
|
private final TaskImageCacheMapper taskImageCacheMapper;
|
||||||
|
private final TaskImageCacheCleanupProperties cleanupProperties;
|
||||||
|
private final DistributedJobLockService distributedJobLockService;
|
||||||
|
|
||||||
|
@Scheduled(cron = "${aiimage.task-image-cache-cleanup.cron:0 30 2 * * *}")
|
||||||
|
public void cleanupExpiredImageCache() {
|
||||||
|
if (!cleanupProperties.isEnabled()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
DistributedJobLockService.LockHandle lockHandle =
|
||||||
|
distributedJobLockService.tryLock("task-image-cache:cleanup", CLEANUP_LOCK_TTL);
|
||||||
|
if (lockHandle == null) {
|
||||||
|
log.info("[task-image-cache-cleanup] skip because another instance holds the distributed lock");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try (lockHandle) {
|
||||||
|
int retentionDays = Math.max(1, cleanupProperties.getRetentionDays());
|
||||||
|
int batchSize = Math.max(1, cleanupProperties.getBatchSize());
|
||||||
|
int maxBatches = Math.max(1, cleanupProperties.getMaxBatchesPerRun());
|
||||||
|
LocalDateTime cutoff = LocalDateTime.now().minusDays(retentionDays);
|
||||||
|
|
||||||
|
int deletedTotal = 0;
|
||||||
|
int batches = 0;
|
||||||
|
while (batches < maxBatches) {
|
||||||
|
int deleted = taskImageCacheMapper.deleteExpiredBatch(cutoff, batchSize);
|
||||||
|
if (deleted <= 0) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
deletedTotal += deleted;
|
||||||
|
batches++;
|
||||||
|
if (!lockHandle.renew(CLEANUP_LOCK_TTL)) {
|
||||||
|
log.warn("[task-image-cache-cleanup] stop because lock renew failed deletedTotal={} batches={}",
|
||||||
|
deletedTotal, batches);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if (deleted < batchSize) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (deletedTotal > 0 || batches >= maxBatches) {
|
||||||
|
log.info("[task-image-cache-cleanup] finished cutoff={} retentionDays={} deletedTotal={} batches={} batchSize={} maxBatches={}",
|
||||||
|
cutoff, retentionDays, deletedTotal, batches, batchSize, maxBatches);
|
||||||
|
}
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[task-image-cache-cleanup] failed msg={}", ex.getMessage(), ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -121,6 +121,13 @@ public class TaskResultFileJobWorker {
|
|||||||
TaskDistributedLockService.LockHandle lockHandle =
|
TaskDistributedLockService.LockHandle lockHandle =
|
||||||
taskDistributedLockService.acquire(job.getModuleType(), job.getTaskId(), TaskDistributedLockService.DEFAULT_WAIT_MILLIS);
|
taskDistributedLockService.acquire(job.getModuleType(), job.getTaskId(), TaskDistributedLockService.DEFAULT_WAIT_MILLIS);
|
||||||
if (lockHandle == null) {
|
if (lockHandle == null) {
|
||||||
|
TaskFileJobEntity latestOnBusy = taskFileJobService.findById(job.getId());
|
||||||
|
if (latestOnBusy != null && "RUNNING".equals(latestOnBusy.getStatus())) {
|
||||||
|
taskFileJobService.touchRunning(job.getId());
|
||||||
|
log.info("[task-file-job] process skipped because task lock is busy and job is already running jobId={} taskId={} moduleType={} resultId={}",
|
||||||
|
job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId());
|
||||||
|
return;
|
||||||
|
}
|
||||||
taskFileJobService.requeue(job.getId(), "Task is busy, waiting for previous task operation");
|
taskFileJobService.requeue(job.getId(), "Task is busy, waiting for previous task operation");
|
||||||
log.info("[task-file-job] process requeued because task lock is busy jobId={} taskId={} moduleType={} resultId={}",
|
log.info("[task-file-job] process requeued because task lock is busy jobId={} taskId={} moduleType={} resultId={}",
|
||||||
job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId());
|
job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId());
|
||||||
|
|||||||
@@ -75,6 +75,14 @@ aiimage:
|
|||||||
# For IDEA/local startup, set -Daiimage.instance-id=<stable-id> or configure the AIIMAGE_INSTANCE_ID env var.
|
# For IDEA/local startup, set -Daiimage.instance-id=<stable-id> or configure the AIIMAGE_INSTANCE_ID env var.
|
||||||
# Avoid relying on container hostname/container ID, otherwise task owner continuity may break after restart/redeploy.
|
# Avoid relying on container hostname/container ID, otherwise task owner continuity may break after restart/redeploy.
|
||||||
instance-id: ${AIIMAGE_INSTANCE_ID:local-dev}
|
instance-id: ${AIIMAGE_INSTANCE_ID:local-dev}
|
||||||
|
instance-routing:
|
||||||
|
routes:
|
||||||
|
server-110: ${AIIMAGE_INSTANCE_ROUTE_SERVER_110:http://192.168.0.171:18080}
|
||||||
|
server-111: ${AIIMAGE_INSTANCE_ROUTE_SERVER_111:http://47.111.163.154:18080}
|
||||||
|
server-121: ${AIIMAGE_INSTANCE_ROUTE_SERVER_121:http://192.168.0.170:18080}
|
||||||
|
connect-timeout-millis: ${AIIMAGE_INSTANCE_ROUTE_CONNECT_TIMEOUT_MILLIS:3000}
|
||||||
|
read-timeout-millis: ${AIIMAGE_INSTANCE_ROUTE_READ_TIMEOUT_MILLIS:300000}
|
||||||
|
request-body-cache-limit-bytes: ${AIIMAGE_INSTANCE_ROUTE_REQUEST_BODY_CACHE_LIMIT_BYTES:104857600}
|
||||||
oss:
|
oss:
|
||||||
region: ${AIIMAGE_OSS_REGION:cn-hangzhou}
|
region: ${AIIMAGE_OSS_REGION:cn-hangzhou}
|
||||||
endpoint: ${AIIMAGE_OSS_ENDPOINT:oss-cn-hangzhou.aliyuncs.com}
|
endpoint: ${AIIMAGE_OSS_ENDPOINT:oss-cn-hangzhou.aliyuncs.com}
|
||||||
@@ -104,21 +112,17 @@ aiimage:
|
|||||||
brand-progress:
|
brand-progress:
|
||||||
ttl-hours: ${AIIMAGE_BRAND_PROGRESS_TTL_HOURS:24}
|
ttl-hours: ${AIIMAGE_BRAND_PROGRESS_TTL_HOURS:24}
|
||||||
failed-ttl-hours: ${AIIMAGE_BRAND_PROGRESS_FAILED_TTL_HOURS:2}
|
failed-ttl-hours: ${AIIMAGE_BRAND_PROGRESS_FAILED_TTL_HOURS:2}
|
||||||
heartbeat-timeout-minutes: ${AIIMAGE_BRAND_PROGRESS_HEARTBEAT_TIMEOUT_MINUTES:15}
|
heartbeat-timeout-minutes: ${AIIMAGE_BRAND_PROGRESS_HEARTBEAT_TIMEOUT_MINUTES:30}
|
||||||
stale-check-cron: ${AIIMAGE_BRAND_PROGRESS_STALE_CHECK_CRON:0 */2 * * * *}
|
stale-check-cron: ${AIIMAGE_BRAND_PROGRESS_STALE_CHECK_CRON:0 */2 * * * *}
|
||||||
delete-brand-progress:
|
delete-brand-progress:
|
||||||
heartbeat-timeout-minutes: ${AIIMAGE_DELETE_BRAND_HEARTBEAT_TIMEOUT_MINUTES:15}
|
heartbeat-timeout-minutes: ${AIIMAGE_DELETE_BRAND_HEARTBEAT_TIMEOUT_MINUTES:30}
|
||||||
delete-brand-initial-timeout-minutes: ${AIIMAGE_DELETE_BRAND_INITIAL_TIMEOUT_MINUTES:20}
|
|
||||||
stale-check-cron: ${AIIMAGE_DELETE_BRAND_STALE_CHECK_CRON:*/30 * * * * *}
|
stale-check-cron: ${AIIMAGE_DELETE_BRAND_STALE_CHECK_CRON:*/30 * * * * *}
|
||||||
finalize-check-cron: ${AIIMAGE_DELETE_BRAND_FINALIZE_CHECK_CRON:30 */2 * * * *}
|
finalize-check-cron: ${AIIMAGE_DELETE_BRAND_FINALIZE_CHECK_CRON:30 */2 * * * *}
|
||||||
product-risk-stale-timeout-minutes: ${AIIMAGE_PRODUCT_RISK_STALE_TIMEOUT_MINUTES:20}
|
product-risk-stale-timeout-minutes: ${AIIMAGE_PRODUCT_RISK_STALE_TIMEOUT_MINUTES:30}
|
||||||
product-risk-initial-timeout-minutes: ${AIIMAGE_PRODUCT_RISK_INITIAL_TIMEOUT_MINUTES:20}
|
price-track-stale-timeout-minutes: ${AIIMAGE_PRICE_TRACK_STALE_TIMEOUT_MINUTES:30}
|
||||||
price-track-stale-timeout-minutes: ${AIIMAGE_PRICE_TRACK_STALE_TIMEOUT_MINUTES:20}
|
shop-match-stale-timeout-minutes: ${AIIMAGE_SHOP_MATCH_STALE_TIMEOUT_MINUTES:30}
|
||||||
price-track-initial-timeout-minutes: ${AIIMAGE_PRICE_TRACK_INITIAL_TIMEOUT_MINUTES:20}
|
patrol-delete-stale-timeout-minutes: ${AIIMAGE_PATROL_DELETE_STALE_TIMEOUT_MINUTES:30}
|
||||||
shop-match-stale-timeout-minutes: ${AIIMAGE_SHOP_MATCH_STALE_TIMEOUT_MINUTES:20}
|
query-asin-stale-timeout-minutes: ${AIIMAGE_QUERY_ASIN_STALE_TIMEOUT_MINUTES:30}
|
||||||
shop-match-initial-timeout-minutes: ${AIIMAGE_SHOP_MATCH_INITIAL_TIMEOUT_MINUTES:20}
|
|
||||||
patrol-delete-stale-timeout-minutes: ${AIIMAGE_PATROL_DELETE_STALE_TIMEOUT_MINUTES:20}
|
|
||||||
patrol-delete-initial-timeout-minutes: ${AIIMAGE_PATROL_DELETE_INITIAL_TIMEOUT_MINUTES:20}
|
|
||||||
module-cleanup:
|
module-cleanup:
|
||||||
enabled: ${AIIMAGE_MODULE_CLEANUP_ENABLED:true}
|
enabled: ${AIIMAGE_MODULE_CLEANUP_ENABLED:true}
|
||||||
cron: ${AIIMAGE_MODULE_CLEANUP_CRON:0 0 0 * * *}
|
cron: ${AIIMAGE_MODULE_CLEANUP_CRON:0 0 0 * * *}
|
||||||
@@ -132,6 +136,12 @@ aiimage:
|
|||||||
scope-payload-buffer-retention-hours: ${AIIMAGE_TASK_SCOPE_PAYLOAD_BUFFER_RETENTION_HOURS:24}
|
scope-payload-buffer-retention-hours: ${AIIMAGE_TASK_SCOPE_PAYLOAD_BUFFER_RETENTION_HOURS:24}
|
||||||
scope-payload-recovery-max-files: ${AIIMAGE_TASK_SCOPE_PAYLOAD_RECOVERY_MAX_FILES:200}
|
scope-payload-recovery-max-files: ${AIIMAGE_TASK_SCOPE_PAYLOAD_RECOVERY_MAX_FILES:200}
|
||||||
scope-payload-cleanup-cron: ${AIIMAGE_TASK_SCOPE_PAYLOAD_CLEANUP_CRON:15 */30 * * * *}
|
scope-payload-cleanup-cron: ${AIIMAGE_TASK_SCOPE_PAYLOAD_CLEANUP_CRON:15 */30 * * * *}
|
||||||
|
task-image-cache-cleanup:
|
||||||
|
enabled: ${AIIMAGE_TASK_IMAGE_CACHE_CLEANUP_ENABLED:true}
|
||||||
|
cron: ${AIIMAGE_TASK_IMAGE_CACHE_CLEANUP_CRON:0 30 2 * * *}
|
||||||
|
retention-days: ${AIIMAGE_TASK_IMAGE_CACHE_RETENTION_DAYS:3}
|
||||||
|
batch-size: ${AIIMAGE_TASK_IMAGE_CACHE_CLEANUP_BATCH_SIZE:5000}
|
||||||
|
max-batches-per-run: ${AIIMAGE_TASK_IMAGE_CACHE_CLEANUP_MAX_BATCHES_PER_RUN:200}
|
||||||
result-file-job:
|
result-file-job:
|
||||||
mq-enabled: ${AIIMAGE_RESULT_FILE_JOB_MQ_ENABLED:true}
|
mq-enabled: ${AIIMAGE_RESULT_FILE_JOB_MQ_ENABLED:true}
|
||||||
topic: ${AIIMAGE_RESULT_FILE_JOB_TOPIC:aiimage-result-file-job}
|
topic: ${AIIMAGE_RESULT_FILE_JOB_TOPIC:aiimage-result-file-job}
|
||||||
@@ -149,6 +159,13 @@ aiimage:
|
|||||||
batch-size: ${AIIMAGE_RESULT_FILE_JOB_BATCH_SIZE:20}
|
batch-size: ${AIIMAGE_RESULT_FILE_JOB_BATCH_SIZE:20}
|
||||||
coze-task:
|
coze-task:
|
||||||
max-concurrent: ${AIIMAGE_COZE_TASK_MAX_CONCURRENT:12}
|
max-concurrent: ${AIIMAGE_COZE_TASK_MAX_CONCURRENT:12}
|
||||||
|
brand-check:
|
||||||
|
base-url: ${AIIMAGE_BRAND_CHECK_BASE_URL:http://47.110.241.161:16890}
|
||||||
|
path: ${AIIMAGE_BRAND_CHECK_PATH:/brand_check}
|
||||||
|
token: ${AIIMAGE_BRAND_CHECK_TOKEN:eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9}
|
||||||
|
default-strategy: ${AIIMAGE_BRAND_CHECK_DEFAULT_STRATEGY:Terms}
|
||||||
|
connect-timeout-millis: ${AIIMAGE_BRAND_CHECK_CONNECT_TIMEOUT_MILLIS:10000}
|
||||||
|
read-timeout-millis: ${AIIMAGE_BRAND_CHECK_READ_TIMEOUT_MILLIS:60000}
|
||||||
appearance-patent:
|
appearance-patent:
|
||||||
coze-base-url: ${AIIMAGE_APPEARANCE_PATENT_COZE_BASE_URL:https://api.coze.cn}
|
coze-base-url: ${AIIMAGE_APPEARANCE_PATENT_COZE_BASE_URL:https://api.coze.cn}
|
||||||
coze-workflow-path: ${AIIMAGE_APPEARANCE_PATENT_COZE_WORKFLOW_PATH:/v1/workflow/run}
|
coze-workflow-path: ${AIIMAGE_APPEARANCE_PATENT_COZE_WORKFLOW_PATH:/v1/workflow/run}
|
||||||
@@ -159,6 +176,7 @@ aiimage:
|
|||||||
coze-read-timeout-millis: ${AIIMAGE_APPEARANCE_PATENT_COZE_READ_TIMEOUT_MILLIS:60000}
|
coze-read-timeout-millis: ${AIIMAGE_APPEARANCE_PATENT_COZE_READ_TIMEOUT_MILLIS:60000}
|
||||||
coze-poll-interval-millis: ${AIIMAGE_APPEARANCE_PATENT_COZE_POLL_INTERVAL_MILLIS:30000}
|
coze-poll-interval-millis: ${AIIMAGE_APPEARANCE_PATENT_COZE_POLL_INTERVAL_MILLIS:30000}
|
||||||
coze-poll-timeout-millis: ${AIIMAGE_APPEARANCE_PATENT_COZE_POLL_TIMEOUT_MILLIS:600000}
|
coze-poll-timeout-millis: ${AIIMAGE_APPEARANCE_PATENT_COZE_POLL_TIMEOUT_MILLIS:600000}
|
||||||
|
coze-flush-pending-minutes: ${AIIMAGE_APPEARANCE_PATENT_COZE_FLUSH_PENDING_MINUTES:1}
|
||||||
stale-timeout-minutes: ${AIIMAGE_APPEARANCE_PATENT_STALE_TIMEOUT_MINUTES:30}
|
stale-timeout-minutes: ${AIIMAGE_APPEARANCE_PATENT_STALE_TIMEOUT_MINUTES:30}
|
||||||
stale-finalize-cron: ${AIIMAGE_APPEARANCE_PATENT_STALE_FINALIZE_CRON:0 */2 * * * *}
|
stale-finalize-cron: ${AIIMAGE_APPEARANCE_PATENT_STALE_FINALIZE_CRON:0 */2 * * * *}
|
||||||
similar-asin:
|
similar-asin:
|
||||||
|
|||||||
@@ -0,0 +1,80 @@
|
|||||||
|
package com.nanri.aiimage.common.util;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
|
||||||
|
class CozeGroupResultPropagatorTest {
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void doesNotTreatNoInfringementAsInfringementHit() {
|
||||||
|
List<ParsedRow> parsedRows = List.of(
|
||||||
|
new ParsedRow("20_1"),
|
||||||
|
new ParsedRow("20_2")
|
||||||
|
);
|
||||||
|
List<ResultRow> resultRows = List.of(
|
||||||
|
new ResultRow("无侵权"),
|
||||||
|
new ResultRow("无侵权")
|
||||||
|
);
|
||||||
|
|
||||||
|
int updatedRows = CozeGroupResultPropagator.propagateByGroup(
|
||||||
|
parsedRows,
|
||||||
|
ParsedRow::displayId,
|
||||||
|
row -> resultRows.get(parsedRows.indexOf(row)),
|
||||||
|
ResultRow::conclusion,
|
||||||
|
ResultRow::setConclusion,
|
||||||
|
List.of("已侵权", "侵权")
|
||||||
|
);
|
||||||
|
|
||||||
|
assertEquals(0, updatedRows);
|
||||||
|
assertEquals("无侵权", resultRows.get(0).conclusion());
|
||||||
|
assertEquals("无侵权", resultRows.get(1).conclusion());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void propagatesWhenGroupHasRealInfringementHit() {
|
||||||
|
List<ParsedRow> parsedRows = List.of(
|
||||||
|
new ParsedRow("20_1"),
|
||||||
|
new ParsedRow("20_2")
|
||||||
|
);
|
||||||
|
List<ResultRow> resultRows = List.of(
|
||||||
|
new ResultRow("无侵权"),
|
||||||
|
new ResultRow("侵权")
|
||||||
|
);
|
||||||
|
|
||||||
|
int updatedRows = CozeGroupResultPropagator.propagateByGroup(
|
||||||
|
parsedRows,
|
||||||
|
ParsedRow::displayId,
|
||||||
|
row -> resultRows.get(parsedRows.indexOf(row)),
|
||||||
|
ResultRow::conclusion,
|
||||||
|
ResultRow::setConclusion,
|
||||||
|
List.of("已侵权", "侵权")
|
||||||
|
);
|
||||||
|
|
||||||
|
assertEquals(1, updatedRows);
|
||||||
|
assertEquals("侵权", resultRows.get(0).conclusion());
|
||||||
|
assertEquals("侵权", resultRows.get(1).conclusion());
|
||||||
|
}
|
||||||
|
|
||||||
|
private record ParsedRow(String displayId) {
|
||||||
|
}
|
||||||
|
|
||||||
|
private static final class ResultRow {
|
||||||
|
|
||||||
|
private String conclusion;
|
||||||
|
|
||||||
|
private ResultRow(String conclusion) {
|
||||||
|
this.conclusion = conclusion;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String conclusion() {
|
||||||
|
return conclusion;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void setConclusion(String conclusion) {
|
||||||
|
this.conclusion = conclusion;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,64 @@
|
|||||||
|
package com.nanri.aiimage.modules.appearancepatent.client;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import com.nanri.aiimage.config.AppearancePatentProperties;
|
||||||
|
import com.nanri.aiimage.config.BrandCheckProperties;
|
||||||
|
import com.nanri.aiimage.modules.brand.client.BrandCheckClient;
|
||||||
|
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultRowDto;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import static org.assertj.core.api.Assertions.assertThat;
|
||||||
|
import static org.mockito.ArgumentMatchers.anyString;
|
||||||
|
import static org.mockito.Mockito.mock;
|
||||||
|
import static org.mockito.Mockito.never;
|
||||||
|
import static org.mockito.Mockito.verify;
|
||||||
|
|
||||||
|
class AppearancePatentCozeClientTest {
|
||||||
|
|
||||||
|
private final AppearancePatentCozeClient client = new AppearancePatentCozeClient(
|
||||||
|
new AppearancePatentProperties(),
|
||||||
|
new ObjectMapper(),
|
||||||
|
null,
|
||||||
|
new BrandCheckClient(new BrandCheckProperties())
|
||||||
|
);
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void titleRiskIsInfringementWhenBrandCheckHasFailedData() {
|
||||||
|
BrandCheckClient.BrandCheckBatchResult result =
|
||||||
|
new BrandCheckClient.BrandCheckBatchResult(List.of("阿凡达"), List.of("阿凡达"), List.of());
|
||||||
|
|
||||||
|
assertThat(client.buildTitleRisk("阿凡达", result)).isEqualTo("侵权");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void conclusionIsInfringementWhenTitleOrAppearanceIsInfringement() {
|
||||||
|
assertThat(client.buildConclusion("侵权", "无侵权", "")).isEqualTo("侵权");
|
||||||
|
assertThat(client.buildConclusion("无侵权", "侵权", "")).isEqualTo("侵权");
|
||||||
|
assertThat(client.buildConclusion("无侵权", "无侵权", "")).isEqualTo("无侵权");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void skipBrandCheckWhenCozeTitleAndReasonAreNone() throws Exception {
|
||||||
|
BrandCheckClient brandCheckClient = mock(BrandCheckClient.class);
|
||||||
|
AppearancePatentCozeClient cozeClient = new AppearancePatentCozeClient(
|
||||||
|
new AppearancePatentProperties(),
|
||||||
|
new ObjectMapper(),
|
||||||
|
null,
|
||||||
|
brandCheckClient
|
||||||
|
);
|
||||||
|
AppearancePatentResultRowDto row = new AppearancePatentResultRowDto();
|
||||||
|
row.setId("1");
|
||||||
|
String dataText = """
|
||||||
|
{"data":[{"row_id":"1","title":"无","title_reason":"无","appearance":"无侵权","result":"无侵权"}]}
|
||||||
|
""";
|
||||||
|
|
||||||
|
List<AppearancePatentResultRowDto> merged = cozeClient.mergeRowsFromDataText(List.of(row), dataText);
|
||||||
|
|
||||||
|
assertThat(merged).hasSize(1);
|
||||||
|
assertThat(merged.get(0).getTitleRisk()).isEqualTo("无侵权");
|
||||||
|
assertThat(merged.get(0).getConclusion()).isEqualTo("无侵权");
|
||||||
|
verify(brandCheckClient, never()).checkTitleText(anyString(), anyString());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
package com.nanri.aiimage.modules.brand.client;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.config.BrandCheckProperties;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import static org.assertj.core.api.Assertions.assertThat;
|
||||||
|
|
||||||
|
class BrandCheckClientTest {
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void splitTitleTextSupportsCommonSeparatorsAndQuotes() {
|
||||||
|
BrandCheckClient client = new BrandCheckClient(new BrandCheckProperties());
|
||||||
|
|
||||||
|
List<String> brands = client.splitTitleText("'阿凡达,任天堂'; Disney、LEGO\nSony");
|
||||||
|
|
||||||
|
assertThat(brands).containsExactly("阿凡达", "任天堂", "Disney", "LEGO", "Sony");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void splitTitleTextDeduplicatesBlankValues() {
|
||||||
|
BrandCheckClient client = new BrandCheckClient(new BrandCheckProperties());
|
||||||
|
|
||||||
|
List<String> brands = client.splitTitleText(" 任天堂, ,任天堂,Sony ");
|
||||||
|
|
||||||
|
assertThat(brands).containsExactly("任天堂", "Sony");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,86 @@
|
|||||||
|
package com.nanri.aiimage.modules.similarasin.client;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.core.type.TypeReference;
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||||
|
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
|
import java.math.BigDecimal;
|
||||||
|
import java.lang.reflect.Method;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
|
||||||
|
class SimilarAsinCozeClientTest {
|
||||||
|
|
||||||
|
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||||
|
|
||||||
|
@Test
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
void buildParametersIncludesAlibabaPriceFromPythonPayload() throws Exception {
|
||||||
|
List<SimilarAsinResultRowDto> rows = objectMapper.readValue("""
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"asin": "B0TEST123",
|
||||||
|
"url": "https://m.media-amazon.com/images/I/main.jpg",
|
||||||
|
"alibaba": [
|
||||||
|
{"url": "https://cbu01.alicdn.com/img/ibank/a.jpg", "price": 12.80},
|
||||||
|
{"url": "https://cbu01.alicdn.com/img/ibank/b.jpg", "price": 19.99}
|
||||||
|
],
|
||||||
|
"title": "Test title",
|
||||||
|
"sku": "SKU-1"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
""", new TypeReference<>() {
|
||||||
|
});
|
||||||
|
SimilarAsinCozeClient client = new SimilarAsinCozeClient(new SimilarAsinProperties(), objectMapper, null);
|
||||||
|
|
||||||
|
Method method = SimilarAsinCozeClient.class.getDeclaredMethod(
|
||||||
|
"buildParameters", List.class, String.class, String.class, boolean.class);
|
||||||
|
method.setAccessible(true);
|
||||||
|
Map<String, Object> parameters = (Map<String, Object>) method.invoke(client, rows, "", "", true);
|
||||||
|
|
||||||
|
List<Map<String, Object>> items = (List<Map<String, Object>>) parameters.get("items");
|
||||||
|
Map<String, Object> item = items.getFirst();
|
||||||
|
List<Map<String, Object>> alibaba = (List<Map<String, Object>>) item.get("alibaba");
|
||||||
|
|
||||||
|
assertEquals("B0TEST123", item.get("asin"));
|
||||||
|
assertEquals("https://m.media-amazon.com/images/I/main.jpg", item.get("url"));
|
||||||
|
assertEquals("Test title", item.get("title"));
|
||||||
|
assertEquals("SKU-1", item.get("sku"));
|
||||||
|
assertEquals(2, alibaba.size());
|
||||||
|
assertEquals("https://cbu01.alicdn.com/img/ibank/a.jpg", alibaba.get(0).get("url"));
|
||||||
|
assertEquals(0, new BigDecimal("12.8").compareTo((BigDecimal) alibaba.get(0).get("price")));
|
||||||
|
assertEquals("https://cbu01.alicdn.com/img/ibank/b.jpg", alibaba.get(1).get("url"));
|
||||||
|
assertEquals(0, new BigDecimal("19.99").compareTo((BigDecimal) alibaba.get(1).get("price")));
|
||||||
|
assertTrue(rows.getFirst().hasImageUrl());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
void buildParametersFallsBackAlibabaFromLegacyUrlsAndTopLevelPrice() throws Exception {
|
||||||
|
SimilarAsinResultRowDto row = new SimilarAsinResultRowDto();
|
||||||
|
row.setAsin("B0LEGACY1");
|
||||||
|
row.setUrl("https://m.media-amazon.com/images/I/main.jpg");
|
||||||
|
row.setUrls(List.of("https://cbu01.alicdn.com/img/ibank/legacy-a.jpg"));
|
||||||
|
row.setPrice("8.50");
|
||||||
|
row.setTitle("Legacy title");
|
||||||
|
row.setSku("SKU-LEGACY");
|
||||||
|
SimilarAsinCozeClient client = new SimilarAsinCozeClient(new SimilarAsinProperties(), objectMapper, null);
|
||||||
|
|
||||||
|
Method method = SimilarAsinCozeClient.class.getDeclaredMethod(
|
||||||
|
"buildParameters", List.class, String.class, String.class, boolean.class);
|
||||||
|
method.setAccessible(true);
|
||||||
|
Map<String, Object> parameters = (Map<String, Object>) method.invoke(client, List.of(row), "", "", false);
|
||||||
|
|
||||||
|
List<Map<String, Object>> items = (List<Map<String, Object>>) parameters.get("items");
|
||||||
|
List<Map<String, Object>> alibaba = (List<Map<String, Object>>) items.getFirst().get("alibaba");
|
||||||
|
|
||||||
|
assertEquals(1, alibaba.size());
|
||||||
|
assertEquals("https://cbu01.alicdn.com/img/ibank/legacy-a.jpg", alibaba.getFirst().get("url"));
|
||||||
|
assertEquals(0, new BigDecimal("8.5").compareTo((BigDecimal) alibaba.getFirst().get("price")));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -7,13 +7,16 @@ import java.awt.Color;
|
|||||||
import java.awt.Graphics2D;
|
import java.awt.Graphics2D;
|
||||||
import java.awt.image.BufferedImage;
|
import java.awt.image.BufferedImage;
|
||||||
import java.io.ByteArrayOutputStream;
|
import java.io.ByteArrayOutputStream;
|
||||||
|
import java.io.IOException;
|
||||||
import java.net.InetAddress;
|
import java.net.InetAddress;
|
||||||
import java.net.UnknownHostException;
|
import java.net.UnknownHostException;
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import java.util.concurrent.TimeoutException;
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||||
@@ -154,4 +157,47 @@ class SimilarAsinImageEmbedderTest {
|
|||||||
assertEquals(12345678L, ex.size());
|
assertEquals(12345678L, ex.size());
|
||||||
assertTrue(ex.getMessage().contains("oversize"));
|
assertTrue(ex.getMessage().contains("oversize"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void httpClientKeepsImageDownloadResilienceEnabled() {
|
||||||
|
assertTrue(embedder.httpClient().retryOnConnectionFailure(), "OkHttp should retry transient connection failures");
|
||||||
|
assertTrue(embedder.httpClient().followRedirects(), "image CDN redirects should be followed");
|
||||||
|
assertTrue(embedder.httpClient().followSslRedirects(), "signed CDN downloads should follow browser-like SSL redirects");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void buildImageRequestUsesBrowserLikeHeaders() {
|
||||||
|
var req = SimilarAsinImageEmbedder.buildImageRequest("https://m.media-amazon.com/images/I/abc.jpg");
|
||||||
|
|
||||||
|
assertEquals("GET", req.method());
|
||||||
|
assertTrue(req.header("User-Agent").contains("Chrome"), "User-Agent should look browser-like");
|
||||||
|
assertTrue(req.header("Accept").contains("image/"), "Accept should prefer images");
|
||||||
|
assertEquals("https://www.amazon.com/", req.header("Referer"));
|
||||||
|
assertEquals("no-cache", req.header("Cache-Control"));
|
||||||
|
assertNotNull(req.header("Accept-Language"));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void buildImageRequestUsesDownloadHeadersForCozeSignedImages() {
|
||||||
|
var req = SimilarAsinImageEmbedder.buildImageRequest(
|
||||||
|
"https://lf9-bot-platform-tos-sign.coze.cn/bot-studio-bot-platform/bot_files/1/image/jpeg/2/merged_image.jpg?x-expires=1&x-signature=a");
|
||||||
|
|
||||||
|
assertEquals("GET", req.method());
|
||||||
|
assertTrue(req.header("User-Agent").contains("Chrome"), "User-Agent should look browser-like");
|
||||||
|
assertEquals("*/*", req.header("Accept"));
|
||||||
|
assertNull(req.header("Referer"), "Coze signed download links should not carry an Amazon referer");
|
||||||
|
assertEquals("no-cache", req.header("Cache-Control"));
|
||||||
|
assertTrue(SimilarAsinImageEmbedder.isCozeSignedImageUrl(req.url().toString()));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void errorSummaryDoesNotHideEmptyTimeoutMessage() {
|
||||||
|
TimeoutException timeout = new TimeoutException();
|
||||||
|
assertEquals("TimeoutException: <empty>", SimilarAsinImageEmbedder.errorSummary(timeout));
|
||||||
|
|
||||||
|
IOException io = new IOException("outer", new TimeoutException("inner"));
|
||||||
|
String summary = SimilarAsinImageEmbedder.errorSummary(io);
|
||||||
|
assertTrue(summary.contains("IOException: outer"));
|
||||||
|
assertTrue(summary.contains("cause=TimeoutException: inner"));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -196,7 +196,7 @@ def _ensure_column_sort_schema():
|
|||||||
try:
|
try:
|
||||||
with conn.cursor() as cur:
|
with conn.cursor() as cur:
|
||||||
try:
|
try:
|
||||||
cur.execute("ALTER TABLE columns ADD COLUMN sort_order INT NOT NULL DEFAULT 0 COMMENT '鑿滃崟鎺掑簭' AFTER route_path")
|
cur.execute("ALTER TABLE columns ADD COLUMN sort_order INT NOT NULL DEFAULT 0 COMMENT '菜单排序' AFTER route_path")
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -34,7 +34,7 @@
|
|||||||
v-model="secretStates[config.key].value"
|
v-model="secretStates[config.key].value"
|
||||||
class="secret-input"
|
class="secret-input"
|
||||||
type="password"
|
type="password"
|
||||||
placeholder="请输入 Coze 接口密钥"
|
:placeholder="config.placeholder"
|
||||||
autocomplete="off"
|
autocomplete="off"
|
||||||
spellcheck="false"
|
spellcheck="false"
|
||||||
/>
|
/>
|
||||||
@@ -96,21 +96,30 @@ const retentionOptions: Array<{ value: ApiSecretRetention; label: string }> = [
|
|||||||
{ value: 'forever', label: '长期保留' },
|
{ value: 'forever', label: '长期保留' },
|
||||||
]
|
]
|
||||||
|
|
||||||
const secretConfigs: Array<{ key: ApiSecretModuleKey; title: string; description: string }> = [
|
const secretConfigs: Array<{ key: ApiSecretModuleKey; title: string; description: string; placeholder: string }> = [
|
||||||
{
|
{
|
||||||
key: 'appearance-patent',
|
key: 'appearance-patent',
|
||||||
title: '外观专利密钥',
|
title: '外观专利密钥',
|
||||||
description: '仅用于外观专利检测。',
|
description: '仅用于外观专利检测。',
|
||||||
|
placeholder: '请输入 Coze 接口密钥',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
key: 'appearance-patent-token',
|
||||||
|
title: '专利汇令牌',
|
||||||
|
description: '仅用于外观专利检测,非必填。',
|
||||||
|
placeholder: '请输入专利汇令牌,可留空',
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
key: 'similar-asin',
|
key: 'similar-asin',
|
||||||
title: '货源查询密钥',
|
title: '货源查询密钥',
|
||||||
description: '仅用于货源查询。',
|
description: '仅用于货源查询。',
|
||||||
|
placeholder: '请输入 Coze 接口密钥',
|
||||||
},
|
},
|
||||||
]
|
]
|
||||||
|
|
||||||
const secretStates = ref<Record<ApiSecretModuleKey, SecretState>>({
|
const secretStates = ref<Record<ApiSecretModuleKey, SecretState>>({
|
||||||
'appearance-patent': emptySecretState(),
|
'appearance-patent': emptySecretState(),
|
||||||
|
'appearance-patent-token': emptySecretState(),
|
||||||
'similar-asin': emptySecretState(),
|
'similar-asin': emptySecretState(),
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|||||||
@@ -261,6 +261,10 @@ function effectiveCozeApiKey() {
|
|||||||
return getStoredApiSecret('appearance-patent').trim()
|
return getStoredApiSecret('appearance-patent').trim()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function effectivePatentToken() {
|
||||||
|
return getStoredApiSecret('appearance-patent-token').trim()
|
||||||
|
}
|
||||||
|
|
||||||
function maskSecret(secret: string) {
|
function maskSecret(secret: string) {
|
||||||
if (!secret) return ''
|
if (!secret) return ''
|
||||||
if (secret.length <= 10) return '***'
|
if (secret.length <= 10) return '***'
|
||||||
@@ -274,6 +278,7 @@ function payloadForDisplay<T extends { data?: Record<string, unknown> }>(payload
|
|||||||
? {
|
? {
|
||||||
...payload.data,
|
...payload.data,
|
||||||
api_key: maskSecret(String(payload.data.api_key || '')),
|
api_key: maskSecret(String(payload.data.api_key || '')),
|
||||||
|
patent_token: maskSecret(String(payload.data.patent_token || '')),
|
||||||
}
|
}
|
||||||
: payload.data,
|
: payload.data,
|
||||||
}
|
}
|
||||||
@@ -406,7 +411,7 @@ async function parseFiles() {
|
|||||||
originalFilename: f.originalFilename,
|
originalFilename: f.originalFilename,
|
||||||
relativePath: f.relativePath,
|
relativePath: f.relativePath,
|
||||||
}))
|
}))
|
||||||
const res = await parseAppearancePatent(files, effectiveAiPrompt(), effectiveCozeApiKey())
|
const res = await parseAppearancePatent(files, effectiveAiPrompt(), effectiveCozeApiKey(), effectivePatentToken())
|
||||||
parseResult.value = res
|
parseResult.value = res
|
||||||
queuedTaskSummary.value = null
|
queuedTaskSummary.value = null
|
||||||
queuePayloadText.value = ''
|
queuePayloadText.value = ''
|
||||||
@@ -445,6 +450,7 @@ async function pushToPythonQueue() {
|
|||||||
taskId,
|
taskId,
|
||||||
prompt: queueAiPrompt(),
|
prompt: queueAiPrompt(),
|
||||||
api_key: effectiveCozeApiKey(),
|
api_key: effectiveCozeApiKey(),
|
||||||
|
patent_token: effectivePatentToken(),
|
||||||
sourceFileCount: currentParseResult.sourceFileCount || 0,
|
sourceFileCount: currentParseResult.sourceFileCount || 0,
|
||||||
totalRows: currentParseResult.totalRows || 0,
|
totalRows: currentParseResult.totalRows || 0,
|
||||||
acceptedRows: currentParseResult.acceptedRows || 0,
|
acceptedRows: currentParseResult.acceptedRows || 0,
|
||||||
|
|||||||
@@ -1529,6 +1529,7 @@ export interface AppearancePatentParseVo {
|
|||||||
export interface AppearancePatentParsedPayloadDto {
|
export interface AppearancePatentParsedPayloadDto {
|
||||||
aiPrompt?: string;
|
aiPrompt?: string;
|
||||||
apiKey?: string;
|
apiKey?: string;
|
||||||
|
patentToken?: string;
|
||||||
sourceFiles?: UploadedFileRef[];
|
sourceFiles?: UploadedFileRef[];
|
||||||
headers?: string[];
|
headers?: string[];
|
||||||
items?: AppearancePatentParsedRow[];
|
items?: AppearancePatentParsedRow[];
|
||||||
@@ -1590,16 +1591,17 @@ export interface AppearancePatentTaskBatchVo {
|
|||||||
missingTaskIds?: number[];
|
missingTaskIds?: number[];
|
||||||
}
|
}
|
||||||
|
|
||||||
export function parseAppearancePatent(files: UploadedFileRef[], aiPrompt: string, apiKey?: string) {
|
export function parseAppearancePatent(files: UploadedFileRef[], aiPrompt: string, apiKey?: string, patentToken?: string) {
|
||||||
return unwrapJavaResponse(
|
return unwrapJavaResponse(
|
||||||
post<
|
post<
|
||||||
JavaApiResponse<AppearancePatentParseVo>,
|
JavaApiResponse<AppearancePatentParseVo>,
|
||||||
{ user_id: number; files: UploadedFileRef[]; ai_prompt: string; api_key?: string }
|
{ user_id: number; files: UploadedFileRef[]; ai_prompt: string; api_key?: string; patent_token?: string }
|
||||||
>(`${JAVA_API_PREFIX}/appearance-patent/parse`, {
|
>(`${JAVA_API_PREFIX}/appearance-patent/parse`, {
|
||||||
user_id: getCurrentUserId(),
|
user_id: getCurrentUserId(),
|
||||||
files,
|
files,
|
||||||
ai_prompt: aiPrompt,
|
ai_prompt: aiPrompt,
|
||||||
api_key: apiKey,
|
api_key: apiKey,
|
||||||
|
patent_token: patentToken,
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
export type ApiSecretModuleKey = 'appearance-patent' | 'similar-asin'
|
export type ApiSecretModuleKey = 'appearance-patent' | 'appearance-patent-token' | 'similar-asin'
|
||||||
|
|
||||||
export type ApiSecretRetention = 'session' | '1d' | '7d' | '30d' | 'forever'
|
export type ApiSecretRetention = 'session' | '1d' | '7d' | '30d' | 'forever'
|
||||||
|
|
||||||
@@ -19,7 +19,7 @@ export type ApiSecretSnapshot = {
|
|||||||
|
|
||||||
const STORAGE_PREFIX = 'brand:api-secret'
|
const STORAGE_PREFIX = 'brand:api-secret'
|
||||||
const COMMON_SECRET_KEY = 'common'
|
const COMMON_SECRET_KEY = 'common'
|
||||||
const MODULE_SECRET_KEYS: ApiSecretModuleKey[] = ['appearance-patent', 'similar-asin']
|
const MODULE_SECRET_KEYS: ApiSecretModuleKey[] = ['appearance-patent', 'appearance-patent-token', 'similar-asin']
|
||||||
|
|
||||||
function currentUserStorageId() {
|
function currentUserStorageId() {
|
||||||
if (typeof window === 'undefined') return '0'
|
if (typeof window === 'undefined') return '0'
|
||||||
@@ -115,7 +115,7 @@ function clearLegacyStoredApiSecrets() {
|
|||||||
function migrateCommonRecord(record: ApiSecretRecord) {
|
function migrateCommonRecord(record: ApiSecretRecord) {
|
||||||
if (typeof window === 'undefined') return
|
if (typeof window === 'undefined') return
|
||||||
const storage = record.retention === 'session' ? window.sessionStorage : window.localStorage
|
const storage = record.retention === 'session' ? window.sessionStorage : window.localStorage
|
||||||
for (const moduleKey of MODULE_SECRET_KEYS) {
|
for (const moduleKey of ['appearance-patent', 'similar-asin'] satisfies ApiSecretModuleKey[]) {
|
||||||
if (!getLiveRecordFromKey(moduleKey)) {
|
if (!getLiveRecordFromKey(moduleKey)) {
|
||||||
storage.setItem(buildStorageKey(moduleKey), JSON.stringify(record))
|
storage.setItem(buildStorageKey(moduleKey), JSON.stringify(record))
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user