Merge branch 'master' of https://gitee.com/TeaCodeNice/crawler-plugin
This commit is contained in:
2
.gitignore
vendored
2
.gitignore
vendored
@@ -43,3 +43,5 @@ app/assets/
|
|||||||
app/new_web_source
|
app/new_web_source
|
||||||
app/user_data/
|
app/user_data/
|
||||||
OPS_REDIS_MYSQL_OPTIMIZATION_NOTES.md
|
OPS_REDIS_MYSQL_OPTIMIZATION_NOTES.md
|
||||||
|
|
||||||
|
架构.md
|
||||||
6
app/.env
6
app/.env
@@ -13,7 +13,13 @@ client_name=ShuFuAI
|
|||||||
|
|
||||||
|
|
||||||
# java_api_base=http://47.111.163.154:18080
|
# java_api_base=http://47.111.163.154:18080
|
||||||
|
<<<<<<< HEAD
|
||||||
# java_api_base=http://127.0.0.1:18080
|
# java_api_base=http://127.0.0.1:18080
|
||||||
java_api_base=http://8.136.19.173:18080
|
java_api_base=http://8.136.19.173:18080
|
||||||
|
=======
|
||||||
|
java_api_base=http://127.0.0.1:18080
|
||||||
|
# java_api_base=http://8.136.19.173:18080
|
||||||
|
# java_api_base=http://121.196.149.225:18080
|
||||||
|
>>>>>>> e4bf104ae20efcb34816c5d3da9c3372ecd92d93
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Binary file not shown.
BIN
app/amazon/__pycache__/asin_status.cpython-312.pyc
Normal file
BIN
app/amazon/__pycache__/asin_status.cpython-312.pyc
Normal file
Binary file not shown.
BIN
app/amazon/__pycache__/detail_spider.cpython-312.pyc
Normal file
BIN
app/amazon/__pycache__/detail_spider.cpython-312.pyc
Normal file
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -441,26 +441,39 @@ class SpiderTask:
|
|||||||
runing_task[task_id]["status"] = "failed"
|
runing_task[task_id]["status"] = "failed"
|
||||||
runing_task[task_id]["error"] = str(e)
|
runing_task[task_id]["error"] = str(e)
|
||||||
|
|
||||||
def post_result(self, task_id: int, chunkIndex:int,chunkTotal: int, asin: str, error:str="",
|
def post_result(self, task_id: int, chunkIndex:int,chunkTotal: int, asin: str, error:str="",
|
||||||
item_data:dict={},
|
item_data:dict={},
|
||||||
is_done: bool = False):
|
is_done: bool = False):
|
||||||
"""回传处理结果到API
|
"""回传处理结果到API
|
||||||
"""
|
"""
|
||||||
|
|
||||||
url = f"{DELETE_BRAND_API_BASE}/api/appearance-patent/tasks/{task_id}/result"
|
url = f"{DELETE_BRAND_API_BASE}/api/appearance-patent/tasks/{task_id}/result"
|
||||||
|
|
||||||
|
<<<<<<< HEAD
|
||||||
payload ={
|
payload ={
|
||||||
"submissionId": f"{int(time.time())}",
|
"submissionId": f"{int(time.time())}",
|
||||||
"chunkIndex": chunkIndex,
|
"chunkIndex": chunkIndex,
|
||||||
"chunkTotal": chunkTotal,
|
"chunkTotal": chunkTotal,
|
||||||
"error": error,
|
"error": error,
|
||||||
"groups": item_data,
|
"groups": item_data,
|
||||||
|
=======
|
||||||
|
submission_id = f"appearance-patent:{task_id}"
|
||||||
|
payload ={
|
||||||
|
"submissionId": submission_id,
|
||||||
|
"chunkIndex": chunkIndex,
|
||||||
|
"chunkTotal": chunkTotal,
|
||||||
|
"error": error,
|
||||||
|
"items": [
|
||||||
|
item_data
|
||||||
|
],
|
||||||
|
>>>>>>> e4bf104ae20efcb34816c5d3da9c3372ecd92d93
|
||||||
"done": is_done
|
"done": is_done
|
||||||
}
|
}
|
||||||
|
|
||||||
max_retries = 3
|
max_retries = 3
|
||||||
for retry in range(max_retries):
|
for retry in range(max_retries):
|
||||||
try:
|
try:
|
||||||
|
request_timeout = 300 if is_done else 30
|
||||||
print("================【详情采集】=====================")
|
print("================【详情采集】=====================")
|
||||||
self.log(f"尝试回传结果 (第 {retry + 1}/{max_retries} 次)")
|
self.log(f"尝试回传结果 (第 {retry + 1}/{max_retries} 次)")
|
||||||
self.log(f"回传URL: {url}")
|
self.log(f"回传URL: {url}")
|
||||||
@@ -469,7 +482,7 @@ class SpiderTask:
|
|||||||
url,
|
url,
|
||||||
json=payload,
|
json=payload,
|
||||||
headers={"Content-Type": "application/json"},
|
headers={"Content-Type": "application/json"},
|
||||||
timeout=30,
|
timeout=request_timeout,
|
||||||
verify=False
|
verify=False
|
||||||
)
|
)
|
||||||
self.log(f"回传结果: {response.text}")
|
self.log(f"回传结果: {response.text}")
|
||||||
@@ -506,6 +519,7 @@ if __name__ == '__main__':
|
|||||||
"url": "",
|
"url": "",
|
||||||
"title": "低"
|
"title": "低"
|
||||||
},
|
},
|
||||||
|
<<<<<<< HEAD
|
||||||
{
|
{
|
||||||
"sourceFileKey": "",
|
"sourceFileKey": "",
|
||||||
"sourceFilename": "",
|
"sourceFilename": "",
|
||||||
@@ -8374,4 +8388,9 @@ if __name__ == '__main__':
|
|||||||
]
|
]
|
||||||
# spide.process_task(task_data)
|
# spide.process_task(task_data)
|
||||||
res = SpiderTask.group_by_id_prefix(task_data)
|
res = SpiderTask.group_by_id_prefix(task_data)
|
||||||
print(res)
|
print(res)
|
||||||
|
=======
|
||||||
|
"code": None
|
||||||
|
}
|
||||||
|
spide.process_task(task_data)
|
||||||
|
>>>>>>> e4bf104ae20efcb34816c5d3da9c3372ecd92d93
|
||||||
|
|||||||
@@ -785,7 +785,24 @@
|
|||||||
else if (status === 'cancelled') showToast('已取消');
|
else if (status === 'cancelled') showToast('已取消');
|
||||||
}
|
}
|
||||||
|
|
||||||
function pollRunTask(taskId) {
|
function verifyTerminalTaskStatus(taskId, expectedStatus) {
|
||||||
|
return fetch('/api/brand/tasks/' + taskId, {
|
||||||
|
credentials: 'include',
|
||||||
|
headers: withUidHeaders({ 'X-Requested-With': 'XMLHttpRequest' })
|
||||||
|
})
|
||||||
|
.then(function(r) { return r.json(); })
|
||||||
|
.then(function(res) {
|
||||||
|
if (!res.success || !res.task) return null;
|
||||||
|
var actualStatus = (res.task.status || '').toLowerCase();
|
||||||
|
if (actualStatus === 'success' || actualStatus === 'failed' || actualStatus === 'cancelled') {
|
||||||
|
return actualStatus;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
})
|
||||||
|
.catch(function() { return null; });
|
||||||
|
}
|
||||||
|
|
||||||
|
function pollRunTask(taskId) {
|
||||||
pollTaskId = taskId;
|
pollTaskId = taskId;
|
||||||
var runProgressFill = document.getElementById('runProgressFill');
|
var runProgressFill = document.getElementById('runProgressFill');
|
||||||
var runProgressText = document.getElementById('runProgressText');
|
var runProgressText = document.getElementById('runProgressText');
|
||||||
@@ -848,15 +865,18 @@
|
|||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
} catch (err) {}
|
} catch (err) {}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (terminalStatus) {
|
if (terminalStatus) {
|
||||||
try { runTaskEventAbortController.abort(); } catch (e2) {}
|
var verifiedStatus = await verifyTerminalTaskStatus(taskId, terminalStatus);
|
||||||
runTaskEventAbortController = null;
|
if (verifiedStatus) {
|
||||||
onRunTaskFinished(terminalStatus);
|
try { runTaskEventAbortController.abort(); } catch (e2) {}
|
||||||
return;
|
runTaskEventAbortController = null;
|
||||||
}
|
onRunTaskFinished(verifiedStatus);
|
||||||
}
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
} catch (err) {}
|
} catch (err) {}
|
||||||
})();
|
})();
|
||||||
|
|
||||||
@@ -918,7 +938,7 @@
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
tick();
|
tick();
|
||||||
pollTimer = setInterval(tick, 10000);
|
pollTimer = setInterval(tick, 3000);
|
||||||
}
|
}
|
||||||
|
|
||||||
document.getElementById('btnRun').onclick = function() {
|
document.getElementById('btnRun').onclick = function() {
|
||||||
@@ -1128,7 +1148,11 @@
|
|||||||
});
|
});
|
||||||
|
|
||||||
loadTasks();
|
loadTasks();
|
||||||
// setInterval(loadTasks, 10000);
|
setInterval(function() {
|
||||||
|
if (cachedTasks.some(function(t) { return (t.status || '').toLowerCase() === 'running'; })) {
|
||||||
|
loadTasks();
|
||||||
|
}
|
||||||
|
}, 3000);
|
||||||
|
|
||||||
if (window.addEventListener) {
|
if (window.addEventListener) {
|
||||||
window.addEventListener('pywebviewready', function() {
|
window.addEventListener('pywebviewready', function() {
|
||||||
|
|||||||
1106
backend-java/jstack-33832.txt
Normal file
1106
backend-java/jstack-33832.txt
Normal file
File diff suppressed because it is too large
Load Diff
1056
backend-java/jstack-6488.txt
Normal file
1056
backend-java/jstack-6488.txt
Normal file
File diff suppressed because it is too large
Load Diff
@@ -23,6 +23,8 @@
|
|||||||
<easyexcel.version>4.0.3</easyexcel.version>
|
<easyexcel.version>4.0.3</easyexcel.version>
|
||||||
<aliyun.oss.version>3.17.4</aliyun.oss.version>
|
<aliyun.oss.version>3.17.4</aliyun.oss.version>
|
||||||
<hutool.version>5.8.36</hutool.version>
|
<hutool.version>5.8.36</hutool.version>
|
||||||
|
<rocketmq-spring.version>2.3.5</rocketmq-spring.version>
|
||||||
|
<minio.version>8.5.17</minio.version>
|
||||||
</properties>
|
</properties>
|
||||||
|
|
||||||
<dependencies>
|
<dependencies>
|
||||||
@@ -81,6 +83,16 @@
|
|||||||
<artifactId>hutool-all</artifactId>
|
<artifactId>hutool-all</artifactId>
|
||||||
<version>${hutool.version}</version>
|
<version>${hutool.version}</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.rocketmq</groupId>
|
||||||
|
<artifactId>rocketmq-spring-boot-starter</artifactId>
|
||||||
|
<version>${rocketmq-spring.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>io.minio</groupId>
|
||||||
|
<artifactId>minio</artifactId>
|
||||||
|
<version>${minio.version}</version>
|
||||||
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.projectlombok</groupId>
|
<groupId>org.projectlombok</groupId>
|
||||||
<artifactId>lombok</artifactId>
|
<artifactId>lombok</artifactId>
|
||||||
|
|||||||
@@ -13,6 +13,9 @@ public class GlobalExceptionHandler {
|
|||||||
|
|
||||||
@ExceptionHandler(BusinessException.class)
|
@ExceptionHandler(BusinessException.class)
|
||||||
public ApiResponse<Void> handleBusinessException(BusinessException ex) {
|
public ApiResponse<Void> handleBusinessException(BusinessException ex) {
|
||||||
|
if (Integer.valueOf(40901).equals(ex.getCode())) {
|
||||||
|
return ApiResponse.success("任务已结束,忽略重复提交", null);
|
||||||
|
}
|
||||||
return ex.getCode() == null
|
return ex.getCode() == null
|
||||||
? ApiResponse.fail(ex.getMessage())
|
? ApiResponse.fail(ex.getMessage())
|
||||||
: ApiResponse.fail(ex.getCode(), ex.getMessage());
|
: ApiResponse.fail(ex.getCode(), ex.getMessage());
|
||||||
|
|||||||
@@ -36,7 +36,13 @@ public class DistributedJobLockService {
|
|||||||
Duration actualTtl = (ttl == null || ttl.isNegative() || ttl.isZero()) ? Duration.ofMinutes(10) : ttl;
|
Duration actualTtl = (ttl == null || ttl.isNegative() || ttl.isZero()) ? Duration.ofMinutes(10) : ttl;
|
||||||
String key = buildLockKey(jobName);
|
String key = buildLockKey(jobName);
|
||||||
String token = ownerPrefix + ":" + UUID.randomUUID();
|
String token = ownerPrefix + ":" + UUID.randomUUID();
|
||||||
Boolean locked = stringRedisTemplate.opsForValue().setIfAbsent(key, token, actualTtl);
|
Boolean locked;
|
||||||
|
try {
|
||||||
|
locked = stringRedisTemplate.opsForValue().setIfAbsent(key, token, actualTtl);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[job-lock] acquire skipped jobName={} key={} msg={}", jobName, key, ex.getMessage());
|
||||||
|
return null;
|
||||||
|
}
|
||||||
if (!Boolean.TRUE.equals(locked)) {
|
if (!Boolean.TRUE.equals(locked)) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,18 @@
|
|||||||
|
package com.nanri.aiimage.config;
|
||||||
|
|
||||||
|
import lombok.Data;
|
||||||
|
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@ConfigurationProperties(prefix = "aiimage.appearance-patent")
|
||||||
|
public class AppearancePatentProperties {
|
||||||
|
private String cozeBaseUrl = "https://api.coze.cn";
|
||||||
|
private String cozeWorkflowPath = "/v1/workflow/run";
|
||||||
|
private String cozeWorkflowId = "7632683471312355338";
|
||||||
|
private String cozeToken = "";
|
||||||
|
private int cozeBatchSize = 10;
|
||||||
|
private int cozeConnectTimeoutMillis = 10000;
|
||||||
|
private int cozeReadTimeoutMillis = 60000;
|
||||||
|
private int staleTimeoutMinutes = 20;
|
||||||
|
private String staleFinalizeCron = "0 */2 * * * *";
|
||||||
|
}
|
||||||
@@ -7,6 +7,7 @@ import org.springframework.boot.context.properties.ConfigurationProperties;
|
|||||||
@ConfigurationProperties(prefix = "aiimage.delete-brand-progress")
|
@ConfigurationProperties(prefix = "aiimage.delete-brand-progress")
|
||||||
public class DeleteBrandProgressProperties {
|
public class DeleteBrandProgressProperties {
|
||||||
private long heartbeatTimeoutMinutes = 15;
|
private long heartbeatTimeoutMinutes = 15;
|
||||||
|
private long deleteBrandInitialTimeoutMinutes = 20;
|
||||||
private String staleCheckCron = "*/30 * * * * *";
|
private String staleCheckCron = "*/30 * * * * *";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -4,6 +4,6 @@ import org.springframework.boot.context.properties.EnableConfigurationProperties
|
|||||||
import org.springframework.context.annotation.Configuration;
|
import org.springframework.context.annotation.Configuration;
|
||||||
|
|
||||||
@Configuration
|
@Configuration
|
||||||
@EnableConfigurationProperties({OssProperties.class, StorageProperties.class, BrandProgressProperties.class, DeleteBrandProgressProperties.class, ZiniaoProperties.class, ModuleCleanupProperties.class, TaskPressureProperties.class})
|
@EnableConfigurationProperties({OssProperties.class, TransientStorageProperties.class, StorageProperties.class, BrandProgressProperties.class, DeleteBrandProgressProperties.class, ZiniaoProperties.class, ModuleCleanupProperties.class, TaskPressureProperties.class, AppearancePatentProperties.class})
|
||||||
public class PropertiesConfig {
|
public class PropertiesConfig {
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,27 @@
|
|||||||
|
package com.nanri.aiimage.config;
|
||||||
|
|
||||||
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
|
import org.springframework.context.annotation.Bean;
|
||||||
|
import org.springframework.context.annotation.Configuration;
|
||||||
|
import org.springframework.core.task.TaskExecutor;
|
||||||
|
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
|
||||||
|
|
||||||
|
@Configuration
|
||||||
|
public class TaskFileJobConfig {
|
||||||
|
|
||||||
|
@Bean("taskFileJobDispatchExecutor")
|
||||||
|
public TaskExecutor taskFileJobDispatchExecutor(
|
||||||
|
@Value("${aiimage.result-file-job.local-dispatch-pool-size:2}") int poolSize,
|
||||||
|
@Value("${aiimage.result-file-job.local-dispatch-queue-capacity:200}") int queueCapacity) {
|
||||||
|
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
|
||||||
|
int normalizedPoolSize = Math.max(1, poolSize);
|
||||||
|
executor.setCorePoolSize(normalizedPoolSize);
|
||||||
|
executor.setMaxPoolSize(normalizedPoolSize);
|
||||||
|
executor.setQueueCapacity(Math.max(10, queueCapacity));
|
||||||
|
executor.setThreadNamePrefix("task-file-job-dispatch-");
|
||||||
|
executor.setWaitForTasksToCompleteOnShutdown(true);
|
||||||
|
executor.setAwaitTerminationSeconds(30);
|
||||||
|
executor.initialize();
|
||||||
|
return executor;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
package com.nanri.aiimage.config;
|
||||||
|
|
||||||
|
import lombok.Data;
|
||||||
|
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@ConfigurationProperties(prefix = "aiimage.transient-storage")
|
||||||
|
public class TransientStorageProperties {
|
||||||
|
private boolean enabled = false;
|
||||||
|
private String endpoint;
|
||||||
|
private String bucket;
|
||||||
|
private String accessKeyId;
|
||||||
|
private String accessKeySecret;
|
||||||
|
private String region = "us-east-1";
|
||||||
|
}
|
||||||
@@ -0,0 +1,385 @@
|
|||||||
|
package com.nanri.aiimage.modules.appearancepatent.client;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.databind.JsonNode;
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import com.nanri.aiimage.config.AppearancePatentProperties;
|
||||||
|
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultRowDto;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.http.MediaType;
|
||||||
|
import org.springframework.http.client.SimpleClientHttpRequestFactory;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
import org.springframework.util.StreamUtils;
|
||||||
|
import org.springframework.web.client.RestClient;
|
||||||
|
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Locale;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
@Component
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
@Slf4j
|
||||||
|
public class AppearancePatentCozeClient {
|
||||||
|
|
||||||
|
private final AppearancePatentProperties properties;
|
||||||
|
private final ObjectMapper objectMapper;
|
||||||
|
|
||||||
|
public List<AppearancePatentResultRowDto> inspect(List<AppearancePatentResultRowDto> rows, String prompt) {
|
||||||
|
if (rows == null || rows.isEmpty()) {
|
||||||
|
return List.of();
|
||||||
|
}
|
||||||
|
if (properties.getCozeToken() == null || properties.getCozeToken().isBlank()) {
|
||||||
|
log.warn("[appearance-patent] coze token not configured, keep raw rows size={}", rows.size());
|
||||||
|
return rows.stream().map(this::copy).toList();
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return inspectWithFallback(rows, prompt);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
String failureMessage = failureMessage(ex);
|
||||||
|
log.warn("[appearance-patent] coze batch failed size={} err={}", rows.size(), failureMessage);
|
||||||
|
return rows.stream().map(this::copy).map(row -> markFailed(row, failureMessage)).toList();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<AppearancePatentResultRowDto> inspectWithFallback(List<AppearancePatentResultRowDto> rows, String prompt) {
|
||||||
|
try {
|
||||||
|
if (rows.size() == 1) {
|
||||||
|
return inspectSingleRowWithRetry(rows, prompt);
|
||||||
|
}
|
||||||
|
InspectAttempt attempt = inspectOnce(rows, prompt);
|
||||||
|
if (attempt.resolvedCount() < rows.size()) {
|
||||||
|
throw new PartialCozeResultException(attempt.resolvedCount(), rows.size(), attempt.rawResultCount());
|
||||||
|
}
|
||||||
|
return attempt.mergedRows();
|
||||||
|
} catch (Exception ex) {
|
||||||
|
if (shouldSplitBatch(rows, ex)) {
|
||||||
|
int middle = rows.size() / 2;
|
||||||
|
log.warn("[appearance-patent] coze batch fallback split size={} left={} right={} err={}",
|
||||||
|
rows.size(), middle, rows.size() - middle, failureMessage(ex));
|
||||||
|
List<AppearancePatentResultRowDto> merged = new ArrayList<>(rows.size());
|
||||||
|
merged.addAll(inspectWithFallback(rows.subList(0, middle), prompt));
|
||||||
|
merged.addAll(inspectWithFallback(rows.subList(middle, rows.size()), prompt));
|
||||||
|
return merged;
|
||||||
|
}
|
||||||
|
throw propagate(ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<AppearancePatentResultRowDto> inspectSingleRowWithRetry(List<AppearancePatentResultRowDto> rows, String prompt) throws Exception {
|
||||||
|
AppearancePatentResultRowDto row = rows.getFirst();
|
||||||
|
PartialCozeResultException lastFailure = null;
|
||||||
|
for (int attemptIndex = 1; attemptIndex <= 3; attemptIndex++) {
|
||||||
|
InspectAttempt attempt = inspectOnce(rows, prompt);
|
||||||
|
if (attempt.resolvedCount() == rows.size()) {
|
||||||
|
return attempt.mergedRows();
|
||||||
|
}
|
||||||
|
log.warn("[appearance-patent] coze single unresolved attempt={} rowId={} asin={} country={} title={} url={} raw={}",
|
||||||
|
attemptIndex,
|
||||||
|
row.getId(),
|
||||||
|
row.getAsin(),
|
||||||
|
row.getCountry(),
|
||||||
|
abbreviate(row.getTitle(), 120),
|
||||||
|
abbreviate(row.getUrl(), 120),
|
||||||
|
abbreviate(attempt.raw(), 500));
|
||||||
|
lastFailure = new PartialCozeResultException(attempt.resolvedCount(), rows.size(), attempt.rawResultCount());
|
||||||
|
}
|
||||||
|
throw lastFailure == null ? new PartialCozeResultException(0, rows.size(), 0) : lastFailure;
|
||||||
|
}
|
||||||
|
|
||||||
|
private InspectAttempt inspectOnce(List<AppearancePatentResultRowDto> rows, String prompt) throws Exception {
|
||||||
|
String raw = postWorkflow(rows, prompt);
|
||||||
|
List<CozeResult> results = parseResults(raw);
|
||||||
|
List<AppearancePatentResultRowDto> merged = mergeRows(rows, results);
|
||||||
|
return new InspectAttempt(raw, merged, resolvedCount(merged), results.size());
|
||||||
|
}
|
||||||
|
|
||||||
|
private String postWorkflow(List<AppearancePatentResultRowDto> rows, String prompt) {
|
||||||
|
Map<String, Object> parameters = new LinkedHashMap<>();
|
||||||
|
parameters.put("row_id_list", rows.stream().map(row -> nonBlank(row.getId(), "")).toList());
|
||||||
|
parameters.put("asin_list", rows.stream().map(row -> nonBlank(row.getAsin(), "")).toList());
|
||||||
|
parameters.put("country_list", rows.stream().map(row -> nonBlank(row.getCountry(), "")).toList());
|
||||||
|
parameters.put("row_key_list", rows.stream().map(this::rowKey).toList());
|
||||||
|
parameters.put("title_list", rows.stream().map(row -> nonBlank(row.getTitle(), row.getAsin())).toList());
|
||||||
|
parameters.put("url_list", rows.stream().map(row -> nonBlank(row.getUrl(), "")).toList());
|
||||||
|
parameters.put("prompt", prompt == null ? "" : prompt);
|
||||||
|
|
||||||
|
Map<String, Object> body = new LinkedHashMap<>();
|
||||||
|
body.put("workflow_id", properties.getCozeWorkflowId());
|
||||||
|
body.put("parameters", parameters);
|
||||||
|
|
||||||
|
RestClient.RequestBodySpec request = restClient().post()
|
||||||
|
.uri(joinUrl(properties.getCozeBaseUrl(), properties.getCozeWorkflowPath()))
|
||||||
|
.headers(headers -> {
|
||||||
|
headers.setBearerAuth(stripBearer(properties.getCozeToken()));
|
||||||
|
headers.setContentType(MediaType.APPLICATION_JSON);
|
||||||
|
});
|
||||||
|
request.body(body);
|
||||||
|
return request.exchange((clientRequest, clientResponse) -> {
|
||||||
|
byte[] responseBytes = StreamUtils.copyToByteArray(clientResponse.getBody());
|
||||||
|
return responseBytes.length == 0 ? "" : new String(responseBytes, StandardCharsets.UTF_8);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<CozeResult> parseResults(String raw) throws Exception {
|
||||||
|
JsonNode root = objectMapper.readTree(raw);
|
||||||
|
if (root.path("code").asInt(-1) != 0) {
|
||||||
|
throw new IllegalStateException(root.path("msg").asText("Coze response code is not 0"));
|
||||||
|
}
|
||||||
|
String dataText = root.path("data").asText("");
|
||||||
|
if (dataText.isBlank()) {
|
||||||
|
return List.of();
|
||||||
|
}
|
||||||
|
JsonNode dataRoot = objectMapper.readTree(dataText);
|
||||||
|
JsonNode array = dataRoot.path("data");
|
||||||
|
List<CozeResult> results = new ArrayList<>();
|
||||||
|
if (array.isArray()) {
|
||||||
|
for (JsonNode node : array) {
|
||||||
|
results.add(new CozeResult(
|
||||||
|
text(firstNonNull(node.get("row_id"), firstNonNull(node.get("rowId"), node.get("id")))),
|
||||||
|
text(node.get("asin")),
|
||||||
|
text(firstNonNull(node.get("country"), node.get("site"))),
|
||||||
|
text(node.get("title")),
|
||||||
|
text(node.get("appearance")),
|
||||||
|
text(firstNonNull(node.get("patent"), node.get("patent "))),
|
||||||
|
text(node.get("result"))
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<AppearancePatentResultRowDto> mergeRows(List<AppearancePatentResultRowDto> rows, List<CozeResult> results) {
|
||||||
|
Map<String, CozeResult> resultByCompositeKey = new LinkedHashMap<>();
|
||||||
|
Map<String, CozeResult> resultByRowId = new LinkedHashMap<>();
|
||||||
|
for (CozeResult result : results) {
|
||||||
|
String compositeKey = rowKey(result.rowId(), result.asin(), result.country());
|
||||||
|
if (!compositeKey.isBlank()) {
|
||||||
|
resultByCompositeKey.putIfAbsent(compositeKey, result);
|
||||||
|
}
|
||||||
|
String rowIdKey = normalize(result.rowId());
|
||||||
|
if (!rowIdKey.isBlank()) {
|
||||||
|
resultByRowId.putIfAbsent(rowIdKey, result);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
List<AppearancePatentResultRowDto> merged = new ArrayList<>(rows.size());
|
||||||
|
boolean allowIndexFallback = results.size() == rows.size();
|
||||||
|
for (int i = 0; i < rows.size(); i++) {
|
||||||
|
AppearancePatentResultRowDto row = copy(rows.get(i));
|
||||||
|
CozeResult result = resultByCompositeKey.get(rowKey(row.getId(), row.getAsin(), row.getCountry()));
|
||||||
|
if (result == null) {
|
||||||
|
result = resultByRowId.get(normalize(row.getId()));
|
||||||
|
}
|
||||||
|
if (result == null && allowIndexFallback && i < results.size()) {
|
||||||
|
result = results.get(i);
|
||||||
|
}
|
||||||
|
applyResult(row, result);
|
||||||
|
merged.add(row);
|
||||||
|
}
|
||||||
|
return merged;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void applyResult(AppearancePatentResultRowDto row, CozeResult result) {
|
||||||
|
if (row == null || result == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
row.setTitleRisk(result.title());
|
||||||
|
row.setAppearanceRisk(result.appearance());
|
||||||
|
row.setPatentRisk(result.patent());
|
||||||
|
row.setConclusion(result.result());
|
||||||
|
}
|
||||||
|
|
||||||
|
private RestClient restClient() {
|
||||||
|
SimpleClientHttpRequestFactory requestFactory = new SimpleClientHttpRequestFactory();
|
||||||
|
requestFactory.setConnectTimeout(properties.getCozeConnectTimeoutMillis());
|
||||||
|
requestFactory.setReadTimeout(properties.getCozeReadTimeoutMillis());
|
||||||
|
return RestClient.builder().requestFactory(requestFactory).build();
|
||||||
|
}
|
||||||
|
|
||||||
|
private AppearancePatentResultRowDto copy(AppearancePatentResultRowDto source) {
|
||||||
|
AppearancePatentResultRowDto row = new AppearancePatentResultRowDto();
|
||||||
|
row.setId(source.getId());
|
||||||
|
row.setAsin(source.getAsin());
|
||||||
|
row.setCountry(source.getCountry());
|
||||||
|
row.setUrl(source.getUrl());
|
||||||
|
row.setTitle(source.getTitle());
|
||||||
|
row.setError(source.getError());
|
||||||
|
row.setDone(source.getDone());
|
||||||
|
row.setTitleRisk(source.getTitleRisk());
|
||||||
|
row.setAppearanceRisk(source.getAppearanceRisk());
|
||||||
|
row.setPatentRisk(source.getPatentRisk());
|
||||||
|
row.setConclusion(source.getConclusion());
|
||||||
|
return row;
|
||||||
|
}
|
||||||
|
|
||||||
|
private AppearancePatentResultRowDto markFailed(AppearancePatentResultRowDto row, String failureMessage) {
|
||||||
|
if (row.getError() == null || row.getError().isBlank()) {
|
||||||
|
row.setError(failureMessage);
|
||||||
|
}
|
||||||
|
if (row.getConclusion() == null || row.getConclusion().isBlank()) {
|
||||||
|
row.setConclusion(failureMessage);
|
||||||
|
}
|
||||||
|
return row;
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean shouldSplitBatch(List<AppearancePatentResultRowDto> rows, Exception ex) {
|
||||||
|
return rows != null && rows.size() > 1 && isRetryableBatchFailure(ex);
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean isRetryableBatchFailure(Exception ex) {
|
||||||
|
if (ex instanceof PartialCozeResultException) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
String message = ex == null ? "" : nonBlank(ex.getMessage(), "");
|
||||||
|
return message.contains("Workflow node execution limit exceeded")
|
||||||
|
|| message.contains("Read timed out")
|
||||||
|
|| message.contains("Connection reset")
|
||||||
|
|| message.contains("I/O error on POST request")
|
||||||
|
|| message.toLowerCase(Locale.ROOT).contains("timeout");
|
||||||
|
}
|
||||||
|
|
||||||
|
private int resolvedCount(List<AppearancePatentResultRowDto> rows) {
|
||||||
|
int resolved = 0;
|
||||||
|
for (AppearancePatentResultRowDto row : rows) {
|
||||||
|
if (hasResolvedCozeFields(row)) {
|
||||||
|
resolved++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return resolved;
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean hasResolvedCozeFields(AppearancePatentResultRowDto row) {
|
||||||
|
if (row == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return !normalize(row.getTitleRisk()).isBlank()
|
||||||
|
|| !normalize(row.getAppearanceRisk()).isBlank()
|
||||||
|
|| !normalize(row.getPatentRisk()).isBlank()
|
||||||
|
|| !normalize(row.getConclusion()).isBlank();
|
||||||
|
}
|
||||||
|
|
||||||
|
private String abbreviate(String value, int maxLength) {
|
||||||
|
String normalized = value == null ? "" : value.trim();
|
||||||
|
if (normalized.length() <= maxLength) {
|
||||||
|
return normalized;
|
||||||
|
}
|
||||||
|
return normalized.substring(0, Math.max(0, maxLength - 3)) + "...";
|
||||||
|
}
|
||||||
|
|
||||||
|
private RuntimeException propagate(Exception ex) {
|
||||||
|
if (ex instanceof RuntimeException runtimeException) {
|
||||||
|
return runtimeException;
|
||||||
|
}
|
||||||
|
return new IllegalStateException(nonBlank(ex.getMessage(), "Coze call failed"), ex);
|
||||||
|
}
|
||||||
|
|
||||||
|
private JsonNode firstNonNull(JsonNode left, JsonNode right) {
|
||||||
|
return left == null || left.isNull() ? right : left;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String text(JsonNode node) {
|
||||||
|
return node == null || node.isNull() ? null : node.asText();
|
||||||
|
}
|
||||||
|
|
||||||
|
private String nonBlank(String value, String fallback) {
|
||||||
|
return value == null || value.isBlank() ? fallback : value;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String normalize(String value) {
|
||||||
|
return value == null ? "" : value.replace("\ufeff", "").replace("\u3000", " ").trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
private String rowKey(AppearancePatentResultRowDto row) {
|
||||||
|
if (row == null) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
return rowKey(row.getId(), row.getAsin(), row.getCountry());
|
||||||
|
}
|
||||||
|
|
||||||
|
private String rowKey(String rowId, String asin, String country) {
|
||||||
|
return normalize(rowId) + "::" + normalize(asin).toUpperCase(Locale.ROOT) + "::" + normalize(country);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String failureMessage(Exception ex) {
|
||||||
|
if (ex instanceof PartialCozeResultException partial) {
|
||||||
|
return "Coze返回结果不完整(" + partial.resolvedCount() + "/" + partial.expectedCount() + ")";
|
||||||
|
}
|
||||||
|
String message = ex == null ? null : ex.getMessage();
|
||||||
|
if (message == null || message.isBlank()) {
|
||||||
|
return "Coze调用失败";
|
||||||
|
}
|
||||||
|
if (message.contains("Workflow node execution limit exceeded")) {
|
||||||
|
return "Coze工作流节点执行超限,请检查工作流配置";
|
||||||
|
}
|
||||||
|
if (message.contains("Read timed out")) {
|
||||||
|
return "Coze调用超时,请稍后重试";
|
||||||
|
}
|
||||||
|
return "Coze调用失败: " + message;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String stripBearer(String token) {
|
||||||
|
String normalized = token == null ? "" : token.trim();
|
||||||
|
return normalized.regionMatches(true, 0, "Bearer ", 0, 7) ? normalized.substring(7).trim() : normalized;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String joinUrl(String baseUrl, String path) {
|
||||||
|
String base = baseUrl == null ? "" : baseUrl.trim();
|
||||||
|
String suffix = path == null ? "" : path.trim();
|
||||||
|
if (base.endsWith("/") && suffix.startsWith("/")) {
|
||||||
|
return base + suffix.substring(1);
|
||||||
|
}
|
||||||
|
if (!base.endsWith("/") && !suffix.startsWith("/")) {
|
||||||
|
return base + "/" + suffix;
|
||||||
|
}
|
||||||
|
return base + suffix;
|
||||||
|
}
|
||||||
|
|
||||||
|
private record CozeResult(
|
||||||
|
String rowId,
|
||||||
|
String asin,
|
||||||
|
String country,
|
||||||
|
String title,
|
||||||
|
String appearance,
|
||||||
|
String patent,
|
||||||
|
String result
|
||||||
|
) {
|
||||||
|
}
|
||||||
|
|
||||||
|
private record InspectAttempt(
|
||||||
|
String raw,
|
||||||
|
List<AppearancePatentResultRowDto> mergedRows,
|
||||||
|
int resolvedCount,
|
||||||
|
int rawResultCount
|
||||||
|
) {
|
||||||
|
}
|
||||||
|
|
||||||
|
private static final class PartialCozeResultException extends RuntimeException {
|
||||||
|
|
||||||
|
private final int resolvedCount;
|
||||||
|
private final int expectedCount;
|
||||||
|
private final int rawResultCount;
|
||||||
|
|
||||||
|
private PartialCozeResultException(int resolvedCount, int expectedCount, int rawResultCount) {
|
||||||
|
super("partial-result resolved=" + resolvedCount + "/" + expectedCount + " raw=" + rawResultCount);
|
||||||
|
this.resolvedCount = resolvedCount;
|
||||||
|
this.expectedCount = expectedCount;
|
||||||
|
this.rawResultCount = rawResultCount;
|
||||||
|
}
|
||||||
|
|
||||||
|
private int resolvedCount() {
|
||||||
|
return resolvedCount;
|
||||||
|
}
|
||||||
|
|
||||||
|
private int expectedCount() {
|
||||||
|
return expectedCount;
|
||||||
|
}
|
||||||
|
|
||||||
|
@SuppressWarnings("unused")
|
||||||
|
private int rawResultCount() {
|
||||||
|
return rawResultCount;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,143 @@
|
|||||||
|
package com.nanri.aiimage.modules.appearancepatent.controller;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.common.api.ApiResponse;
|
||||||
|
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentParseRequest;
|
||||||
|
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentSubmitResultRequest;
|
||||||
|
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentTaskBatchRequest;
|
||||||
|
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentDashboardVo;
|
||||||
|
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentHistoryVo;
|
||||||
|
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParseVo;
|
||||||
|
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentTaskBatchVo;
|
||||||
|
import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskService;
|
||||||
|
import io.swagger.v3.oas.annotations.Operation;
|
||||||
|
import io.swagger.v3.oas.annotations.Parameter;
|
||||||
|
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||||
|
import jakarta.validation.Valid;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import org.springframework.http.HttpHeaders;
|
||||||
|
import org.springframework.http.HttpStatus;
|
||||||
|
import org.springframework.web.bind.annotation.DeleteMapping;
|
||||||
|
import org.springframework.web.bind.annotation.GetMapping;
|
||||||
|
import org.springframework.web.bind.annotation.PathVariable;
|
||||||
|
import org.springframework.web.bind.annotation.PostMapping;
|
||||||
|
import org.springframework.web.bind.annotation.RequestBody;
|
||||||
|
import org.springframework.web.bind.annotation.RequestMapping;
|
||||||
|
import org.springframework.web.bind.annotation.RequestParam;
|
||||||
|
import org.springframework.web.bind.annotation.RestController;
|
||||||
|
import org.springframework.web.server.ResponseStatusException;
|
||||||
|
|
||||||
|
import java.io.InputStream;
|
||||||
|
import java.net.URI;
|
||||||
|
import java.net.URLEncoder;
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
|
||||||
|
@RestController
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
@RequestMapping("/api/appearance-patent")
|
||||||
|
@Tag(name = "外观专利检测", description = "外观专利检测任务接口。前端上传 Excel 后由 Java 解析并创建任务;Python 回传商品数据;Java 负责攒批调用 Coze、补齐子行、生成最终 xlsx 并上传 OSS。")
|
||||||
|
public class AppearancePatentController {
|
||||||
|
|
||||||
|
private final AppearancePatentTaskService service;
|
||||||
|
|
||||||
|
@PostMapping("/parse")
|
||||||
|
@Operation(summary = "解析 Excel 并创建任务", description = "解析上传后的 Excel 文件,提取 id、ASIN、国家、URL、标题等字段。返回给前端的数据只包含整数 id 和 n_1 行;n_2、n_3 等子行会保存在 OSS 解析载荷中,用于最终结果补齐。创建后的任务状态为 PENDING,不会自动推送 Python。")
|
||||||
|
public ApiResponse<AppearancePatentParseVo> parse(@Valid @RequestBody AppearancePatentParseRequest request) {
|
||||||
|
return ApiResponse.success(service.parseAndCreateTask(request));
|
||||||
|
}
|
||||||
|
|
||||||
|
@GetMapping("/dashboard")
|
||||||
|
@Operation(summary = "查询外观专利检测总览", description = "查询当前用户的运行中、成功、失败和已结束任务数量。页面进入时请求一次即可,不需要持续轮询。")
|
||||||
|
public ApiResponse<AppearancePatentDashboardVo> dashboard(
|
||||||
|
@Parameter(description = "当前用户 ID,用于隔离不同用户的任务和历史记录。", required = true, example = "1")
|
||||||
|
@RequestParam("user_id") Long userId) {
|
||||||
|
return ApiResponse.success(service.dashboard(userId));
|
||||||
|
}
|
||||||
|
|
||||||
|
@GetMapping("/history")
|
||||||
|
@Operation(summary = "查询外观专利检测历史", description = "查询当前用户最近的外观专利检测历史记录,包含源文件名、任务状态、行数、错误信息和最终 xlsx 下载地址。")
|
||||||
|
public ApiResponse<AppearancePatentHistoryVo> history(
|
||||||
|
@Parameter(description = "当前用户 ID,用于查询该用户自己的历史记录。", required = true, example = "1")
|
||||||
|
@RequestParam("user_id") Long userId) {
|
||||||
|
return ApiResponse.success(service.history(userId));
|
||||||
|
}
|
||||||
|
|
||||||
|
@PostMapping("/tasks/progress/batch")
|
||||||
|
@Operation(summary = "批量查询任务进度", description = "前端只对活跃任务调用该接口,建议 6 秒一次。接口只返回轻量任务状态,不返回明细结果。")
|
||||||
|
public ApiResponse<AppearancePatentTaskBatchVo> progress(@Valid @RequestBody AppearancePatentTaskBatchRequest request) {
|
||||||
|
return ApiResponse.success(service.progressBatch(request.getTaskIds()));
|
||||||
|
}
|
||||||
|
|
||||||
|
@PostMapping("/tasks/{taskId}/activate")
|
||||||
|
@Operation(summary = "激活任务", description = "前端手动推送 Python 队列成功后调用,将任务从 PENDING 改为 RUNNING,并记录后端内部活跃时间。后续活跃时间由 Python 回传结果接口自动刷新,不需要单独心跳接口。")
|
||||||
|
public ApiResponse<Void> activate(
|
||||||
|
@Parameter(description = "外观专利检测任务 ID,即解析接口返回的 taskId。", required = true, example = "3938")
|
||||||
|
@PathVariable Long taskId,
|
||||||
|
@Parameter(description = "当前用户 ID,必须与创建任务的用户一致。", required = true, example = "1")
|
||||||
|
@RequestParam("user_id") Long userId) {
|
||||||
|
service.activateTask(taskId, userId);
|
||||||
|
return ApiResponse.success(null);
|
||||||
|
}
|
||||||
|
|
||||||
|
@PostMapping("/tasks/{taskId}/result")
|
||||||
|
@Operation(summary = "提交 Python 回传结果", description = "Python 回传商品数据接口。items 可以是一条或多条;Java 先原样保存回传数据,再内部攒够 10 条调用 Coze。done=true 表示 Python 已完成全部回传,Java 会强制处理剩余不足 10 条的数据并生成最终 xlsx。")
|
||||||
|
public ApiResponse<Void> result(
|
||||||
|
@Parameter(description = "外观专利检测任务 ID。任务必须处于 RUNNING 状态。", required = true, example = "3938")
|
||||||
|
@PathVariable Long taskId,
|
||||||
|
@Valid @RequestBody AppearancePatentSubmitResultRequest request) {
|
||||||
|
service.submitResult(taskId, request);
|
||||||
|
return ApiResponse.success(null);
|
||||||
|
}
|
||||||
|
|
||||||
|
@DeleteMapping("/tasks/{taskId}")
|
||||||
|
@Operation(summary = "删除任务", description = "删除当前用户的一条外观专利检测任务,同时清理任务结果、scope 状态和分片记录。")
|
||||||
|
public ApiResponse<Void> deleteTask(
|
||||||
|
@Parameter(description = "外观专利检测任务 ID。", required = true, example = "3938")
|
||||||
|
@PathVariable Long taskId,
|
||||||
|
@Parameter(description = "当前用户 ID,必须与创建任务的用户一致。", required = true, example = "1")
|
||||||
|
@RequestParam("user_id") Long userId) {
|
||||||
|
service.deleteTask(taskId, userId);
|
||||||
|
return ApiResponse.success(null);
|
||||||
|
}
|
||||||
|
|
||||||
|
@DeleteMapping("/history/{resultId}")
|
||||||
|
@Operation(summary = "删除历史记录", description = "删除当前用户的一条外观专利检测历史记录。只删除 biz_file_result 记录,不主动删除任务主记录。")
|
||||||
|
public ApiResponse<Void> deleteHistory(
|
||||||
|
@Parameter(description = "历史结果记录 ID,即 history 接口返回的 resultId。", required = true, example = "1001")
|
||||||
|
@PathVariable Long resultId,
|
||||||
|
@Parameter(description = "当前用户 ID。", required = true, example = "1")
|
||||||
|
@RequestParam("user_id") Long userId) {
|
||||||
|
service.deleteHistory(resultId, userId);
|
||||||
|
return ApiResponse.success(null);
|
||||||
|
}
|
||||||
|
|
||||||
|
@GetMapping("/results/{resultId}/download")
|
||||||
|
@Operation(summary = "下载外观专利检测结果文件")
|
||||||
|
public void downloadResult(
|
||||||
|
@Parameter(description = "结果记录 ID", required = true, example = "1001")
|
||||||
|
@PathVariable Long resultId,
|
||||||
|
@Parameter(description = "当前用户 ID", required = true, example = "1")
|
||||||
|
@RequestParam("user_id") Long userId,
|
||||||
|
jakarta.servlet.http.HttpServletResponse response) {
|
||||||
|
String url = service.resolveResultDownloadUrl(resultId, userId);
|
||||||
|
String filename = service.resolveResultDownloadFilename(resultId, userId);
|
||||||
|
if (url == null || url.isBlank()) {
|
||||||
|
throw new ResponseStatusException(HttpStatus.NOT_FOUND, "暂无可下载结果");
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
String encodedFilename = URLEncoder.encode(filename, StandardCharsets.UTF_8).replace("+", "%20");
|
||||||
|
response.setContentType("application/octet-stream");
|
||||||
|
response.setHeader(HttpHeaders.CONTENT_DISPOSITION,
|
||||||
|
"attachment; filename=\"" + filename + "\"; filename*=UTF-8''" + encodedFilename);
|
||||||
|
try (InputStream in = URI.create(url).toURL().openStream()) {
|
||||||
|
byte[] buffer = new byte[65536];
|
||||||
|
int read;
|
||||||
|
while ((read = in.read(buffer)) != -1) {
|
||||||
|
response.getOutputStream().write(buffer, 0, read);
|
||||||
|
}
|
||||||
|
response.getOutputStream().flush();
|
||||||
|
}
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "下载失败");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,28 @@
|
|||||||
|
package com.nanri.aiimage.modules.appearancepatent.model.dto;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.annotation.JsonAlias;
|
||||||
|
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||||
|
import io.swagger.v3.oas.annotations.media.Schema;
|
||||||
|
import jakarta.validation.constraints.NotEmpty;
|
||||||
|
import jakarta.validation.constraints.NotNull;
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@Schema(description = "外观专利检测解析请求")
|
||||||
|
public class AppearancePatentParseRequest {
|
||||||
|
@JsonProperty("user_id")
|
||||||
|
@NotNull
|
||||||
|
@Schema(description = "当前用户 ID。后端会把创建的任务、历史记录和结果文件归属到该用户。", example = "1", requiredMode = Schema.RequiredMode.REQUIRED)
|
||||||
|
private Long userId;
|
||||||
|
|
||||||
|
@NotEmpty
|
||||||
|
@Schema(description = "已上传的 Excel 文件列表。当前外观专利检测只读取第一个文件;文件对象来自统一上传接口返回值。", requiredMode = Schema.RequiredMode.REQUIRED)
|
||||||
|
private List<AppearancePatentSourceFileDto> files;
|
||||||
|
|
||||||
|
@JsonProperty("ai_prompt")
|
||||||
|
@JsonAlias({"aiPrompt", "prompt"})
|
||||||
|
@Schema(description = "AI 提示词。非必填;为空时前端会使用默认提示词。后端会保存该提示词,并在调用 Coze workflow 时作为 prompt 参数传入。", example = "请排查这些亚马逊商品在英国及欧洲地区是否存在知识产权侵权风险。")
|
||||||
|
private String aiPrompt;
|
||||||
|
}
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
package com.nanri.aiimage.modules.appearancepatent.model.dto;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedRowVo;
|
||||||
|
import io.swagger.v3.oas.annotations.media.Schema;
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@Schema(description = "外观专利检测解析载荷。通常保存到 OSS,数据库只保存 oss 指针。")
|
||||||
|
public class AppearancePatentParsedPayloadDto {
|
||||||
|
@Schema(description = "AI 提示词。")
|
||||||
|
private String aiPrompt;
|
||||||
|
@Schema(description = "Excel 原始表头列表。")
|
||||||
|
private List<String> headers = new ArrayList<>();
|
||||||
|
@Schema(description = "返回前端和推给 Python 的代表行,只包含整数 id 和 n_1 行。")
|
||||||
|
private List<AppearancePatentParsedRowVo> items = new ArrayList<>();
|
||||||
|
@Schema(description = "完整有效行,包含 n_2、n_3 等解析后被前端过滤但最终需要补回的子行。")
|
||||||
|
private List<AppearancePatentParsedRowVo> allItems = new ArrayList<>();
|
||||||
|
}
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
package com.nanri.aiimage.modules.appearancepatent.model.dto;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.annotation.JsonAlias;
|
||||||
|
import io.swagger.v3.oas.annotations.media.Schema;
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@Schema(description = "外观专利检测单行商品数据")
|
||||||
|
public class AppearancePatentResultRowDto {
|
||||||
|
@Schema(description = "Excel 中的 id。代表行通常是整数 id 或 n_1,例如 2_1;最终生成 xlsx 时,2_2、2_3 会复用同组 2_1 的 Coze 检测结果。", example = "2_1")
|
||||||
|
private String id;
|
||||||
|
@Schema(description = "亚马逊 ASIN。后端会统一按大写处理和匹配。", example = "B0CJ8SNXXV")
|
||||||
|
private String asin;
|
||||||
|
@Schema(description = "站点或国家。来自 Excel 的国家列,例如英国、德国、法国。", example = "英国")
|
||||||
|
private String country;
|
||||||
|
@Schema(description = "商品主图或待检测图片 URL。Java 调用 Coze 时会放入 url_list。", example = "https://webstatic.aiproxy.vip/output/20260425/103322/demo.jpg")
|
||||||
|
private String url;
|
||||||
|
@Schema(description = "商品标题。Java 调用 Coze 时会放入 title_list;为空时会回退使用 ASIN。", example = "Women Floral Dress Summer Casual")
|
||||||
|
private String title;
|
||||||
|
@Schema(description = "单行错误信息。通常用于记录 Python 单行处理异常;Coze 失败时后端会尽量保留原始行,不强行写入风险结果。", example = "图片地址为空")
|
||||||
|
private String error;
|
||||||
|
@Schema(description = "单行完成标记。当前主要使用请求体顶层 done 控制任务收尾,该字段仅作兼容。", example = "true")
|
||||||
|
private Boolean done;
|
||||||
|
@Schema(description = "Java 调用 Coze 后生成的标题维度检测结果,对应最终 xlsx 的“标题维度(商标)”列。Python 回传请求不要传该字段;即使传入,后端也会以 Java/Coze 处理结果为准。", example = "标题未发现明显商标侵权风险。", accessMode = Schema.AccessMode.READ_ONLY)
|
||||||
|
private String titleRisk;
|
||||||
|
@Schema(description = "Java 调用 Coze 后生成的外观维度检测结果,对应最终 xlsx 的“外观维度(外观设计专利)”列。Python 回传请求不要传该字段。", example = "未发现明显外观设计专利侵权风险。", accessMode = Schema.AccessMode.READ_ONLY)
|
||||||
|
private String appearanceRisk;
|
||||||
|
@JsonAlias({"patent ", "patent"})
|
||||||
|
@Schema(description = "Java 调用 Coze 后生成的专利维度检测结果,对应最终 xlsx 的“专利维度(发明/实用新型专利)”列。兼容 Coze 返回字段 patent 和 patent 后带空格的情况;Python 回传请求不要传该字段。", example = "未发现明显发明或实用新型专利侵权风险。", accessMode = Schema.AccessMode.READ_ONLY)
|
||||||
|
private String patentRisk;
|
||||||
|
@Schema(description = "Java 调用 Coze 后生成的最终结论,对应最终 xlsx 的“结论”列。Python 回传请求不要传该字段。", example = "未发现明显侵权风险", accessMode = Schema.AccessMode.READ_ONLY)
|
||||||
|
private String conclusion;
|
||||||
|
}
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
package com.nanri.aiimage.modules.appearancepatent.model.dto;
|
||||||
|
|
||||||
|
import io.swagger.v3.oas.annotations.media.Schema;
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@Schema(description = "外观专利检测源文件信息")
|
||||||
|
public class AppearancePatentSourceFileDto {
|
||||||
|
@Schema(description = "上传接口返回的临时文件 key。后端根据该 key 查找本地临时 Excel 文件并解析。", example = "uploads/20260426/appearance_patent_17.xlsx", requiredMode = Schema.RequiredMode.REQUIRED)
|
||||||
|
private String fileKey;
|
||||||
|
@Schema(description = "原始文件名。用于历史记录展示和最终结果文件命名。", example = "17.xlsx")
|
||||||
|
private String originalFilename;
|
||||||
|
@Schema(description = "相对目录路径。当前仅记录来源,外观专利检测不依赖该字段处理。", example = "xlsx/17.xlsx")
|
||||||
|
private String relativePath;
|
||||||
|
}
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
package com.nanri.aiimage.modules.appearancepatent.model.dto;
|
||||||
|
|
||||||
|
import io.swagger.v3.oas.annotations.media.Schema;
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@Schema(description = "Python 回传外观专利检测结果请求")
|
||||||
|
public class AppearancePatentSubmitResultRequest {
|
||||||
|
@Schema(description = "本次 Python 回传的提交批次标识。建议同一个任务固定使用同一个值,例如 appearance-patent-{taskId};后端会结合该值和 chunkIndex 做分片幂等。", example = "appearance-patent-3938")
|
||||||
|
private String submissionId;
|
||||||
|
@Schema(description = "当前回传分片序号。建议从 1 开始递增;同一个 submissionId 下相同 chunkIndex 重复提交会被后端识别为重复分片并跳过重复处理。", example = "1")
|
||||||
|
private Integer chunkIndex;
|
||||||
|
@Schema(description = "本任务预计总分片数。如果 Python 是一条一条回传,可设置为总商品数;如果无法预估,可传 0 或 1,最终以 done=true 触发收尾。", example = "458")
|
||||||
|
private Integer chunkTotal;
|
||||||
|
@Schema(description = "是否为最后一次回传。true 表示 Python 已完成该任务全部数据回传;Java 会处理剩余不足 10 条的数据、组装最终 xlsx、上传 OSS 并收尾任务。", example = "false")
|
||||||
|
private Boolean done;
|
||||||
|
@Schema(description = "Python 侧任务级错误信息。非空时 Java 会记录错误并按失败任务收尾;普通单行 Coze 失败不建议写这里。", example = "浏览器执行异常,任务提前结束")
|
||||||
|
private String error;
|
||||||
|
@Schema(description = "本次回传的商品原始数据列表。可以一条一条传,也可以一次多条传;Python 只需要传 id、asin、country、url、title、error、done 等原始/执行字段。titleRisk、appearanceRisk、patentRisk、conclusion 由 Java 攒够 10 条调用 Coze 后生成,Python 不要传。")
|
||||||
|
private List<AppearancePatentResultRowDto> items = new ArrayList<>();
|
||||||
|
}
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
package com.nanri.aiimage.modules.appearancepatent.model.dto;
|
||||||
|
|
||||||
|
import io.swagger.v3.oas.annotations.media.Schema;
|
||||||
|
import jakarta.validation.constraints.NotEmpty;
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@Schema(description = "外观专利检测批量进度查询请求")
|
||||||
|
public class AppearancePatentTaskBatchRequest {
|
||||||
|
@NotEmpty
|
||||||
|
@Schema(description = "需要查询进度的任务 ID 列表。前端只传正在轮询的活跃任务;后端会批量查询,避免每个任务单独请求。", example = "[3938,3939]", requiredMode = Schema.RequiredMode.REQUIRED)
|
||||||
|
private List<Long> taskIds;
|
||||||
|
}
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
package com.nanri.aiimage.modules.appearancepatent.model.vo;
|
||||||
|
|
||||||
|
import io.swagger.v3.oas.annotations.media.Schema;
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@Schema(description = "外观专利检测总览统计")
|
||||||
|
public class AppearancePatentDashboardVo {
|
||||||
|
@Schema(description = "运行中任务数量。这里统计 RUNNING 状态任务。", example = "1")
|
||||||
|
private Long pendingTaskCount;
|
||||||
|
@Schema(description = "已结束任务数量,等于成功任务数加失败任务数。", example = "12")
|
||||||
|
private Long processedTaskCount;
|
||||||
|
@Schema(description = "成功任务数量。", example = "10")
|
||||||
|
private Long successTaskCount;
|
||||||
|
@Schema(description = "失败任务数量。", example = "2")
|
||||||
|
private Long failedTaskCount;
|
||||||
|
}
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
package com.nanri.aiimage.modules.appearancepatent.model.vo;
|
||||||
|
|
||||||
|
import io.swagger.v3.oas.annotations.media.Schema;
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@Schema(description = "外观专利检测历史记录项")
|
||||||
|
public class AppearancePatentHistoryItemVo {
|
||||||
|
@Schema(description = "结果记录 ID。删除历史、下载结果时使用。", example = "1001")
|
||||||
|
private Long resultId;
|
||||||
|
@Schema(description = "任务 ID。", example = "3938")
|
||||||
|
private Long taskId;
|
||||||
|
@Schema(description = "源 Excel 文件名。", example = "17.xlsx")
|
||||||
|
private String sourceFilename;
|
||||||
|
@Schema(description = "最终结果文件名。任务完成并生成 xlsx 后返回。", example = "17-result.xlsx")
|
||||||
|
private String resultFilename;
|
||||||
|
@Schema(description = "最终结果文件下载地址。后端基于 OSS objectKey 生成的新鲜预签名 URL。", example = "https://bucket.oss-cn-hangzhou.aliyuncs.com/result/appearance_patent/xxx/17-result.xlsx?Expires=...")
|
||||||
|
private String downloadUrl;
|
||||||
|
private Long fileJobId;
|
||||||
|
private String fileStatus;
|
||||||
|
private String fileError;
|
||||||
|
private Boolean fileReady;
|
||||||
|
@Schema(description = "任务状态:PENDING=已解析待推送,RUNNING=执行中,SUCCESS=成功,FAILED=失败。", example = "SUCCESS")
|
||||||
|
private String taskStatus;
|
||||||
|
@Schema(description = "结果是否成功。true 表示任务完成并生成结果文件;false 表示失败或未完成。", example = "true")
|
||||||
|
private Boolean success;
|
||||||
|
@Schema(description = "错误信息。任务失败时返回,例如 Python 超时、结果文件生成失败等。", example = "Python interrupted before uploading final appearance patent result")
|
||||||
|
private String error;
|
||||||
|
@Schema(description = "最终结果行数。包含解析阶段被过滤但最终需要补回的 2_2、2_3 等子行。", example = "716")
|
||||||
|
private Integer rowCount;
|
||||||
|
@Schema(description = "历史记录创建时间,ISO 本地时间字符串。", example = "2026-04-26T10:30:00")
|
||||||
|
private String createdAt;
|
||||||
|
}
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
package com.nanri.aiimage.modules.appearancepatent.model.vo;
|
||||||
|
|
||||||
|
import io.swagger.v3.oas.annotations.media.Schema;
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@Schema(description = "外观专利检测历史记录列表")
|
||||||
|
public class AppearancePatentHistoryVo {
|
||||||
|
@Schema(description = "历史记录项列表,默认返回最近 100 条。")
|
||||||
|
private List<AppearancePatentHistoryItemVo> items = new ArrayList<>();
|
||||||
|
}
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
package com.nanri.aiimage.modules.appearancepatent.model.vo;
|
||||||
|
|
||||||
|
import io.swagger.v3.oas.annotations.media.Schema;
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@Schema(description = "外观专利检测解析结果")
|
||||||
|
public class AppearancePatentParseVo {
|
||||||
|
@Schema(description = "新创建的任务 ID。后续手动推 Python、激活任务、回传结果、查询进度都使用该 ID。", example = "3938")
|
||||||
|
private Long taskId;
|
||||||
|
@Schema(description = "源 Excel 文件名。", example = "17.xlsx")
|
||||||
|
private String sourceFilename;
|
||||||
|
@Schema(description = "Excel 中检测到的有效数据总行数,不含空行。", example = "716")
|
||||||
|
private Integer totalRows;
|
||||||
|
@Schema(description = "返回前端并准备推给 Python 的代表行数量。只包含整数 id 和 n_1 行。", example = "458")
|
||||||
|
private Integer acceptedRows;
|
||||||
|
@Schema(description = "解析时被过滤或缺少必要字段的行数。n_2、n_3 等子行会计入过滤数,但仍会保存在 OSS 解析载荷中用于最终补齐。", example = "258")
|
||||||
|
private Integer droppedRows;
|
||||||
|
@Schema(description = "本任务最终使用的 AI 提示词。", example = "请排查这些亚马逊商品在英国及欧洲地区是否存在知识产权侵权风险。")
|
||||||
|
private String aiPrompt;
|
||||||
|
@Schema(description = "返回前端的代表行列表。前端只展示样例,不展示完整明细;完整 allItems 已保存在 OSS 解析载荷中。")
|
||||||
|
private List<AppearancePatentParsedRowVo> items = new ArrayList<>();
|
||||||
|
}
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
package com.nanri.aiimage.modules.appearancepatent.model.vo;
|
||||||
|
|
||||||
|
import io.swagger.v3.oas.annotations.media.Schema;
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@Schema(description = "外观专利检测解析出的单行数据")
|
||||||
|
public class AppearancePatentParsedRowVo {
|
||||||
|
@Schema(description = "Excel 原始行号,从 1 开始。", example = "2")
|
||||||
|
private Integer rowIndex;
|
||||||
|
@Schema(description = "Excel 中原始 id 值。", example = "2_1")
|
||||||
|
private String sourceId;
|
||||||
|
@Schema(description = "前端展示和 Python 回传使用的 id。整数 id 原样保留,子数据第一条如 2_1 原样保留。", example = "2_1")
|
||||||
|
private String displayId;
|
||||||
|
@Schema(description = "亚马逊 ASIN。", example = "B0CJ8SNXXV")
|
||||||
|
private String groupKey;
|
||||||
|
private String asin;
|
||||||
|
@Schema(description = "国家或站点。", example = "英国")
|
||||||
|
private String country;
|
||||||
|
@Schema(description = "商品图片 URL 或商品 URL,供 Coze 检测使用。", example = "https://webstatic.aiproxy.vip/output/demo.jpg")
|
||||||
|
private String url;
|
||||||
|
@Schema(description = "商品标题。", example = "Women Floral Dress Summer Casual")
|
||||||
|
private String title;
|
||||||
|
@Schema(description = "该 Excel 行的原始列值映射。最终生成 xlsx 时可从这里读取价格等字段。", example = "{\"id\":\"2_1\",\"asin\":\"B0CJ8SNXXV\",\"国家\":\"英国\",\"价格\":\"12.99\"}")
|
||||||
|
private Map<String, String> values = new LinkedHashMap<>();
|
||||||
|
}
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
package com.nanri.aiimage.modules.appearancepatent.model.vo;
|
||||||
|
|
||||||
|
import io.swagger.v3.oas.annotations.media.Schema;
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@Schema(description = "外观专利检测批量进度响应")
|
||||||
|
public class AppearancePatentTaskBatchVo {
|
||||||
|
@Schema(description = "查询到的任务详情列表。顺序按请求 taskIds 处理。")
|
||||||
|
private List<AppearancePatentTaskDetailVo> items = new ArrayList<>();
|
||||||
|
@Schema(description = "未找到或不属于外观专利检测模块的任务 ID 列表。", example = "[99999]")
|
||||||
|
private List<Long> missingTaskIds = new ArrayList<>();
|
||||||
|
}
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
package com.nanri.aiimage.modules.appearancepatent.model.vo;
|
||||||
|
|
||||||
|
import io.swagger.v3.oas.annotations.media.Schema;
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@Schema(description = "外观专利检测任务进度详情")
|
||||||
|
public class AppearancePatentTaskDetailVo {
|
||||||
|
@Schema(description = "任务主记录轻量信息。")
|
||||||
|
private AppearancePatentTaskItemVo task;
|
||||||
|
@Schema(description = "预留的任务明细列表。当前进度接口主要返回任务轻量状态,不返回完整结果明细。")
|
||||||
|
private List<AppearancePatentHistoryItemVo> items = new ArrayList<>();
|
||||||
|
}
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
package com.nanri.aiimage.modules.appearancepatent.model.vo;
|
||||||
|
|
||||||
|
import io.swagger.v3.oas.annotations.media.Schema;
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@Schema(description = "外观专利检测任务轻量信息")
|
||||||
|
public class AppearancePatentTaskItemVo {
|
||||||
|
@Schema(description = "任务 ID。", example = "3938")
|
||||||
|
private Long id;
|
||||||
|
@Schema(description = "任务编号,后端自动生成。", example = "APPEARANCE_PATENT-1780000000000000000")
|
||||||
|
private String taskNo;
|
||||||
|
@Schema(description = "任务状态:PENDING=已解析待推送,RUNNING=执行中,SUCCESS=成功,FAILED=失败。", example = "RUNNING")
|
||||||
|
private String status;
|
||||||
|
@Schema(description = "任务级错误信息。失败时返回。", example = "生成外观专利检测结果失败")
|
||||||
|
private String errorMessage;
|
||||||
|
@Schema(description = "创建时间,ISO 本地时间字符串。", example = "2026-04-26T10:00:00")
|
||||||
|
private String createdAt;
|
||||||
|
@Schema(description = "最后更新时间,通常由 Python 回传结果或任务收尾更新。", example = "2026-04-26T10:05:00")
|
||||||
|
private String updatedAt;
|
||||||
|
@Schema(description = "完成时间。任务未结束时为空。", example = "2026-04-26T10:10:00")
|
||||||
|
private String finishedAt;
|
||||||
|
}
|
||||||
@@ -0,0 +1,144 @@
|
|||||||
|
package com.nanri.aiimage.modules.appearancepatent.service;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultRowDto;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
|
import java.time.Duration;
|
||||||
|
import java.time.Instant;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
@Service
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
@Slf4j
|
||||||
|
public class AppearancePatentTaskCacheService {
|
||||||
|
|
||||||
|
private static final long TTL_HOURS = 24;
|
||||||
|
|
||||||
|
private final StringRedisTemplate stringRedisTemplate;
|
||||||
|
private final ObjectMapper objectMapper;
|
||||||
|
|
||||||
|
public void appendPendingRow(Long taskId, String scopeHash, Integer chunkIndex, AppearancePatentResultRowDto row) {
|
||||||
|
if (taskId == null || taskId <= 0 || scopeHash == null || scopeHash.isBlank() || chunkIndex == null || row == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
stringRedisTemplate.opsForList().rightPush(
|
||||||
|
pendingRowsKey(taskId),
|
||||||
|
objectMapper.writeValueAsString(new PendingRow(scopeHash, chunkIndex, row))
|
||||||
|
);
|
||||||
|
stringRedisTemplate.expire(pendingRowsKey(taskId), Duration.ofHours(TTL_HOURS));
|
||||||
|
touchTaskHeartbeat(taskId);
|
||||||
|
} catch (Exception ignored) {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public long pendingRowCount(Long taskId) {
|
||||||
|
Long size;
|
||||||
|
try {
|
||||||
|
size = stringRedisTemplate.opsForList().size(pendingRowsKey(taskId));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[appearance-patent-cache] pending row count degraded taskId={} msg={}", taskId, ex.getMessage());
|
||||||
|
return 0L;
|
||||||
|
}
|
||||||
|
return size == null ? 0L : size;
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<PendingRow> drainPendingRows(Long taskId, int limit) {
|
||||||
|
if (taskId == null || taskId <= 0 || limit <= 0) {
|
||||||
|
return List.of();
|
||||||
|
}
|
||||||
|
String key = pendingRowsKey(taskId);
|
||||||
|
List<String> values;
|
||||||
|
try {
|
||||||
|
values = stringRedisTemplate.opsForList().range(key, 0, limit - 1L);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[appearance-patent-cache] drain range degraded taskId={} msg={}", taskId, ex.getMessage());
|
||||||
|
return List.of();
|
||||||
|
}
|
||||||
|
if (values == null || values.isEmpty()) {
|
||||||
|
return List.of();
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
stringRedisTemplate.opsForList().trim(key, values.size(), -1);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[appearance-patent-cache] drain trim degraded taskId={} msg={}", taskId, ex.getMessage());
|
||||||
|
return List.of();
|
||||||
|
}
|
||||||
|
List<PendingRow> rows = new ArrayList<>();
|
||||||
|
for (String value : values) {
|
||||||
|
if (value == null || value.isBlank()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
rows.add(objectMapper.readValue(value, PendingRow.class));
|
||||||
|
} catch (Exception ignored) {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
touchTaskHeartbeat(taskId);
|
||||||
|
return rows;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void touchTaskHeartbeat(Long taskId) {
|
||||||
|
if (taskId == null || taskId <= 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
stringRedisTemplate.opsForValue().set(
|
||||||
|
heartbeatKey(taskId),
|
||||||
|
String.valueOf(Instant.now().toEpochMilli()),
|
||||||
|
Duration.ofHours(TTL_HOURS)
|
||||||
|
);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[appearance-patent-cache] touch heartbeat degraded taskId={} msg={}", taskId, ex.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public long getTaskHeartbeatMillis(Long taskId) {
|
||||||
|
if (taskId == null || taskId <= 0) {
|
||||||
|
return 0L;
|
||||||
|
}
|
||||||
|
String raw;
|
||||||
|
try {
|
||||||
|
raw = stringRedisTemplate.opsForValue().get(heartbeatKey(taskId));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[appearance-patent-cache] get heartbeat degraded taskId={} msg={}", taskId, ex.getMessage());
|
||||||
|
return 0L;
|
||||||
|
}
|
||||||
|
if (raw == null || raw.isBlank()) {
|
||||||
|
return 0L;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return Long.parseLong(raw);
|
||||||
|
} catch (NumberFormatException ignored) {
|
||||||
|
return 0L;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void deleteTaskCache(Long taskId) {
|
||||||
|
if (taskId == null || taskId <= 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
stringRedisTemplate.delete(pendingRowsKey(taskId));
|
||||||
|
stringRedisTemplate.delete(heartbeatKey(taskId));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[appearance-patent-cache] delete degraded taskId={} msg={}", taskId, ex.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private String pendingRowsKey(Long taskId) {
|
||||||
|
return "appearance-patent:task:pending-rows:" + taskId;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String heartbeatKey(Long taskId) {
|
||||||
|
return "appearance-patent:task:heartbeat:" + taskId;
|
||||||
|
}
|
||||||
|
|
||||||
|
public record PendingRow(String scopeHash, Integer chunkIndex, AppearancePatentResultRowDto row) {
|
||||||
|
}
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.brand.service;
|
|||||||
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
import com.nanri.aiimage.config.BrandProgressProperties;
|
import com.nanri.aiimage.config.BrandProgressProperties;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
@@ -11,6 +12,7 @@ import java.util.LinkedHashMap;
|
|||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
|
@Slf4j
|
||||||
public class BrandTaskProgressCacheService {
|
public class BrandTaskProgressCacheService {
|
||||||
|
|
||||||
public static final String PHASE_CRAWLING = "crawling";
|
public static final String PHASE_CRAWLING = "crawling";
|
||||||
@@ -53,8 +55,12 @@ public class BrandTaskProgressCacheService {
|
|||||||
values.put("finished_files", String.valueOf(Math.max(finishedFiles, 0)));
|
values.put("finished_files", String.valueOf(Math.max(finishedFiles, 0)));
|
||||||
values.put("updated_at", now);
|
values.put("updated_at", now);
|
||||||
values.put("last_heartbeat_at", now);
|
values.put("last_heartbeat_at", now);
|
||||||
stringRedisTemplate.opsForHash().putAll(key, values);
|
try {
|
||||||
stringRedisTemplate.expire(key, ttl());
|
stringRedisTemplate.opsForHash().putAll(key, values);
|
||||||
|
stringRedisTemplate.expire(key, ttl());
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[brand-progress-cache] save progress degraded taskId={} msg={}", taskId, ex.getMessage());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public void updatePhase(Long taskId, String phase, int finishedFiles, int fileTotal) {
|
public void updatePhase(Long taskId, String phase, int finishedFiles, int fileTotal) {
|
||||||
@@ -66,8 +72,12 @@ public class BrandTaskProgressCacheService {
|
|||||||
values.put("file_total", String.valueOf(Math.max(fileTotal, 0)));
|
values.put("file_total", String.valueOf(Math.max(fileTotal, 0)));
|
||||||
values.put("updated_at", now);
|
values.put("updated_at", now);
|
||||||
values.put("last_heartbeat_at", now);
|
values.put("last_heartbeat_at", now);
|
||||||
stringRedisTemplate.opsForHash().putAll(key, values);
|
try {
|
||||||
stringRedisTemplate.expire(key, ttl());
|
stringRedisTemplate.opsForHash().putAll(key, values);
|
||||||
|
stringRedisTemplate.expire(key, ttl());
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[brand-progress-cache] update phase degraded taskId={} msg={}", taskId, ex.getMessage());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public void markFailed(Long taskId, String message) {
|
public void markFailed(Long taskId, String message) {
|
||||||
@@ -77,27 +87,49 @@ public class BrandTaskProgressCacheService {
|
|||||||
values.put("phase", PHASE_FAILED);
|
values.put("phase", PHASE_FAILED);
|
||||||
values.put("updated_at", now);
|
values.put("updated_at", now);
|
||||||
values.put("error_message", blankToEmpty(message));
|
values.put("error_message", blankToEmpty(message));
|
||||||
stringRedisTemplate.opsForHash().putAll(key, values);
|
try {
|
||||||
stringRedisTemplate.expire(key, Duration.ofHours(brandProgressProperties.getFailedTtlHours()));
|
stringRedisTemplate.opsForHash().putAll(key, values);
|
||||||
|
stringRedisTemplate.expire(key, Duration.ofHours(brandProgressProperties.getFailedTtlHours()));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[brand-progress-cache] mark failed degraded taskId={} msg={}", taskId, ex.getMessage());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public Map<Object, Object> getProgress(Long taskId) {
|
public Map<Object, Object> getProgress(Long taskId) {
|
||||||
return stringRedisTemplate.opsForHash().entries(buildKey(taskId));
|
try {
|
||||||
|
return stringRedisTemplate.opsForHash().entries(buildKey(taskId));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[brand-progress-cache] get progress degraded taskId={} msg={}", taskId, ex.getMessage());
|
||||||
|
return Map.of();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public boolean acquireFinalizeLock(Long taskId) {
|
public boolean acquireFinalizeLock(Long taskId) {
|
||||||
Boolean ok = stringRedisTemplate.opsForValue()
|
try {
|
||||||
.setIfAbsent(buildFinalizeLockKey(taskId), String.valueOf(Instant.now().toEpochMilli()), FINALIZE_LOCK_TTL);
|
Boolean ok = stringRedisTemplate.opsForValue()
|
||||||
return Boolean.TRUE.equals(ok);
|
.setIfAbsent(buildFinalizeLockKey(taskId), String.valueOf(Instant.now().toEpochMilli()), FINALIZE_LOCK_TTL);
|
||||||
|
return Boolean.TRUE.equals(ok);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[brand-progress-cache] acquire finalize lock degraded taskId={} msg={}", taskId, ex.getMessage());
|
||||||
|
return false;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public void releaseFinalizeLock(Long taskId) {
|
public void releaseFinalizeLock(Long taskId) {
|
||||||
stringRedisTemplate.delete(buildFinalizeLockKey(taskId));
|
try {
|
||||||
|
stringRedisTemplate.delete(buildFinalizeLockKey(taskId));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[brand-progress-cache] release finalize lock degraded taskId={} msg={}", taskId, ex.getMessage());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public void delete(Long taskId) {
|
public void delete(Long taskId) {
|
||||||
stringRedisTemplate.delete(buildKey(taskId));
|
try {
|
||||||
stringRedisTemplate.delete(buildFinalizeLockKey(taskId));
|
stringRedisTemplate.delete(buildKey(taskId));
|
||||||
|
stringRedisTemplate.delete(buildFinalizeLockKey(taskId));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[brand-progress-cache] delete degraded taskId={} msg={}", taskId, ex.getMessage());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public String buildKey(Long taskId) {
|
public String buildKey(Long taskId) {
|
||||||
|
|||||||
@@ -31,6 +31,9 @@ import com.nanri.aiimage.modules.brand.model.vo.LegacyBrandTaskDetailVo;
|
|||||||
import com.nanri.aiimage.modules.brand.model.vo.LegacyBrandTaskItemVo;
|
import com.nanri.aiimage.modules.brand.model.vo.LegacyBrandTaskItemVo;
|
||||||
import com.nanri.aiimage.modules.brand.model.vo.LegacyBrandTaskListVo;
|
import com.nanri.aiimage.modules.brand.model.vo.LegacyBrandTaskListVo;
|
||||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.apache.poi.ss.usermodel.Cell;
|
import org.apache.poi.ss.usermodel.Cell;
|
||||||
@@ -77,6 +80,7 @@ public class BrandTaskService {
|
|||||||
private static final String STATUS_SUCCESS = "success";
|
private static final String STATUS_SUCCESS = "success";
|
||||||
private static final String STATUS_FAILED = "failed";
|
private static final String STATUS_FAILED = "failed";
|
||||||
private static final String STATUS_CANCELLED = "cancelled";
|
private static final String STATUS_CANCELLED = "cancelled";
|
||||||
|
private static final String MODULE_TYPE = "BRAND";
|
||||||
|
|
||||||
private final BrandCrawlTaskMapper brandCrawlTaskMapper;
|
private final BrandCrawlTaskMapper brandCrawlTaskMapper;
|
||||||
private final OssStorageService ossStorageService;
|
private final OssStorageService ossStorageService;
|
||||||
@@ -86,6 +90,8 @@ public class BrandTaskService {
|
|||||||
private final BrandTaskStorageService brandTaskStorageService;
|
private final BrandTaskStorageService brandTaskStorageService;
|
||||||
private final DistributedJobLockService distributedJobLockService;
|
private final DistributedJobLockService distributedJobLockService;
|
||||||
private final ObjectMapper objectMapper;
|
private final ObjectMapper objectMapper;
|
||||||
|
private final TaskFileJobService taskFileJobService;
|
||||||
|
private final TaskProgressSnapshotService taskProgressSnapshotService;
|
||||||
|
|
||||||
public BrandTaskCreateVo createTask(Long userId, BrandTaskCreateRequest request) {
|
public BrandTaskCreateVo createTask(Long userId, BrandTaskCreateRequest request) {
|
||||||
if (userId == null || userId <= 0) {
|
if (userId == null || userId <= 0) {
|
||||||
@@ -244,6 +250,7 @@ public class BrandTaskService {
|
|||||||
|
|
||||||
int totalCount = sourceFiles.size();
|
int totalCount = sourceFiles.size();
|
||||||
markTaskRunning(taskId, totalCount);
|
markTaskRunning(taskId, totalCount);
|
||||||
|
afterTaskStarted(taskId, totalCount);
|
||||||
|
|
||||||
log.info("[brand-submit] taskId={} chunks={} files={} thread={}",
|
log.info("[brand-submit] taskId={} chunks={} files={} thread={}",
|
||||||
taskId,
|
taskId,
|
||||||
@@ -295,6 +302,7 @@ public class BrandTaskService {
|
|||||||
currentTotalLines,
|
currentTotalLines,
|
||||||
finishedCount);
|
finishedCount);
|
||||||
updateTaskProgress(taskId, finishedCount, totalCount);
|
updateTaskProgress(taskId, finishedCount, totalCount);
|
||||||
|
saveBrandProgressSnapshot(taskId, STATUS_RUNNING, totalCount, finishedCount, 0, null);
|
||||||
}
|
}
|
||||||
|
|
||||||
tryFinalizeTask(taskId, request.getStrategy(), sourceFiles, cachedByUrl, finishedCount, totalCount);
|
tryFinalizeTask(taskId, request.getStrategy(), sourceFiles, cachedByUrl, finishedCount, totalCount);
|
||||||
@@ -357,6 +365,23 @@ public class BrandTaskService {
|
|||||||
return ossStorageService.generateFreshDownloadUrl(stored);
|
return ossStorageService.generateFreshDownloadUrl(stored);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public String resolveResultObjectKey(Long taskId) {
|
||||||
|
BrandCrawlTaskEntity task = requireTask(taskId);
|
||||||
|
Object raw = parseJsonValue(task.getResultPaths());
|
||||||
|
if (!(raw instanceof Map<?, ?> map)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
Object zipObjectKey = map.get("zip_object_key");
|
||||||
|
if (zipObjectKey instanceof String key && !key.isBlank()) {
|
||||||
|
return key;
|
||||||
|
}
|
||||||
|
Object zipUrl = map.get("zip_url");
|
||||||
|
if (zipUrl instanceof String url && !url.isBlank()) {
|
||||||
|
return url;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
private List<BrandCrawlTaskEntity> listTaskEntities(Long userId) {
|
private List<BrandCrawlTaskEntity> listTaskEntities(Long userId) {
|
||||||
if (userId == null || userId <= 0) {
|
if (userId == null || userId <= 0) {
|
||||||
throw new BusinessException("userId 不合法");
|
throw new BusinessException("userId 不合法");
|
||||||
@@ -496,7 +521,7 @@ public class BrandTaskService {
|
|||||||
private void markTaskRunning(Long taskId, int totalCount) {
|
private void markTaskRunning(Long taskId, int totalCount) {
|
||||||
int started = brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper<BrandCrawlTaskEntity>()
|
int started = brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper<BrandCrawlTaskEntity>()
|
||||||
.eq(BrandCrawlTaskEntity::getId, taskId)
|
.eq(BrandCrawlTaskEntity::getId, taskId)
|
||||||
.ne(BrandCrawlTaskEntity::getStatus, STATUS_CANCELLED)
|
.in(BrandCrawlTaskEntity::getStatus, STATUS_PENDING, STATUS_RUNNING)
|
||||||
.set(BrandCrawlTaskEntity::getStatus, STATUS_RUNNING)
|
.set(BrandCrawlTaskEntity::getStatus, STATUS_RUNNING)
|
||||||
.set(BrandCrawlTaskEntity::getProgressTotal, totalCount)
|
.set(BrandCrawlTaskEntity::getProgressTotal, totalCount)
|
||||||
.set(BrandCrawlTaskEntity::getUpdatedAt, LocalDateTime.now())
|
.set(BrandCrawlTaskEntity::getUpdatedAt, LocalDateTime.now())
|
||||||
@@ -505,6 +530,13 @@ public class BrandTaskService {
|
|||||||
throw new BusinessException("任务已取消");
|
throw new BusinessException("任务已取消");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
private void afterTaskStarted(Long taskId, int totalCount) {
|
||||||
|
saveBrandProgressSnapshot(taskId, STATUS_RUNNING, totalCount, 0, 0, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void saveBrandProgressSnapshot(Long taskId, String status, int totalCount, int successCount, int failedCount, String message) {
|
||||||
|
taskProgressSnapshotService.save(taskId, MODULE_TYPE, status, totalCount, successCount, failedCount, null, message, null);
|
||||||
|
}
|
||||||
|
|
||||||
private BrandCrawlTaskEntity requireActiveTask(Long taskId) {
|
private BrandCrawlTaskEntity requireActiveTask(Long taskId) {
|
||||||
BrandCrawlTaskEntity task = requireTask(taskId);
|
BrandCrawlTaskEntity task = requireTask(taskId);
|
||||||
@@ -533,12 +565,33 @@ public class BrandTaskService {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
finalizeTask(taskId, strategy, sourceFiles, cachedByUrl, totalCount);
|
enqueueFinalizeTask(taskId, totalCount);
|
||||||
} finally {
|
} finally {
|
||||||
brandTaskProgressCacheService.releaseFinalizeLock(taskId);
|
brandTaskProgressCacheService.releaseFinalizeLock(taskId);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void enqueueFinalizeTask(Long taskId, int totalCount) {
|
||||||
|
brandTaskProgressCacheService.updatePhase(taskId, BrandTaskProgressCacheService.PHASE_ASSEMBLING, totalCount, totalCount);
|
||||||
|
taskProgressSnapshotService.save(taskId, MODULE_TYPE, STATUS_RUNNING, totalCount, totalCount, 0,
|
||||||
|
null, "result file assembly queued", null);
|
||||||
|
taskFileJobService.enqueueAssembleResult(taskId, MODULE_TYPE, taskId, "task:" + taskId);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void processResultFileJob(TaskFileJobEntity job) {
|
||||||
|
if (job == null || job.getTaskId() == null) {
|
||||||
|
throw new BusinessException("结果文件任务参数不完整");
|
||||||
|
}
|
||||||
|
BrandCrawlTaskEntity task = requireTask(job.getTaskId());
|
||||||
|
List<BrandSourceFileDto> sourceFiles = parseSourceFiles(task.getFilePaths());
|
||||||
|
if (sourceFiles.isEmpty()) {
|
||||||
|
throw new BusinessException("任务没有源文件");
|
||||||
|
}
|
||||||
|
List<BrandParsedFileCacheDto> cachedFiles = brandTaskStorageService.getParsedPayload(task.getId());
|
||||||
|
Map<String, BrandParsedFileCacheDto> cachedByUrl = indexCachedFiles(cachedFiles);
|
||||||
|
finalizeTask(task.getId(), normalizeStrategy(task.getStrategy()), sourceFiles, cachedByUrl, sourceFiles.size());
|
||||||
|
}
|
||||||
|
|
||||||
private void finalizeTask(Long taskId,
|
private void finalizeTask(Long taskId,
|
||||||
String strategy,
|
String strategy,
|
||||||
List<BrandSourceFileDto> sourceFiles,
|
List<BrandSourceFileDto> sourceFiles,
|
||||||
@@ -601,6 +654,7 @@ public class BrandTaskService {
|
|||||||
if (updated == 0) {
|
if (updated == 0) {
|
||||||
throw new BusinessException("任务已取消");
|
throw new BusinessException("任务已取消");
|
||||||
}
|
}
|
||||||
|
saveBrandProgressSnapshot(taskId, STATUS_SUCCESS, totalCount, totalCount, 0, null);
|
||||||
brandTaskStorageService.deleteTaskData(taskId);
|
brandTaskStorageService.deleteTaskData(taskId);
|
||||||
brandTaskProgressCacheService.delete(taskId);
|
brandTaskProgressCacheService.delete(taskId);
|
||||||
log.info("[brand-finalize] taskId={} finalized success files={} elapsedMs={}",
|
log.info("[brand-finalize] taskId={} finalized success files={} elapsedMs={}",
|
||||||
@@ -617,6 +671,7 @@ public class BrandTaskService {
|
|||||||
.set(BrandCrawlTaskEntity::getProgressTotal, totalCount)
|
.set(BrandCrawlTaskEntity::getProgressTotal, totalCount)
|
||||||
.set(BrandCrawlTaskEntity::getErrorMessage, ex.getMessage()));
|
.set(BrandCrawlTaskEntity::getErrorMessage, ex.getMessage()));
|
||||||
brandTaskProgressCacheService.markFailed(taskId, ex.getMessage());
|
brandTaskProgressCacheService.markFailed(taskId, ex.getMessage());
|
||||||
|
saveBrandProgressSnapshot(taskId, STATUS_FAILED, totalCount, brandTaskStorageService.countCompletedFiles(taskId), 1, ex.getMessage());
|
||||||
if (ex instanceof BusinessException businessException) {
|
if (ex instanceof BusinessException businessException) {
|
||||||
throw businessException;
|
throw businessException;
|
||||||
}
|
}
|
||||||
@@ -643,10 +698,23 @@ public class BrandTaskService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private Map<String, BrandParsedFileCacheDto> indexCachedFiles(List<BrandParsedFileCacheDto> cachedFiles) {
|
||||||
|
Map<String, BrandParsedFileCacheDto> cachedByUrl = new LinkedHashMap<>();
|
||||||
|
if (cachedFiles == null) {
|
||||||
|
return cachedByUrl;
|
||||||
|
}
|
||||||
|
for (BrandParsedFileCacheDto cachedFile : cachedFiles) {
|
||||||
|
if (cachedFile != null && cachedFile.getFileUrl() != null) {
|
||||||
|
cachedByUrl.put(cachedFile.getFileUrl(), cachedFile);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return cachedByUrl;
|
||||||
|
}
|
||||||
|
|
||||||
private void updateTaskProgress(Long taskId, int finishedCount, int totalCount) {
|
private void updateTaskProgress(Long taskId, int finishedCount, int totalCount) {
|
||||||
brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper<BrandCrawlTaskEntity>()
|
brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper<BrandCrawlTaskEntity>()
|
||||||
.eq(BrandCrawlTaskEntity::getId, taskId)
|
.eq(BrandCrawlTaskEntity::getId, taskId)
|
||||||
.ne(BrandCrawlTaskEntity::getStatus, STATUS_CANCELLED)
|
.in(BrandCrawlTaskEntity::getStatus, STATUS_PENDING, STATUS_RUNNING)
|
||||||
.set(BrandCrawlTaskEntity::getStatus, STATUS_RUNNING)
|
.set(BrandCrawlTaskEntity::getStatus, STATUS_RUNNING)
|
||||||
.set(BrandCrawlTaskEntity::getProgressCurrent, finishedCount)
|
.set(BrandCrawlTaskEntity::getProgressCurrent, finishedCount)
|
||||||
.set(BrandCrawlTaskEntity::getProgressTotal, totalCount)
|
.set(BrandCrawlTaskEntity::getProgressTotal, totalCount)
|
||||||
@@ -951,13 +1019,14 @@ public class BrandTaskService {
|
|||||||
List<String> fullUrls = new ArrayList<>();
|
List<String> fullUrls = new ArrayList<>();
|
||||||
for (OutputEntry entry : entries) {
|
for (OutputEntry entry : entries) {
|
||||||
// 存储完整公开地址,下载时会自动通过 resolveObjectKey 提取并重新签名(见 OssStorageService.resolveObjectKey)
|
// 存储完整公开地址,下载时会自动通过 resolveObjectKey 提取并重新签名(见 OssStorageService.resolveObjectKey)
|
||||||
String objectKey = ossStorageService.uploadResultFile(entry.resultFile(), "BRAND");
|
String objectKey = ossStorageService.uploadResultFile(entry.resultFile(), MODULE_TYPE);
|
||||||
fullUrls.add(ossStorageService.getPublicUrl(objectKey));
|
fullUrls.add(ossStorageService.getPublicUrl(objectKey));
|
||||||
}
|
}
|
||||||
Map<String, Object> result = new LinkedHashMap<>();
|
Map<String, Object> result = new LinkedHashMap<>();
|
||||||
result.put("urls", fullUrls);
|
result.put("urls", fullUrls);
|
||||||
File zipFile = packageAsZip(taskId, entries);
|
File zipFile = packageAsZip(taskId, entries);
|
||||||
String zipObjectKey = ossStorageService.uploadResultFile(zipFile, "BRAND");
|
String zipObjectKey = ossStorageService.uploadResultFile(zipFile, MODULE_TYPE);
|
||||||
|
result.put("zip_object_key", zipObjectKey);
|
||||||
result.put("zip_url", ossStorageService.getPublicUrl(zipObjectKey));
|
result.put("zip_url", ossStorageService.getPublicUrl(zipObjectKey));
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,11 +7,11 @@ import com.nanri.aiimage.common.exception.BusinessException;
|
|||||||
import com.nanri.aiimage.modules.brand.model.dto.BrandCrawlResultFileDto;
|
import com.nanri.aiimage.modules.brand.model.dto.BrandCrawlResultFileDto;
|
||||||
import com.nanri.aiimage.modules.brand.model.dto.BrandFileAggregateCacheDto;
|
import com.nanri.aiimage.modules.brand.model.dto.BrandFileAggregateCacheDto;
|
||||||
import com.nanri.aiimage.modules.brand.model.dto.BrandParsedFileCacheDto;
|
import com.nanri.aiimage.modules.brand.model.dto.BrandParsedFileCacheDto;
|
||||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
|
||||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||||
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import org.springframework.dao.DuplicateKeyException;
|
import org.springframework.dao.DuplicateKeyException;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
@@ -30,12 +30,11 @@ import java.util.Map;
|
|||||||
public class BrandTaskStorageService {
|
public class BrandTaskStorageService {
|
||||||
|
|
||||||
private static final String MODULE_TYPE = "BRAND";
|
private static final String MODULE_TYPE = "BRAND";
|
||||||
private static final String OSS_POINTER_PREFIX = "oss:";
|
|
||||||
|
|
||||||
private final TaskChunkMapper taskChunkMapper;
|
private final TaskChunkMapper taskChunkMapper;
|
||||||
private final TaskScopeStateMapper taskScopeStateMapper;
|
private final TaskScopeStateMapper taskScopeStateMapper;
|
||||||
private final ObjectMapper objectMapper;
|
private final ObjectMapper objectMapper;
|
||||||
private final OssStorageService ossStorageService;
|
private final TransientPayloadStorageService transientPayloadStorageService;
|
||||||
|
|
||||||
@Transactional
|
@Transactional
|
||||||
public void saveParsedPayload(Long taskId, List<BrandParsedFileCacheDto> payload) {
|
public void saveParsedPayload(Long taskId, List<BrandParsedFileCacheDto> payload) {
|
||||||
@@ -73,7 +72,7 @@ public class BrandTaskStorageService {
|
|||||||
if (state == null) {
|
if (state == null) {
|
||||||
throw new BusinessException("Save brand task parsed payload failed");
|
throw new BusinessException("Save brand task parsed payload failed");
|
||||||
}
|
}
|
||||||
deleteParsedPayloadIfNeeded(state.getParsedPayloadJson(), parsedJson);
|
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(state.getParsedPayloadJson(), parsedJson);
|
||||||
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
|
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
|
||||||
.eq(TaskScopeStateEntity::getId, state.getId())
|
.eq(TaskScopeStateEntity::getId, state.getId())
|
||||||
.set(TaskScopeStateEntity::getScopeKey, scopeKey)
|
.set(TaskScopeStateEntity::getScopeKey, scopeKey)
|
||||||
@@ -132,6 +131,7 @@ public class BrandTaskStorageService {
|
|||||||
throw new BusinessException("Brand task chunk conflict: " + scopeKey + "#" + file.getChunkIndex());
|
throw new BusinessException("Brand task chunk conflict: " + scopeKey + "#" + file.getChunkIndex());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
String storedPayload = transientPayloadStorageService.storeChunkPayload(MODULE_TYPE, taskId, scopeHash, file.getChunkIndex(), payloadJson);
|
||||||
TaskChunkEntity entity = new TaskChunkEntity();
|
TaskChunkEntity entity = new TaskChunkEntity();
|
||||||
entity.setTaskId(taskId);
|
entity.setTaskId(taskId);
|
||||||
entity.setModuleType(MODULE_TYPE);
|
entity.setModuleType(MODULE_TYPE);
|
||||||
@@ -139,7 +139,7 @@ public class BrandTaskStorageService {
|
|||||||
entity.setScopeHash(scopeHash);
|
entity.setScopeHash(scopeHash);
|
||||||
entity.setChunkIndex(file.getChunkIndex());
|
entity.setChunkIndex(file.getChunkIndex());
|
||||||
entity.setChunkTotal(file.getChunkTotal());
|
entity.setChunkTotal(file.getChunkTotal());
|
||||||
entity.setPayloadJson(payloadJson);
|
entity.setPayloadJson(storedPayload);
|
||||||
entity.setPayloadHash(payloadHash);
|
entity.setPayloadHash(payloadHash);
|
||||||
entity.setCreatedAt(now);
|
entity.setCreatedAt(now);
|
||||||
entity.setUpdatedAt(now);
|
entity.setUpdatedAt(now);
|
||||||
@@ -153,9 +153,11 @@ public class BrandTaskStorageService {
|
|||||||
.eq(TaskChunkEntity::getChunkIndex, file.getChunkIndex())
|
.eq(TaskChunkEntity::getChunkIndex, file.getChunkIndex())
|
||||||
.last("limit 1"));
|
.last("limit 1"));
|
||||||
if (concurrentChunk != null && payloadHash.equals(concurrentChunk.getPayloadHash())) {
|
if (concurrentChunk != null && payloadHash.equals(concurrentChunk.getPayloadHash())) {
|
||||||
|
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
|
||||||
BrandFileAggregateCacheDto aggregate = rebuildAggregate(taskId, scopeKey, scopeHash);
|
BrandFileAggregateCacheDto aggregate = rebuildAggregate(taskId, scopeKey, scopeHash);
|
||||||
return new ChunkStoreResult(false, false, countCompletedFiles(taskId), aggregate);
|
return new ChunkStoreResult(false, false, countCompletedFiles(taskId), aggregate);
|
||||||
}
|
}
|
||||||
|
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
|
||||||
throw new BusinessException("Brand task chunk conflict: " + scopeKey + "#" + file.getChunkIndex());
|
throw new BusinessException("Brand task chunk conflict: " + scopeKey + "#" + file.getChunkIndex());
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -171,7 +173,8 @@ public class BrandTaskStorageService {
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
return objectMapper.readValue(state.getStateJson(), BrandFileAggregateCacheDto.class);
|
String aggregateJson = transientPayloadStorageService.resolvePayload(state.getStateJson(), "read brand task aggregate failed");
|
||||||
|
return objectMapper.readValue(aggregateJson, BrandFileAggregateCacheDto.class);
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
throw new BusinessException("Read brand task aggregate failed");
|
throw new BusinessException("Read brand task aggregate failed");
|
||||||
}
|
}
|
||||||
@@ -195,7 +198,8 @@ public class BrandTaskStorageService {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
result.put(state.getScopeKey(), objectMapper.readValue(state.getStateJson(), BrandFileAggregateCacheDto.class));
|
String aggregateJson = transientPayloadStorageService.resolvePayload(state.getStateJson(), "read brand task aggregate failed");
|
||||||
|
result.put(state.getScopeKey(), objectMapper.readValue(aggregateJson, BrandFileAggregateCacheDto.class));
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
throw new BusinessException("Read brand task aggregate failed");
|
throw new BusinessException("Read brand task aggregate failed");
|
||||||
}
|
}
|
||||||
@@ -220,12 +224,22 @@ public class BrandTaskStorageService {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||||
.select(TaskScopeStateEntity::getParsedPayloadJson)
|
.select(TaskScopeStateEntity::getParsedPayloadJson, TaskScopeStateEntity::getStateJson)
|
||||||
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||||
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
|
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
|
||||||
if (states != null) {
|
if (states != null) {
|
||||||
for (TaskScopeStateEntity state : states) {
|
for (TaskScopeStateEntity state : states) {
|
||||||
deleteParsedPayloadIfNeeded(state.getParsedPayloadJson(), null);
|
transientPayloadStorageService.deletePayloadIfPresent(state.getParsedPayloadJson());
|
||||||
|
transientPayloadStorageService.deletePayloadIfPresent(state.getStateJson());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||||
|
.select(TaskChunkEntity::getPayloadJson)
|
||||||
|
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
|
||||||
|
if (chunks != null) {
|
||||||
|
for (TaskChunkEntity chunk : chunks) {
|
||||||
|
transientPayloadStorageService.deletePayloadIfPresent(chunk.getPayloadJson());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
taskChunkMapper.delete(new LambdaQueryWrapper<TaskChunkEntity>()
|
taskChunkMapper.delete(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||||
@@ -246,12 +260,14 @@ public class BrandTaskStorageService {
|
|||||||
if (state == null) {
|
if (state == null) {
|
||||||
throw new BusinessException("Brand task aggregate state missing: " + scopeKey);
|
throw new BusinessException("Brand task aggregate state missing: " + scopeKey);
|
||||||
}
|
}
|
||||||
|
String storedAggregate = transientPayloadStorageService.storeScopePayload(MODULE_TYPE, taskId, scopeHash, aggregateJson, true);
|
||||||
int chunkTotal = aggregate.getChunkTotal() == null ? 0 : aggregate.getChunkTotal();
|
int chunkTotal = aggregate.getChunkTotal() == null ? 0 : aggregate.getChunkTotal();
|
||||||
int receivedCount = aggregate.getReceivedChunkCount() == null ? 0 : aggregate.getReceivedChunkCount();
|
int receivedCount = aggregate.getReceivedChunkCount() == null ? 0 : aggregate.getReceivedChunkCount();
|
||||||
|
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(state.getStateJson(), storedAggregate);
|
||||||
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
|
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
|
||||||
.eq(TaskScopeStateEntity::getId, state.getId())
|
.eq(TaskScopeStateEntity::getId, state.getId())
|
||||||
.set(TaskScopeStateEntity::getScopeKey, scopeKey)
|
.set(TaskScopeStateEntity::getScopeKey, scopeKey)
|
||||||
.set(TaskScopeStateEntity::getStateJson, aggregateJson)
|
.set(TaskScopeStateEntity::getStateJson, storedAggregate)
|
||||||
.set(TaskScopeStateEntity::getChunkTotal, chunkTotal)
|
.set(TaskScopeStateEntity::getChunkTotal, chunkTotal)
|
||||||
.set(TaskScopeStateEntity::getReceivedChunkCount, receivedCount)
|
.set(TaskScopeStateEntity::getReceivedChunkCount, receivedCount)
|
||||||
.set(TaskScopeStateEntity::getCompleted, Boolean.TRUE.equals(aggregate.getCompleted()) ? 1 : 0)
|
.set(TaskScopeStateEntity::getCompleted, Boolean.TRUE.equals(aggregate.getCompleted()) ? 1 : 0)
|
||||||
@@ -362,7 +378,8 @@ public class BrandTaskStorageService {
|
|||||||
|
|
||||||
private BrandCrawlResultFileDto readChunkPayload(String payloadJson) {
|
private BrandCrawlResultFileDto readChunkPayload(String payloadJson) {
|
||||||
try {
|
try {
|
||||||
return objectMapper.readValue(payloadJson, BrandCrawlResultFileDto.class);
|
String resolvedPayload = transientPayloadStorageService.resolvePayload(payloadJson, "read brand task chunk failed");
|
||||||
|
return objectMapper.readValue(resolvedPayload, BrandCrawlResultFileDto.class);
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
throw new BusinessException("Read brand task chunk failed");
|
throw new BusinessException("Read brand task chunk failed");
|
||||||
}
|
}
|
||||||
@@ -382,56 +399,17 @@ public class BrandTaskStorageService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private String storeParsedPayload(Long taskId, String scopeHash, String payloadJson) {
|
private String storeParsedPayload(Long taskId, String scopeHash, String payloadJson) {
|
||||||
try {
|
return transientPayloadStorageService.storeParsedPayload(MODULE_TYPE, taskId, scopeHash, payloadJson, true);
|
||||||
String pointer = OSS_POINTER_PREFIX + ossStorageService.uploadTaskParsedPayload(MODULE_TYPE, taskId, scopeHash, payloadJson);
|
|
||||||
return objectMapper.writeValueAsString(pointer);
|
|
||||||
} catch (Exception ex) {
|
|
||||||
throw new BusinessException("Save brand task parsed payload failed");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private String resolveParsedPayload(String value) {
|
private String resolveParsedPayload(String value) {
|
||||||
if (isBlank(value)) {
|
|
||||||
return value;
|
|
||||||
}
|
|
||||||
String ossPointer = extractOssPointer(value);
|
|
||||||
if (ossPointer == null) {
|
|
||||||
return value;
|
|
||||||
}
|
|
||||||
try {
|
try {
|
||||||
return ossStorageService.readObjectAsString(ossPointer.substring(OSS_POINTER_PREFIX.length()));
|
return transientPayloadStorageService.resolvePayload(value, "Read brand task parsed payload failed");
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
throw new BusinessException("Read brand task parsed payload failed");
|
throw new BusinessException("Read brand task parsed payload failed");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void deleteParsedPayloadIfNeeded(String oldValue, String newValue) {
|
|
||||||
String oldPointer = extractOssPointer(oldValue);
|
|
||||||
String newPointer = extractOssPointer(newValue);
|
|
||||||
if (oldPointer == null || oldPointer.equals(newPointer)) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
ossStorageService.deleteObject(oldPointer.substring(OSS_POINTER_PREFIX.length()));
|
|
||||||
} catch (Exception ignored) {
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private String extractOssPointer(String value) {
|
|
||||||
if (isBlank(value)) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
if (value.startsWith(OSS_POINTER_PREFIX)) {
|
|
||||||
return value;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
String decoded = objectMapper.readValue(value, String.class);
|
|
||||||
return decoded != null && decoded.startsWith(OSS_POINTER_PREFIX) ? decoded : null;
|
|
||||||
} catch (Exception ignored) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private String writeJson(Object value, String message) {
|
private String writeJson(Object value, String message) {
|
||||||
try {
|
try {
|
||||||
return objectMapper.writeValueAsString(value);
|
return objectMapper.writeValueAsString(value);
|
||||||
|
|||||||
@@ -205,9 +205,7 @@ public class ConvertRunService {
|
|||||||
vo.setResultId(entity.getId());
|
vo.setResultId(entity.getId());
|
||||||
vo.setSourceFilename(entity.getSourceFilename());
|
vo.setSourceFilename(entity.getSourceFilename());
|
||||||
vo.setOutputFilename(entity.getResultFilename());
|
vo.setOutputFilename(entity.getResultFilename());
|
||||||
vo.setDownloadUrl(entity.getSuccess() != null && entity.getSuccess() == 1
|
vo.setDownloadUrl(null);
|
||||||
? ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl())
|
|
||||||
: null);
|
|
||||||
vo.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1);
|
vo.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1);
|
||||||
vo.setError(entity.getErrorMessage());
|
vo.setError(entity.getErrorMessage());
|
||||||
return vo;
|
return vo;
|
||||||
|
|||||||
@@ -229,9 +229,7 @@ public class DedupeRunService {
|
|||||||
vo.setResultId(entity.getId());
|
vo.setResultId(entity.getId());
|
||||||
vo.setSourceFilename(entity.getSourceFilename());
|
vo.setSourceFilename(entity.getSourceFilename());
|
||||||
vo.setOutputFilename(entity.getResultFilename());
|
vo.setOutputFilename(entity.getResultFilename());
|
||||||
vo.setDownloadUrl(entity.getSuccess() != null && entity.getSuccess() == 1
|
vo.setDownloadUrl(null);
|
||||||
? ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl())
|
|
||||||
: null);
|
|
||||||
vo.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1);
|
vo.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1);
|
||||||
vo.setError(entity.getErrorMessage());
|
vo.setError(entity.getErrorMessage());
|
||||||
return vo;
|
return vo;
|
||||||
|
|||||||
@@ -50,6 +50,10 @@ public class DeleteBrandResultItemVo {
|
|||||||
|
|
||||||
@Schema(description = "下载地址")
|
@Schema(description = "下载地址")
|
||||||
private String downloadUrl;
|
private String downloadUrl;
|
||||||
|
private Long fileJobId;
|
||||||
|
private String fileStatus;
|
||||||
|
private String fileError;
|
||||||
|
private Boolean fileReady;
|
||||||
|
|
||||||
@Schema(description = "任务ID")
|
@Schema(description = "任务ID")
|
||||||
private Long taskId;
|
private Long taskId;
|
||||||
|
|||||||
@@ -34,6 +34,8 @@ import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
|||||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||||
import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoShopMatchResultVo;
|
import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoShopMatchResultVo;
|
||||||
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopIndexService;
|
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopIndexService;
|
||||||
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
|
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
|
||||||
@@ -80,6 +82,7 @@ public class DeleteBrandRunService {
|
|||||||
private final ObjectMapper objectMapper;
|
private final ObjectMapper objectMapper;
|
||||||
private final DeleteBrandProgressProperties deleteBrandProgressProperties;
|
private final DeleteBrandProgressProperties deleteBrandProgressProperties;
|
||||||
private final TaskPressureProperties taskPressureProperties;
|
private final TaskPressureProperties taskPressureProperties;
|
||||||
|
private final TaskFileJobService taskFileJobService;
|
||||||
|
|
||||||
public DeleteBrandRunVo run(DeleteBrandRunRequest request) {
|
public DeleteBrandRunVo run(DeleteBrandRunRequest request) {
|
||||||
if (request.getUserId() == null || request.getUserId() <= 0) {
|
if (request.getUserId() == null || request.getUserId() <= 0) {
|
||||||
@@ -271,7 +274,8 @@ public class DeleteBrandRunService {
|
|||||||
item.setSourceFilename(entity.getSourceFilename());
|
item.setSourceFilename(entity.getSourceFilename());
|
||||||
item.setShopName(entity.getSourceFilename() == null ? null : FileUtil.mainName(entity.getSourceFilename()));
|
item.setShopName(entity.getSourceFilename() == null ? null : FileUtil.mainName(entity.getSourceFilename()));
|
||||||
item.setOutputFilename(entity.getResultFilename());
|
item.setOutputFilename(entity.getResultFilename());
|
||||||
item.setDownloadUrl(entity.getResultFileUrl() == null ? null : ossStorageService.getPublicUrl(entity.getResultFileUrl()));
|
item.setDownloadUrl(null);
|
||||||
|
attachFileJobState(item, entity);
|
||||||
item.setTotalRows(entity.getRowCount());
|
item.setTotalRows(entity.getRowCount());
|
||||||
item.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1);
|
item.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1);
|
||||||
item.setError(entity.getErrorMessage());
|
item.setError(entity.getErrorMessage());
|
||||||
@@ -320,6 +324,18 @@ public class DeleteBrandRunService {
|
|||||||
return vo;
|
return vo;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void attachFileJobState(DeleteBrandResultItemVo item, FileResultEntity entity) {
|
||||||
|
TaskFileJobEntity job = taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId());
|
||||||
|
item.setFileReady(entity.getResultFileUrl() != null && !entity.getResultFileUrl().isBlank());
|
||||||
|
if (job == null) {
|
||||||
|
item.setFileStatus(Boolean.TRUE.equals(item.getFileReady()) ? "SUCCESS" : null);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
item.setFileJobId(job.getId());
|
||||||
|
item.setFileStatus(job.getStatus());
|
||||||
|
item.setFileError(job.getErrorMessage());
|
||||||
|
}
|
||||||
|
|
||||||
public DeleteBrandTaskDeletionStatusVo getTaskDeletionStatus(Long taskId, Long userId) {
|
public DeleteBrandTaskDeletionStatusVo getTaskDeletionStatus(Long taskId, Long userId) {
|
||||||
FileTaskEntity task = loadTaskForExecution(taskId);
|
FileTaskEntity task = loadTaskForExecution(taskId);
|
||||||
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !userId.equals(task.getUserId())) {
|
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !userId.equals(task.getUserId())) {
|
||||||
@@ -373,12 +389,19 @@ public class DeleteBrandRunService {
|
|||||||
Map<String, Map<String, DeleteBrandCountryAsinVo>> grouped = new LinkedHashMap<>();
|
Map<String, Map<String, DeleteBrandCountryAsinVo>> grouped = new LinkedHashMap<>();
|
||||||
Map<String, String> displayCountryNames = new LinkedHashMap<>();
|
Map<String, String> displayCountryNames = new LinkedHashMap<>();
|
||||||
|
|
||||||
for (int rowNum = 2; rowNum <= sheet.getLastRowNum(); rowNum++) {
|
int firstDataRow = pairs.stream()
|
||||||
|
.mapToInt(CountryColumnPair::dataStartRowIndex)
|
||||||
|
.min()
|
||||||
|
.orElse(2);
|
||||||
|
for (int rowNum = firstDataRow; rowNum <= sheet.getLastRowNum(); rowNum++) {
|
||||||
Row row = sheet.getRow(rowNum);
|
Row row = sheet.getRow(rowNum);
|
||||||
if (row == null) {
|
if (row == null) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
for (CountryColumnPair pair : pairs) {
|
for (CountryColumnPair pair : pairs) {
|
||||||
|
if (rowNum < pair.dataStartRowIndex()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
String country = pair.country();
|
String country = pair.country();
|
||||||
String asin = normalizeCellText(formatter.formatCellValue(row.getCell(pair.asinColumnIndex())));
|
String asin = normalizeCellText(formatter.formatCellValue(row.getCell(pair.asinColumnIndex())));
|
||||||
String status = normalizeCellText(formatter.formatCellValue(row.getCell(pair.statusColumnIndex())));
|
String status = normalizeCellText(formatter.formatCellValue(row.getCell(pair.statusColumnIndex())));
|
||||||
@@ -457,7 +480,10 @@ public class DeleteBrandRunService {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if ("删除ASIN".equals(firstHeader) && "状态".equals(secondHeader)) {
|
if ("删除ASIN".equals(firstHeader) && "状态".equals(secondHeader)) {
|
||||||
pairs.add(new CountryColumnPair(title, i, i + 1));
|
pairs.add(new CountryColumnPair(title, i, i + 1, 2));
|
||||||
|
i++;
|
||||||
|
} else if (!"店铺名".equals(title) && "状态".equals(secondHeader)) {
|
||||||
|
pairs.add(new CountryColumnPair(title, i, i + 1, 1));
|
||||||
i++;
|
i++;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -486,7 +512,7 @@ public class DeleteBrandRunService {
|
|||||||
.replaceAll("\\s+", " ");
|
.replaceAll("\\s+", " ");
|
||||||
}
|
}
|
||||||
|
|
||||||
private record CountryColumnPair(String country, int asinColumnIndex, int statusColumnIndex) {
|
private record CountryColumnPair(String country, int asinColumnIndex, int statusColumnIndex, int dataStartRowIndex) {
|
||||||
}
|
}
|
||||||
|
|
||||||
private String resolveFileIdentity(DeleteBrandSourceFileDto sourceFile) {
|
private String resolveFileIdentity(DeleteBrandSourceFileDto sourceFile) {
|
||||||
@@ -1085,58 +1111,111 @@ public class DeleteBrandRunService {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
Map<String, DeleteBrandParsedFileCacheDto> parsedPayload = deleteBrandTaskStorageService.loadParsedPayload(taskId);
|
int expectedFiles = deleteBrandTaskStorageService.countParsedPayloadScopes(taskId);
|
||||||
if (parsedPayload == null || parsedPayload.isEmpty()) {
|
if (expectedFiles <= 0) {
|
||||||
|
tryFinalizeTaskFromTerminalResults(task);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
int expectedFiles = parsedPayload.size();
|
|
||||||
// 移除 finishedFilesHint 的强依赖,因为 Redis 里的 finished_files 可能是旧值
|
// 移除 finishedFilesHint 的强依赖,因为 Redis 里的 finished_files 可能是旧值
|
||||||
// 既然进入 tryFinalizeTask,就说明有新分片到达,应该以 loadMergedChunks 为准
|
// 既然进入 tryFinalizeTask,就说明有新分片到达,应该以 loadMergedChunks 为准
|
||||||
|
|
||||||
|
|
||||||
Map<String, List<DeleteBrandResultFileDto>> mergedByFile = deleteBrandTaskStorageService.loadMergedChunks(taskId);
|
int finishedFiles = deleteBrandTaskStorageService.countCompletedScopes(taskId);
|
||||||
int finishedFiles = countCompletedFiles(mergedByFile);
|
|
||||||
|
|
||||||
if (finishedFiles < expectedFiles) {
|
if (finishedFiles < expectedFiles) {
|
||||||
Map<String, String> progress = new LinkedHashMap<>();
|
updateTaskFinalizeProgress(task, finishedFiles, fromCompensation);
|
||||||
progress.put("finished_files", String.valueOf(finishedFiles));
|
|
||||||
// 定时补偿只负责“再试一次 finalize”,不能把缺片任务重新续命
|
|
||||||
// 否则 stale-check 会一直看不到超时任务
|
|
||||||
if (!fromCompensation) {
|
|
||||||
progress.put("updated_at", String.valueOf(System.currentTimeMillis()));
|
|
||||||
}
|
|
||||||
deleteBrandTaskCacheService.saveProgress(taskId, progress);
|
|
||||||
|
|
||||||
task.setSuccessFileCount(finishedFiles);
|
|
||||||
if (!fromCompensation) {
|
|
||||||
task.setUpdatedAt(LocalDateTime.now());
|
|
||||||
}
|
|
||||||
fileTaskMapper.updateById(task);
|
|
||||||
deleteBrandTaskCacheService.saveTaskCache(task);
|
|
||||||
|
|
||||||
log.info("[DeleteBrand] tryFinalizeTask skipping -> finishedFiles ({}) < expectedFiles ({})", finishedFiles, expectedFiles);
|
log.info("[DeleteBrand] tryFinalizeTask skipping -> finishedFiles ({}) < expectedFiles ({})", finishedFiles, expectedFiles);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
Map<String, String> progress = new LinkedHashMap<>();
|
updateTaskFinalizeProgress(task, finishedFiles, true);
|
||||||
progress.put("finished_files", String.valueOf(finishedFiles));
|
Map<String, DeleteBrandParsedFileCacheDto> parsedPayload = deleteBrandTaskStorageService.loadParsedPayload(taskId);
|
||||||
progress.put("updated_at", String.valueOf(System.currentTimeMillis()));
|
if (parsedPayload == null || parsedPayload.isEmpty()) {
|
||||||
if (fromCompensation) {
|
tryFinalizeTaskFromTerminalResults(task);
|
||||||
progress.put("last_heartbeat_at", String.valueOf(System.currentTimeMillis()));
|
return;
|
||||||
}
|
}
|
||||||
deleteBrandTaskCacheService.saveProgress(taskId, progress, true);
|
Map<String, List<DeleteBrandResultFileDto>> mergedByFile = deleteBrandTaskStorageService.loadMergedChunks(taskId);
|
||||||
|
|
||||||
task.setSuccessFileCount(finishedFiles);
|
|
||||||
task.setUpdatedAt(LocalDateTime.now());
|
|
||||||
fileTaskMapper.updateById(task);
|
|
||||||
deleteBrandTaskCacheService.saveTaskCache(task);
|
|
||||||
|
|
||||||
log.info("[DeleteBrand] tryFinalizeTask calling finalizeTask -> finishedFiles: {}, expectedFiles: {}", finishedFiles, expectedFiles);
|
log.info("[DeleteBrand] tryFinalizeTask calling finalizeTask -> finishedFiles: {}, expectedFiles: {}", finishedFiles, expectedFiles);
|
||||||
|
|
||||||
finalizeTask(task, parsedPayload, mergedByFile);
|
finalizeTask(task, parsedPayload, mergedByFile);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void updateTaskFinalizeProgress(FileTaskEntity task, int finishedFiles, boolean fromCompensation) {
|
||||||
|
Map<String, String> progress = new LinkedHashMap<>();
|
||||||
|
progress.put("finished_files", String.valueOf(finishedFiles));
|
||||||
|
if (!fromCompensation) {
|
||||||
|
progress.put("updated_at", String.valueOf(System.currentTimeMillis()));
|
||||||
|
}
|
||||||
|
deleteBrandTaskCacheService.saveProgress(task.getId(), progress);
|
||||||
|
|
||||||
|
Integer currentFinished = task.getSuccessFileCount();
|
||||||
|
if (fromCompensation && currentFinished != null && currentFinished == finishedFiles) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
task.setSuccessFileCount(finishedFiles);
|
||||||
|
if (!fromCompensation) {
|
||||||
|
task.setUpdatedAt(LocalDateTime.now());
|
||||||
|
}
|
||||||
|
fileTaskMapper.updateById(task);
|
||||||
|
deleteBrandTaskCacheService.saveTaskCache(task);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void tryFinalizeTaskFromTerminalResults(FileTaskEntity task) {
|
||||||
|
if (task == null || task.getId() == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
List<FileResultEntity> resultEntities = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
|
||||||
|
.eq(FileResultEntity::getModuleType, MODULE_TYPE)
|
||||||
|
.eq(FileResultEntity::getTaskId, task.getId())
|
||||||
|
.orderByAsc(FileResultEntity::getId));
|
||||||
|
if (resultEntities == null || resultEntities.isEmpty()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (taskFileJobService.countUnfinishedAssembleJobs(task.getId(), MODULE_TYPE) > 0L) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
int successCount = 0;
|
||||||
|
int failedCount = 0;
|
||||||
|
for (FileResultEntity entity : resultEntities) {
|
||||||
|
boolean success = (entity.getSuccess() != null && entity.getSuccess() == 1)
|
||||||
|
|| (entity.getResultFileUrl() != null && !entity.getResultFileUrl().isBlank());
|
||||||
|
boolean failed = !success && entity.getErrorMessage() != null && !entity.getErrorMessage().isBlank();
|
||||||
|
if (!success && !failed) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (success) {
|
||||||
|
successCount++;
|
||||||
|
} else {
|
||||||
|
failedCount++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (task.getSourceFileCount() != null
|
||||||
|
&& task.getSourceFileCount() > 0
|
||||||
|
&& successCount + failedCount < task.getSourceFileCount()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
task.setStatus(successCount > 0 ? "SUCCESS" : "FAILED");
|
||||||
|
task.setSuccessFileCount(successCount);
|
||||||
|
task.setFailedFileCount(failedCount);
|
||||||
|
task.setErrorMessage(successCount > 0 ? null : "任务中间数据缺失,已按结果行状态自动收尾");
|
||||||
|
task.setUpdatedAt(LocalDateTime.now());
|
||||||
|
task.setFinishedAt(LocalDateTime.now());
|
||||||
|
fileTaskMapper.updateById(task);
|
||||||
|
deleteBrandTaskCacheService.saveTaskCache(task);
|
||||||
|
deleteBrandTaskCacheService.saveProgress(task.getId(), Map.of(
|
||||||
|
"phase", successCount > 0 ? "success" : DeleteBrandTaskCacheService.PHASE_FAILED,
|
||||||
|
"finished_files", String.valueOf(successCount),
|
||||||
|
"updated_at", String.valueOf(System.currentTimeMillis())
|
||||||
|
), true);
|
||||||
|
log.info("[DeleteBrand] finalized orphan running task from terminal results -> taskId: {}, status: {}, successFiles: {}, failedFiles: {}",
|
||||||
|
task.getId(), task.getStatus(), successCount, failedCount);
|
||||||
|
}
|
||||||
|
|
||||||
private int countCompletedFiles(Map<String, List<DeleteBrandResultFileDto>> mergedByFile) {
|
private int countCompletedFiles(Map<String, List<DeleteBrandResultFileDto>> mergedByFile) {
|
||||||
int finishedFiles = 0;
|
int finishedFiles = 0;
|
||||||
for (List<DeleteBrandResultFileDto> chunks : mergedByFile.values()) {
|
for (List<DeleteBrandResultFileDto> chunks : mergedByFile.values()) {
|
||||||
@@ -1205,28 +1284,22 @@ public class DeleteBrandRunService {
|
|||||||
if (!isMergedFileCompleted(chunks)) {
|
if (!isMergedFileCompleted(chunks)) {
|
||||||
throw new BusinessException("存在未完成的结果分片: " + parsedFile.getSourceFilename());
|
throw new BusinessException("存在未完成的结果分片: " + parsedFile.getSourceFilename());
|
||||||
}
|
}
|
||||||
MergedDeleteBrandFile mergedFile = mergeChunks(parsedFile, chunks);
|
mergeChunks(parsedFile, chunks);
|
||||||
File outputFile = buildResultWorkbookPreserveLayout(task.getId(), parsedFile, mergedFile);
|
|
||||||
|
|
||||||
deleteBrandTaskCacheService.saveProgress(task.getId(), Map.of(
|
|
||||||
"phase", DeleteBrandTaskCacheService.PHASE_UPLOADING,
|
|
||||||
"file_name", blankToEmpty(parsedFile.getSourceFilename()),
|
|
||||||
"updated_at", String.valueOf(System.currentTimeMillis())
|
|
||||||
));
|
|
||||||
|
|
||||||
String objectKey = ossStorageService.uploadResultFile(outputFile, MODULE_TYPE);
|
|
||||||
FileResultEntity resultEntity = resultEntityByIdentity.get(fileIdentity);
|
FileResultEntity resultEntity = resultEntityByIdentity.get(fileIdentity);
|
||||||
if (resultEntity == null) {
|
if (resultEntity == null) {
|
||||||
throw new BusinessException("结果记录不存在: " + parsedFile.getSourceFilename());
|
throw new BusinessException("结果记录不存在: " + parsedFile.getSourceFilename());
|
||||||
}
|
}
|
||||||
resultEntity.setResultFilename(outputFile.getName());
|
String outputFilename = buildResultFilename(parsedFile);
|
||||||
resultEntity.setResultFileUrl(objectKey);
|
resultEntity.setResultFilename(outputFilename);
|
||||||
resultEntity.setResultFileSize(outputFile.length());
|
resultEntity.setResultFileUrl(null);
|
||||||
|
resultEntity.setResultFileSize(0L);
|
||||||
resultEntity.setResultContentType(CONTENT_TYPE_XLSX);
|
resultEntity.setResultContentType(CONTENT_TYPE_XLSX);
|
||||||
resultEntity.setRowCount(parsedFile.getTotalRows());
|
resultEntity.setRowCount(parsedFile.getTotalRows());
|
||||||
resultEntity.setSuccess(1);
|
resultEntity.setSuccess(1);
|
||||||
resultEntity.setErrorMessage(null);
|
resultEntity.setErrorMessage(null);
|
||||||
fileResultMapper.updateById(resultEntity);
|
fileResultMapper.updateById(resultEntity);
|
||||||
|
taskFileJobService.enqueueAssembleResult(task.getId(), MODULE_TYPE, resultEntity.getId(), fileIdentity);
|
||||||
|
|
||||||
DeleteBrandResultItemVo item = new DeleteBrandResultItemVo();
|
DeleteBrandResultItemVo item = new DeleteBrandResultItemVo();
|
||||||
item.setResultId(resultEntity.getId());
|
item.setResultId(resultEntity.getId());
|
||||||
@@ -1235,8 +1308,8 @@ public class DeleteBrandRunService {
|
|||||||
item.setSourceFilename(parsedFile.getSourceFilename());
|
item.setSourceFilename(parsedFile.getSourceFilename());
|
||||||
item.setShopName(parsedFile.getShopName());
|
item.setShopName(parsedFile.getShopName());
|
||||||
item.setCompanyName(parsedFile.getCompanyName());
|
item.setCompanyName(parsedFile.getCompanyName());
|
||||||
item.setOutputFilename(outputFile.getName());
|
item.setOutputFilename(outputFilename);
|
||||||
item.setDownloadUrl(ossStorageService.getPublicUrl(objectKey));
|
item.setDownloadUrl(null);
|
||||||
item.setTotalRows(parsedFile.getTotalRows());
|
item.setTotalRows(parsedFile.getTotalRows());
|
||||||
item.setCountryCount(parsedFile.getCountries() == null ? 0 : parsedFile.getCountries().size());
|
item.setCountryCount(parsedFile.getCountries() == null ? 0 : parsedFile.getCountries().size());
|
||||||
item.setCountries(parsedFile.getCountries());
|
item.setCountries(parsedFile.getCountries());
|
||||||
@@ -1263,8 +1336,6 @@ public class DeleteBrandRunService {
|
|||||||
task.setFinishedAt(LocalDateTime.now());
|
task.setFinishedAt(LocalDateTime.now());
|
||||||
fileTaskMapper.updateById(task);
|
fileTaskMapper.updateById(task);
|
||||||
deleteBrandTaskCacheService.saveTaskCache(task);
|
deleteBrandTaskCacheService.saveTaskCache(task);
|
||||||
deleteBrandTaskStorageService.deleteTaskData(task.getId());
|
|
||||||
|
|
||||||
deleteBrandTaskCacheService.saveProgress(task.getId(), Map.of(
|
deleteBrandTaskCacheService.saveProgress(task.getId(), Map.of(
|
||||||
"phase", "success",
|
"phase", "success",
|
||||||
"file_index", String.valueOf(parsedPayload.size()),
|
"file_index", String.valueOf(parsedPayload.size()),
|
||||||
@@ -1293,6 +1364,75 @@ public class DeleteBrandRunService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void processResultFileJob(TaskFileJobEntity job) {
|
||||||
|
if (job == null || job.getTaskId() == null || job.getResultId() == null) {
|
||||||
|
throw new BusinessException("结果文件任务参数不完整");
|
||||||
|
}
|
||||||
|
FileTaskEntity task = fileTaskMapper.selectById(job.getTaskId());
|
||||||
|
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
|
||||||
|
throw new BusinessException("任务不存在");
|
||||||
|
}
|
||||||
|
FileResultEntity resultEntity = fileResultMapper.selectById(job.getResultId());
|
||||||
|
if (resultEntity == null || !MODULE_TYPE.equals(resultEntity.getModuleType())) {
|
||||||
|
throw new BusinessException("结果记录不存在");
|
||||||
|
}
|
||||||
|
Map<String, DeleteBrandParsedFileCacheDto> parsedPayload = deleteBrandTaskStorageService.loadParsedPayload(job.getTaskId());
|
||||||
|
Map<String, List<DeleteBrandResultFileDto>> mergedByFile = deleteBrandTaskStorageService.loadMergedChunks(job.getTaskId());
|
||||||
|
String fileIdentity = blankToEmpty(job.getScopeKey());
|
||||||
|
if (fileIdentity.isBlank()) {
|
||||||
|
fileIdentity = blankToEmpty(resultEntity.getSourceFileUrl());
|
||||||
|
}
|
||||||
|
if (fileIdentity.isBlank()) {
|
||||||
|
fileIdentity = blankToEmpty(resultEntity.getSourceFilename());
|
||||||
|
}
|
||||||
|
DeleteBrandParsedFileCacheDto parsedFile = parsedPayload.get(fileIdentity);
|
||||||
|
if (parsedFile == null) {
|
||||||
|
throw new BusinessException("任务原始数据不存在: " + fileIdentity);
|
||||||
|
}
|
||||||
|
List<DeleteBrandResultFileDto> chunks = mergedByFile.get(fileIdentity);
|
||||||
|
if (!isMergedFileCompleted(chunks)) {
|
||||||
|
throw new BusinessException("结果分片未完整: " + fileIdentity);
|
||||||
|
}
|
||||||
|
MergedDeleteBrandFile mergedFile = mergeChunks(parsedFile, chunks);
|
||||||
|
File outputFile = buildResultWorkbookPreserveLayout(job.getTaskId(), parsedFile, mergedFile);
|
||||||
|
try {
|
||||||
|
deleteBrandTaskCacheService.saveProgress(job.getTaskId(), Map.of(
|
||||||
|
"phase", DeleteBrandTaskCacheService.PHASE_UPLOADING,
|
||||||
|
"file_name", blankToEmpty(parsedFile.getSourceFilename()),
|
||||||
|
"updated_at", String.valueOf(System.currentTimeMillis())
|
||||||
|
));
|
||||||
|
String objectKey = ossStorageService.uploadResultFile(outputFile, MODULE_TYPE);
|
||||||
|
resultEntity.setResultFilename(outputFile.getName());
|
||||||
|
resultEntity.setResultFileUrl(objectKey);
|
||||||
|
resultEntity.setResultFileSize(outputFile.length());
|
||||||
|
resultEntity.setResultContentType(CONTENT_TYPE_XLSX);
|
||||||
|
resultEntity.setRowCount(parsedFile.getTotalRows());
|
||||||
|
resultEntity.setSuccess(1);
|
||||||
|
resultEntity.setErrorMessage(null);
|
||||||
|
fileResultMapper.updateById(resultEntity);
|
||||||
|
} finally {
|
||||||
|
deleteQuietly(outputFile);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void cleanupResultFileJob(TaskFileJobEntity job) {
|
||||||
|
if (job == null || job.getTaskId() == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (taskFileJobService.countUnfinishedAssembleJobs(job.getTaskId(), MODULE_TYPE) == 0L) {
|
||||||
|
deleteBrandTaskStorageService.deleteTaskData(job.getTaskId());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private String buildResultFilename(DeleteBrandParsedFileCacheDto parsedFile) {
|
||||||
|
String filename = blankToDefault(parsedFile.getSourceFilename(), "delete-brand-result.xlsx");
|
||||||
|
String lower = filename.toLowerCase(Locale.ROOT);
|
||||||
|
if (!lower.endsWith(".xlsx") && !lower.endsWith(".xls")) {
|
||||||
|
return filename + ".xlsx";
|
||||||
|
}
|
||||||
|
return filename;
|
||||||
|
}
|
||||||
|
|
||||||
private MergedDeleteBrandFile mergeChunks(DeleteBrandParsedFileCacheDto parsedFile, List<DeleteBrandResultFileDto> chunks) {
|
private MergedDeleteBrandFile mergeChunks(DeleteBrandParsedFileCacheDto parsedFile, List<DeleteBrandResultFileDto> chunks) {
|
||||||
Integer chunkTotal = chunks.get(0).getChunkTotal();
|
Integer chunkTotal = chunks.get(0).getChunkTotal();
|
||||||
for (DeleteBrandResultFileDto chunk : chunks) {
|
for (DeleteBrandResultFileDto chunk : chunks) {
|
||||||
|
|||||||
@@ -27,7 +27,6 @@ import java.nio.file.attribute.FileTime;
|
|||||||
import java.time.Duration;
|
import java.time.Duration;
|
||||||
import java.time.Instant;
|
import java.time.Instant;
|
||||||
import java.time.LocalDateTime;
|
import java.time.LocalDateTime;
|
||||||
import java.time.ZoneId;
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
@@ -47,6 +46,7 @@ public class DeleteBrandStaleTaskService {
|
|||||||
|
|
||||||
private final FileTaskMapper fileTaskMapper;
|
private final FileTaskMapper fileTaskMapper;
|
||||||
private final DeleteBrandTaskCacheService deleteBrandTaskCacheService;
|
private final DeleteBrandTaskCacheService deleteBrandTaskCacheService;
|
||||||
|
private final DeleteBrandTaskStorageService deleteBrandTaskStorageService;
|
||||||
private final DeleteBrandRunService deleteBrandRunService;
|
private final DeleteBrandRunService deleteBrandRunService;
|
||||||
private final ProductRiskTaskService productRiskTaskService;
|
private final ProductRiskTaskService productRiskTaskService;
|
||||||
private final ProductRiskTaskCacheService productRiskTaskCacheService;
|
private final ProductRiskTaskCacheService productRiskTaskCacheService;
|
||||||
@@ -110,7 +110,13 @@ public class DeleteBrandStaleTaskService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private void failStaleDeleteBrandTasks() {
|
private void failStaleDeleteBrandTasks() {
|
||||||
LocalDateTime threshold = LocalDateTime.now().minusMinutes(deleteBrandProgressProperties.getHeartbeatTimeoutMinutes());
|
long minutes = Math.max(1L, deleteBrandProgressProperties.getHeartbeatTimeoutMinutes());
|
||||||
|
long initialMinutes = Math.max(minutes, deleteBrandProgressProperties.getDeleteBrandInitialTimeoutMinutes());
|
||||||
|
long staleTimeoutMillis = Duration.ofMinutes(minutes).toMillis();
|
||||||
|
long nowMillis = System.currentTimeMillis();
|
||||||
|
LocalDateTime now = LocalDateTime.now();
|
||||||
|
LocalDateTime threshold = now.minusMinutes(minutes);
|
||||||
|
LocalDateTime initialThreshold = now.minusMinutes(initialMinutes);
|
||||||
|
|
||||||
java.util.List<FileTaskEntity> runningTasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
java.util.List<FileTaskEntity> runningTasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||||
.eq(FileTaskEntity::getModuleType, MODULE_TYPE_DELETE_BRAND)
|
.eq(FileTaskEntity::getModuleType, MODULE_TYPE_DELETE_BRAND)
|
||||||
@@ -120,36 +126,35 @@ public class DeleteBrandStaleTaskService {
|
|||||||
|
|
||||||
for (FileTaskEntity task : runningTasks) {
|
for (FileTaskEntity task : runningTasks) {
|
||||||
Map<Object, Object> progress = deleteBrandTaskCacheService.getProgress(task.getId());
|
Map<Object, Object> progress = deleteBrandTaskCacheService.getProgress(task.getId());
|
||||||
boolean hasProgress = progress != null && !progress.isEmpty() && progress.containsKey("last_heartbeat_at");
|
|
||||||
|
|
||||||
boolean isActive = false;
|
|
||||||
if (progress != null && progress.containsKey("has_progress")) {
|
|
||||||
Object hpObj = progress.get("has_progress");
|
|
||||||
if (hpObj instanceof Boolean) {
|
|
||||||
isActive = (Boolean) hpObj;
|
|
||||||
} else if (hpObj instanceof String) {
|
|
||||||
isActive = Boolean.parseBoolean((String) hpObj);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
long lastHeartbeatAt = 0L;
|
long lastHeartbeatAt = 0L;
|
||||||
if (hasProgress) {
|
if (progress != null && !progress.isEmpty() && progress.containsKey("last_heartbeat_at")) {
|
||||||
try {
|
try {
|
||||||
lastHeartbeatAt = Long.parseLong(String.valueOf(progress.get("last_heartbeat_at")));
|
lastHeartbeatAt = Long.parseLong(String.valueOf(progress.get("last_heartbeat_at")));
|
||||||
} catch (Exception ignored) {
|
} catch (Exception ignored) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
boolean hasStartedProgress = lastHeartbeatAt > 0L;
|
||||||
|
if (!hasStartedProgress) {
|
||||||
|
hasStartedProgress = deleteBrandTaskStorageService.countCompletedScopes(task.getId()) > 0
|
||||||
|
|| deleteBrandTaskStorageService.countParsedPayloadScopes(task.getId()) > 0;
|
||||||
|
}
|
||||||
|
|
||||||
if (lastHeartbeatAt > 0 && isActive) {
|
if (lastHeartbeatAt > 0L && nowMillis - lastHeartbeatAt < staleTimeoutMillis) {
|
||||||
LocalDateTime lastActivityTime = LocalDateTime.ofInstant(Instant.ofEpochMilli(lastHeartbeatAt), ZoneId.systemDefault());
|
continue;
|
||||||
if (lastActivityTime.isAfter(threshold)) {
|
}
|
||||||
continue;
|
if (!hasStartedProgress && task.getCreatedAt() != null && task.getCreatedAt().isAfter(initialThreshold)) {
|
||||||
}
|
continue;
|
||||||
} else {
|
}
|
||||||
LocalDateTime queuedThreshold = LocalDateTime.now().minusHours(12);
|
|
||||||
if (task.getCreatedAt() != null && task.getCreatedAt().isAfter(queuedThreshold)) {
|
try {
|
||||||
|
deleteBrandRunService.tryFinalizeTask(task.getId(), true);
|
||||||
|
FileTaskEntity refreshed = fileTaskMapper.selectById(task.getId());
|
||||||
|
if (refreshed != null && !"RUNNING".equals(refreshed.getStatus())) {
|
||||||
|
deleteBrandTaskCacheService.saveTaskCache(refreshed);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[stale-check] delete-brand finalize threw taskId={} msg={}", task.getId(), ex.getMessage());
|
||||||
}
|
}
|
||||||
|
|
||||||
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
|
||||||
@@ -185,20 +190,24 @@ public class DeleteBrandStaleTaskService {
|
|||||||
if (runningTasks.isEmpty()) {
|
if (runningTasks.isEmpty()) {
|
||||||
return stats;
|
return stats;
|
||||||
}
|
}
|
||||||
|
Map<Long, Long> heartbeatByTaskId = productRiskTaskCacheService.getTaskHeartbeatMillisBatch(
|
||||||
|
runningTasks.stream().map(FileTaskEntity::getId).toList());
|
||||||
log.info("[stale-check] product-risk candidates={} threshold={} timeoutMinutes={} initialThreshold={} initialTimeoutMinutes={}",
|
log.info("[stale-check] product-risk candidates={} threshold={} timeoutMinutes={} initialThreshold={} initialTimeoutMinutes={}",
|
||||||
runningTasks.size(), threshold, minutes, initialThreshold, initialMinutes);
|
runningTasks.size(), threshold, minutes, initialThreshold, initialMinutes);
|
||||||
for (FileTaskEntity task : runningTasks) {
|
for (FileTaskEntity task : runningTasks) {
|
||||||
boolean hasUploadedPayload = productRiskTaskCacheService.hasAnyShopMergedPayload(task.getId());
|
long lastPayloadHeartbeatMillis = heartbeatByTaskId.getOrDefault(task.getId(), 0L);
|
||||||
long lastPayloadHeartbeatMillis = productRiskTaskCacheService.getTaskHeartbeatMillis(task.getId());
|
|
||||||
boolean hasFinishedRows = (task.getSuccessFileCount() != null && task.getSuccessFileCount() > 0)
|
boolean hasFinishedRows = (task.getSuccessFileCount() != null && task.getSuccessFileCount() > 0)
|
||||||
|| (task.getFailedFileCount() != null && task.getFailedFileCount() > 0);
|
|| (task.getFailedFileCount() != null && task.getFailedFileCount() > 0);
|
||||||
boolean hasStartedProgress = hasUploadedPayload || hasFinishedRows;
|
|
||||||
if (lastPayloadHeartbeatMillis > 0L && nowMillis - lastPayloadHeartbeatMillis < staleTimeoutMillis) {
|
if (lastPayloadHeartbeatMillis > 0L && nowMillis - lastPayloadHeartbeatMillis < staleTimeoutMillis) {
|
||||||
stats.skippedTaskCount++;
|
stats.skippedTaskCount++;
|
||||||
log.info("[stale-check] product-risk skip recent-task-heartbeat taskId={} lastPayloadHeartbeatMillis={} timeoutMinutes={} updatedAt={}",
|
log.info("[stale-check] product-risk skip recent-task-heartbeat taskId={} lastPayloadHeartbeatMillis={} timeoutMinutes={} updatedAt={}",
|
||||||
task.getId(), lastPayloadHeartbeatMillis, minutes, task.getUpdatedAt());
|
task.getId(), lastPayloadHeartbeatMillis, minutes, task.getUpdatedAt());
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
boolean hasStartedProgress = lastPayloadHeartbeatMillis > 0L || hasFinishedRows;
|
||||||
|
if (!hasStartedProgress) {
|
||||||
|
hasStartedProgress = productRiskTaskCacheService.hasAnyShopMergedPayload(task.getId());
|
||||||
|
}
|
||||||
if (!hasStartedProgress && task.getCreatedAt() != null && task.getCreatedAt().isAfter(initialThreshold)) {
|
if (!hasStartedProgress && task.getCreatedAt() != null && task.getCreatedAt().isAfter(initialThreshold)) {
|
||||||
stats.skippedTaskCount++;
|
stats.skippedTaskCount++;
|
||||||
log.info("[stale-check] product-risk skip initial-grace taskId={} createdAt={} initialThreshold={}",
|
log.info("[stale-check] product-risk skip initial-grace taskId={} createdAt={} initialThreshold={}",
|
||||||
@@ -250,18 +259,22 @@ public class DeleteBrandStaleTaskService {
|
|||||||
if (runningTasks.isEmpty()) {
|
if (runningTasks.isEmpty()) {
|
||||||
return stats;
|
return stats;
|
||||||
}
|
}
|
||||||
|
Map<Long, Long> heartbeatByTaskId = priceTrackTaskCacheService.getTaskHeartbeatMillisBatch(
|
||||||
|
runningTasks.stream().map(FileTaskEntity::getId).toList());
|
||||||
log.info("[stale-check] price-track candidates={} threshold={} timeoutMinutes={} initialThreshold={} initialTimeoutMinutes={}",
|
log.info("[stale-check] price-track candidates={} threshold={} timeoutMinutes={} initialThreshold={} initialTimeoutMinutes={}",
|
||||||
runningTasks.size(), threshold, minutes, initialThreshold, initialMinutes);
|
runningTasks.size(), threshold, minutes, initialThreshold, initialMinutes);
|
||||||
for (FileTaskEntity task : runningTasks) {
|
for (FileTaskEntity task : runningTasks) {
|
||||||
long lastHeartbeatMillis = priceTrackTaskCacheService.getTaskHeartbeatMillis(task.getId());
|
long lastHeartbeatMillis = heartbeatByTaskId.getOrDefault(task.getId(), 0L);
|
||||||
boolean hasUploadedPayload = priceTrackTaskCacheService.hasAnyShopMergedPayload(task.getId());
|
|
||||||
boolean hasFinishedRows = (task.getSuccessFileCount() != null && task.getSuccessFileCount() > 0)
|
boolean hasFinishedRows = (task.getSuccessFileCount() != null && task.getSuccessFileCount() > 0)
|
||||||
|| (task.getFailedFileCount() != null && task.getFailedFileCount() > 0);
|
|| (task.getFailedFileCount() != null && task.getFailedFileCount() > 0);
|
||||||
boolean hasStartedProgress = lastHeartbeatMillis > 0L || hasUploadedPayload || hasFinishedRows;
|
|
||||||
if (lastHeartbeatMillis > 0L && nowMillis - lastHeartbeatMillis < staleTimeoutMillis) {
|
if (lastHeartbeatMillis > 0L && nowMillis - lastHeartbeatMillis < staleTimeoutMillis) {
|
||||||
stats.skippedTaskCount++;
|
stats.skippedTaskCount++;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
boolean hasStartedProgress = lastHeartbeatMillis > 0L || hasFinishedRows;
|
||||||
|
if (!hasStartedProgress) {
|
||||||
|
hasStartedProgress = priceTrackTaskCacheService.hasAnyShopMergedPayload(task.getId());
|
||||||
|
}
|
||||||
if (!hasStartedProgress && task.getCreatedAt() != null && task.getCreatedAt().isAfter(initialThreshold)) {
|
if (!hasStartedProgress && task.getCreatedAt() != null && task.getCreatedAt().isAfter(initialThreshold)) {
|
||||||
stats.skippedTaskCount++;
|
stats.skippedTaskCount++;
|
||||||
continue;
|
continue;
|
||||||
@@ -312,20 +325,24 @@ public class DeleteBrandStaleTaskService {
|
|||||||
if (runningTasks.isEmpty()) {
|
if (runningTasks.isEmpty()) {
|
||||||
return stats;
|
return stats;
|
||||||
}
|
}
|
||||||
|
Map<Long, Long> heartbeatByTaskId = shopMatchTaskCacheService.getTaskHeartbeatMillisBatch(
|
||||||
|
runningTasks.stream().map(FileTaskEntity::getId).toList());
|
||||||
log.info("[stale-check] shop-match candidates={} threshold={} timeoutMinutes={} initialThreshold={} initialTimeoutMinutes={}",
|
log.info("[stale-check] shop-match candidates={} threshold={} timeoutMinutes={} initialThreshold={} initialTimeoutMinutes={}",
|
||||||
runningTasks.size(), threshold, minutes, initialThreshold, initialMinutes);
|
runningTasks.size(), threshold, minutes, initialThreshold, initialMinutes);
|
||||||
for (FileTaskEntity task : runningTasks) {
|
for (FileTaskEntity task : runningTasks) {
|
||||||
long lastHeartbeatMillis = shopMatchTaskCacheService.getTaskHeartbeatMillis(task.getId());
|
long lastHeartbeatMillis = heartbeatByTaskId.getOrDefault(task.getId(), 0L);
|
||||||
boolean hasUploadedPayload = shopMatchTaskCacheService.hasAnyShopMergedPayload(task.getId());
|
|
||||||
boolean hasFinishedRows = (task.getSuccessFileCount() != null && task.getSuccessFileCount() > 0)
|
boolean hasFinishedRows = (task.getSuccessFileCount() != null && task.getSuccessFileCount() > 0)
|
||||||
|| (task.getFailedFileCount() != null && task.getFailedFileCount() > 0);
|
|| (task.getFailedFileCount() != null && task.getFailedFileCount() > 0);
|
||||||
boolean hasStartedProgress = lastHeartbeatMillis > 0L || hasUploadedPayload || hasFinishedRows;
|
|
||||||
if (lastHeartbeatMillis > 0L && nowMillis - lastHeartbeatMillis < staleTimeoutMillis) {
|
if (lastHeartbeatMillis > 0L && nowMillis - lastHeartbeatMillis < staleTimeoutMillis) {
|
||||||
stats.skippedTaskCount++;
|
stats.skippedTaskCount++;
|
||||||
log.info("[stale-check] shop-match skip recent-task-heartbeat taskId={} lastHeartbeatMillis={} timeoutMinutes={} updatedAt={}",
|
log.info("[stale-check] shop-match skip recent-task-heartbeat taskId={} lastHeartbeatMillis={} timeoutMinutes={} updatedAt={}",
|
||||||
task.getId(), lastHeartbeatMillis, minutes, task.getUpdatedAt());
|
task.getId(), lastHeartbeatMillis, minutes, task.getUpdatedAt());
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
boolean hasStartedProgress = lastHeartbeatMillis > 0L || hasFinishedRows;
|
||||||
|
if (!hasStartedProgress) {
|
||||||
|
hasStartedProgress = shopMatchTaskCacheService.hasAnyShopMergedPayload(task.getId());
|
||||||
|
}
|
||||||
if (!hasStartedProgress && task.getCreatedAt() != null && task.getCreatedAt().isAfter(initialThreshold)) {
|
if (!hasStartedProgress && task.getCreatedAt() != null && task.getCreatedAt().isAfter(initialThreshold)) {
|
||||||
stats.skippedTaskCount++;
|
stats.skippedTaskCount++;
|
||||||
log.info("[stale-check] shop-match skip initial-grace taskId={} createdAt={} initialThreshold={}",
|
log.info("[stale-check] shop-match skip initial-grace taskId={} createdAt={} initialThreshold={}",
|
||||||
@@ -378,16 +395,20 @@ public class DeleteBrandStaleTaskService {
|
|||||||
if (runningTasks.isEmpty()) {
|
if (runningTasks.isEmpty()) {
|
||||||
return stats;
|
return stats;
|
||||||
}
|
}
|
||||||
|
Map<Long, Long> heartbeatByTaskId = patrolDeleteTaskCacheService.getTaskHeartbeatMillisBatch(
|
||||||
|
runningTasks.stream().map(FileTaskEntity::getId).toList());
|
||||||
for (FileTaskEntity task : runningTasks) {
|
for (FileTaskEntity task : runningTasks) {
|
||||||
long lastHeartbeatMillis = patrolDeleteTaskCacheService.getTaskHeartbeatMillis(task.getId());
|
long lastHeartbeatMillis = heartbeatByTaskId.getOrDefault(task.getId(), 0L);
|
||||||
boolean hasUploadedPayload = patrolDeleteTaskCacheService.hasAnyShopMergedPayload(task.getId());
|
|
||||||
boolean hasFinishedRows = (task.getSuccessFileCount() != null && task.getSuccessFileCount() > 0)
|
boolean hasFinishedRows = (task.getSuccessFileCount() != null && task.getSuccessFileCount() > 0)
|
||||||
|| (task.getFailedFileCount() != null && task.getFailedFileCount() > 0);
|
|| (task.getFailedFileCount() != null && task.getFailedFileCount() > 0);
|
||||||
boolean hasStartedProgress = lastHeartbeatMillis > 0L || hasUploadedPayload || hasFinishedRows;
|
|
||||||
if (lastHeartbeatMillis > 0L && nowMillis - lastHeartbeatMillis < staleTimeoutMillis) {
|
if (lastHeartbeatMillis > 0L && nowMillis - lastHeartbeatMillis < staleTimeoutMillis) {
|
||||||
stats.skippedTaskCount++;
|
stats.skippedTaskCount++;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
boolean hasStartedProgress = lastHeartbeatMillis > 0L || hasFinishedRows;
|
||||||
|
if (!hasStartedProgress) {
|
||||||
|
hasStartedProgress = patrolDeleteTaskCacheService.hasAnyShopMergedPayload(task.getId());
|
||||||
|
}
|
||||||
if (!hasStartedProgress && task.getCreatedAt() != null && task.getCreatedAt().isAfter(initialThreshold)) {
|
if (!hasStartedProgress && task.getCreatedAt() != null && task.getCreatedAt().isAfter(initialThreshold)) {
|
||||||
stats.skippedTaskCount++;
|
stats.skippedTaskCount++;
|
||||||
continue;
|
continue;
|
||||||
@@ -431,6 +452,10 @@ public class DeleteBrandStaleTaskService {
|
|||||||
java.util.List<FileTaskEntity> runningTasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
java.util.List<FileTaskEntity> runningTasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
|
||||||
.eq(FileTaskEntity::getModuleType, MODULE_TYPE_DELETE_BRAND)
|
.eq(FileTaskEntity::getModuleType, MODULE_TYPE_DELETE_BRAND)
|
||||||
.eq(FileTaskEntity::getStatus, "RUNNING")
|
.eq(FileTaskEntity::getStatus, "RUNNING")
|
||||||
|
.and(wrapper -> wrapper
|
||||||
|
.gt(FileTaskEntity::getSuccessFileCount, 0)
|
||||||
|
.or()
|
||||||
|
.gt(FileTaskEntity::getFailedFileCount, 0))
|
||||||
.orderByAsc(FileTaskEntity::getUpdatedAt)
|
.orderByAsc(FileTaskEntity::getUpdatedAt)
|
||||||
.last("limit 100"));
|
.last("limit 100"));
|
||||||
|
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package com.nanri.aiimage.modules.deletebrand.service;
|
|||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
import com.nanri.aiimage.config.TaskPressureProperties;
|
import com.nanri.aiimage.config.TaskPressureProperties;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
@@ -11,6 +12,7 @@ import java.util.concurrent.ConcurrentHashMap;
|
|||||||
|
|
||||||
@Service
|
@Service
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
|
@Slf4j
|
||||||
public class DeleteBrandTaskCacheService {
|
public class DeleteBrandTaskCacheService {
|
||||||
|
|
||||||
public static final String PHASE_CRAWLING = "crawling";
|
public static final String PHASE_CRAWLING = "crawling";
|
||||||
@@ -52,9 +54,13 @@ public class DeleteBrandTaskCacheService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
String key = buildProgressKey(taskId);
|
String key = buildProgressKey(taskId);
|
||||||
stringRedisTemplate.opsForHash().putAll(key, merged);
|
try {
|
||||||
stringRedisTemplate.expire(key, Duration.ofHours(PAYLOAD_TTL_HOURS));
|
stringRedisTemplate.opsForHash().putAll(key, merged);
|
||||||
progressRedisFlushAt.put(taskId, now);
|
stringRedisTemplate.expire(key, Duration.ofHours(PAYLOAD_TTL_HOURS));
|
||||||
|
progressRedisFlushAt.put(taskId, now);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[delete-brand-cache] save progress degraded taskId={} msg={}", taskId, ex.getMessage());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public java.util.Map<Object, Object> getProgress(Long taskId) {
|
public java.util.Map<Object, Object> getProgress(Long taskId) {
|
||||||
@@ -63,7 +69,13 @@ public class DeleteBrandTaskCacheService {
|
|||||||
if (cached != null && cached.isFresh(now)) {
|
if (cached != null && cached.isFresh(now)) {
|
||||||
return new java.util.LinkedHashMap<>(cached.values());
|
return new java.util.LinkedHashMap<>(cached.values());
|
||||||
}
|
}
|
||||||
java.util.Map<Object, Object> values = stringRedisTemplate.opsForHash().entries(buildProgressKey(taskId));
|
java.util.Map<Object, Object> values;
|
||||||
|
try {
|
||||||
|
values = stringRedisTemplate.opsForHash().entries(buildProgressKey(taskId));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[delete-brand-cache] load progress degraded taskId={} msg={}", taskId, ex.getMessage());
|
||||||
|
return java.util.Collections.emptyMap();
|
||||||
|
}
|
||||||
if (!values.isEmpty()) {
|
if (!values.isEmpty()) {
|
||||||
progressLocalCache.put(taskId, new LocalProgressCacheEntry(now, toStringMap(values)));
|
progressLocalCache.put(taskId, new LocalProgressCacheEntry(now, toStringMap(values)));
|
||||||
}
|
}
|
||||||
@@ -98,15 +110,24 @@ public class DeleteBrandTaskCacheService {
|
|||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
java.util.List<Object> pipelineResults = stringRedisTemplate.executePipelined(
|
java.util.List<Object> pipelineResults;
|
||||||
(org.springframework.data.redis.core.RedisCallback<Object>) connection -> {
|
try {
|
||||||
org.springframework.data.redis.serializer.RedisSerializer<String> serializer =
|
pipelineResults = stringRedisTemplate.executePipelined(
|
||||||
stringRedisTemplate.getStringSerializer();
|
(org.springframework.data.redis.core.RedisCallback<Object>) connection -> {
|
||||||
for (Long taskId : missingTaskIds) {
|
org.springframework.data.redis.serializer.RedisSerializer<String> serializer =
|
||||||
connection.hGetAll(serializer.serialize(buildProgressKey(taskId)));
|
stringRedisTemplate.getStringSerializer();
|
||||||
}
|
for (Long taskId : missingTaskIds) {
|
||||||
return null;
|
connection.hGetAll(serializer.serialize(buildProgressKey(taskId)));
|
||||||
});
|
}
|
||||||
|
return null;
|
||||||
|
});
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[delete-brand-cache] batch load progress degraded taskIds={} msg={}", missingTaskIds, ex.getMessage());
|
||||||
|
for (Long taskId : missingTaskIds) {
|
||||||
|
result.put(taskId, java.util.Collections.emptyMap());
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
for (int i = 0; i < missingTaskIds.size(); i++) {
|
for (int i = 0; i < missingTaskIds.size(); i++) {
|
||||||
Long taskId = missingTaskIds.get(i);
|
Long taskId = missingTaskIds.get(i);
|
||||||
@@ -129,8 +150,12 @@ public class DeleteBrandTaskCacheService {
|
|||||||
progressLocalCache.remove(taskId);
|
progressLocalCache.remove(taskId);
|
||||||
progressRedisFlushAt.remove(taskId);
|
progressRedisFlushAt.remove(taskId);
|
||||||
taskEntityLocalCache.remove(taskId);
|
taskEntityLocalCache.remove(taskId);
|
||||||
stringRedisTemplate.delete(buildProgressKey(taskId));
|
try {
|
||||||
stringRedisTemplate.delete(buildTaskEntityKey(taskId));
|
stringRedisTemplate.delete(buildProgressKey(taskId));
|
||||||
|
stringRedisTemplate.delete(buildTaskEntityKey(taskId));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[delete-brand-cache] delete degraded taskId={} msg={}", taskId, ex.getMessage());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public void saveTaskCache(com.nanri.aiimage.modules.task.model.entity.FileTaskEntity task) {
|
public void saveTaskCache(com.nanri.aiimage.modules.task.model.entity.FileTaskEntity task) {
|
||||||
@@ -180,7 +205,13 @@ public class DeleteBrandTaskCacheService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
java.util.List<String> keys = missingIds.stream().map(this::buildTaskEntityKey).toList();
|
java.util.List<String> keys = missingIds.stream().map(this::buildTaskEntityKey).toList();
|
||||||
java.util.List<String> values = stringRedisTemplate.opsForValue().multiGet(keys);
|
java.util.List<String> values;
|
||||||
|
try {
|
||||||
|
values = stringRedisTemplate.opsForValue().multiGet(keys);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[delete-brand-cache] batch load task cache degraded taskIds={} msg={}", missingIds, ex.getMessage());
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
for (int i = 0; i < missingIds.size(); i++) {
|
for (int i = 0; i < missingIds.size(); i++) {
|
||||||
Long taskId = missingIds.get(i);
|
Long taskId = missingIds.get(i);
|
||||||
|
|||||||
@@ -6,11 +6,11 @@ import com.fasterxml.jackson.databind.ObjectMapper;
|
|||||||
import com.nanri.aiimage.common.exception.BusinessException;
|
import com.nanri.aiimage.common.exception.BusinessException;
|
||||||
import com.nanri.aiimage.modules.deletebrand.model.cache.DeleteBrandParsedFileCacheDto;
|
import com.nanri.aiimage.modules.deletebrand.model.cache.DeleteBrandParsedFileCacheDto;
|
||||||
import com.nanri.aiimage.modules.deletebrand.model.dto.DeleteBrandResultFileDto;
|
import com.nanri.aiimage.modules.deletebrand.model.dto.DeleteBrandResultFileDto;
|
||||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
|
||||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||||
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.dao.DuplicateKeyException;
|
import org.springframework.dao.DuplicateKeyException;
|
||||||
@@ -32,12 +32,11 @@ import java.util.Map;
|
|||||||
public class DeleteBrandTaskStorageService {
|
public class DeleteBrandTaskStorageService {
|
||||||
|
|
||||||
private static final String MODULE_TYPE = "DELETE_BRAND";
|
private static final String MODULE_TYPE = "DELETE_BRAND";
|
||||||
private static final String OSS_POINTER_PREFIX = "oss:";
|
|
||||||
|
|
||||||
private final TaskChunkMapper taskChunkMapper;
|
private final TaskChunkMapper taskChunkMapper;
|
||||||
private final TaskScopeStateMapper taskScopeStateMapper;
|
private final TaskScopeStateMapper taskScopeStateMapper;
|
||||||
private final ObjectMapper objectMapper;
|
private final ObjectMapper objectMapper;
|
||||||
private final OssStorageService ossStorageService;
|
private final TransientPayloadStorageService transientPayloadStorageService;
|
||||||
|
|
||||||
@Transactional
|
@Transactional
|
||||||
public void saveParsedPayload(Long taskId, Map<String, DeleteBrandParsedFileCacheDto> parsedPayloadByScope) {
|
public void saveParsedPayload(Long taskId, Map<String, DeleteBrandParsedFileCacheDto> parsedPayloadByScope) {
|
||||||
@@ -81,7 +80,7 @@ public class DeleteBrandTaskStorageService {
|
|||||||
if (existing == null) {
|
if (existing == null) {
|
||||||
throw new BusinessException("failed to save delete-brand parsed payload");
|
throw new BusinessException("failed to save delete-brand parsed payload");
|
||||||
}
|
}
|
||||||
deleteParsedPayloadIfNeeded(existing.getParsedPayloadJson(), parsedPayloadPointer);
|
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(existing.getParsedPayloadJson(), parsedPayloadPointer);
|
||||||
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
|
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
|
||||||
.eq(TaskScopeStateEntity::getId, existing.getId())
|
.eq(TaskScopeStateEntity::getId, existing.getId())
|
||||||
.set(TaskScopeStateEntity::getScopeKey, scopeKey)
|
.set(TaskScopeStateEntity::getScopeKey, scopeKey)
|
||||||
@@ -120,6 +119,28 @@ public class DeleteBrandTaskStorageService {
|
|||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public int countParsedPayloadScopes(Long taskId) {
|
||||||
|
if (taskId == null || taskId <= 0) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
Long count = taskScopeStateMapper.selectCount(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||||
|
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
|
||||||
|
.isNotNull(TaskScopeStateEntity::getParsedPayloadJson));
|
||||||
|
return count == null ? 0 : count.intValue();
|
||||||
|
}
|
||||||
|
|
||||||
|
public int countCompletedScopes(Long taskId) {
|
||||||
|
if (taskId == null || taskId <= 0) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
Long count = taskScopeStateMapper.selectCount(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||||
|
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
|
||||||
|
.eq(TaskScopeStateEntity::getCompleted, 1));
|
||||||
|
return count == null ? 0 : count.intValue();
|
||||||
|
}
|
||||||
|
|
||||||
@Transactional
|
@Transactional
|
||||||
public boolean storeResultChunkIfChanged(Long taskId, String scopeKey, Integer chunkIndex, DeleteBrandResultFileDto chunk) {
|
public boolean storeResultChunkIfChanged(Long taskId, String scopeKey, Integer chunkIndex, DeleteBrandResultFileDto chunk) {
|
||||||
if (taskId == null || taskId <= 0) {
|
if (taskId == null || taskId <= 0) {
|
||||||
@@ -145,6 +166,7 @@ public class DeleteBrandTaskStorageService {
|
|||||||
.last("limit 1"));
|
.last("limit 1"));
|
||||||
boolean changed = true;
|
boolean changed = true;
|
||||||
if (existingChunk == null) {
|
if (existingChunk == null) {
|
||||||
|
String storedPayload = transientPayloadStorageService.storeChunkPayload(MODULE_TYPE, taskId, scopeHash, chunkIndex, payloadJson);
|
||||||
TaskChunkEntity entity = new TaskChunkEntity();
|
TaskChunkEntity entity = new TaskChunkEntity();
|
||||||
entity.setTaskId(taskId);
|
entity.setTaskId(taskId);
|
||||||
entity.setModuleType(MODULE_TYPE);
|
entity.setModuleType(MODULE_TYPE);
|
||||||
@@ -152,7 +174,7 @@ public class DeleteBrandTaskStorageService {
|
|||||||
entity.setScopeHash(scopeHash);
|
entity.setScopeHash(scopeHash);
|
||||||
entity.setChunkIndex(chunkIndex);
|
entity.setChunkIndex(chunkIndex);
|
||||||
entity.setChunkTotal(chunk.getChunkTotal() == null ? 0 : chunk.getChunkTotal());
|
entity.setChunkTotal(chunk.getChunkTotal() == null ? 0 : chunk.getChunkTotal());
|
||||||
entity.setPayloadJson(payloadJson);
|
entity.setPayloadJson(storedPayload);
|
||||||
entity.setPayloadHash(payloadHash);
|
entity.setPayloadHash(payloadHash);
|
||||||
entity.setCreatedAt(now);
|
entity.setCreatedAt(now);
|
||||||
entity.setUpdatedAt(now);
|
entity.setUpdatedAt(now);
|
||||||
@@ -169,11 +191,13 @@ public class DeleteBrandTaskStorageService {
|
|||||||
changed = !payloadHash.equals(concurrentChunk.getPayloadHash());
|
changed = !payloadHash.equals(concurrentChunk.getPayloadHash());
|
||||||
concurrentChunk.setScopeKey(normalizedScopeKey);
|
concurrentChunk.setScopeKey(normalizedScopeKey);
|
||||||
concurrentChunk.setChunkTotal(chunk.getChunkTotal() == null ? 0 : chunk.getChunkTotal());
|
concurrentChunk.setChunkTotal(chunk.getChunkTotal() == null ? 0 : chunk.getChunkTotal());
|
||||||
concurrentChunk.setPayloadJson(payloadJson);
|
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(concurrentChunk.getPayloadJson(), storedPayload);
|
||||||
|
concurrentChunk.setPayloadJson(storedPayload);
|
||||||
concurrentChunk.setPayloadHash(payloadHash);
|
concurrentChunk.setPayloadHash(payloadHash);
|
||||||
concurrentChunk.setUpdatedAt(now);
|
concurrentChunk.setUpdatedAt(now);
|
||||||
taskChunkMapper.updateById(concurrentChunk);
|
taskChunkMapper.updateById(concurrentChunk);
|
||||||
} else {
|
} else {
|
||||||
|
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
|
||||||
throw ex;
|
throw ex;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -181,7 +205,9 @@ public class DeleteBrandTaskStorageService {
|
|||||||
changed = !payloadHash.equals(existingChunk.getPayloadHash());
|
changed = !payloadHash.equals(existingChunk.getPayloadHash());
|
||||||
existingChunk.setScopeKey(normalizedScopeKey);
|
existingChunk.setScopeKey(normalizedScopeKey);
|
||||||
existingChunk.setChunkTotal(chunk.getChunkTotal() == null ? 0 : chunk.getChunkTotal());
|
existingChunk.setChunkTotal(chunk.getChunkTotal() == null ? 0 : chunk.getChunkTotal());
|
||||||
existingChunk.setPayloadJson(payloadJson);
|
String storedPayload = transientPayloadStorageService.storeChunkPayload(MODULE_TYPE, taskId, scopeHash, chunkIndex, payloadJson);
|
||||||
|
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(existingChunk.getPayloadJson(), storedPayload);
|
||||||
|
existingChunk.setPayloadJson(storedPayload);
|
||||||
existingChunk.setPayloadHash(payloadHash);
|
existingChunk.setPayloadHash(payloadHash);
|
||||||
existingChunk.setUpdatedAt(now);
|
existingChunk.setUpdatedAt(now);
|
||||||
taskChunkMapper.updateById(existingChunk);
|
taskChunkMapper.updateById(existingChunk);
|
||||||
@@ -210,7 +236,8 @@ public class DeleteBrandTaskStorageService {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
DeleteBrandResultFileDto dto = objectMapper.readValue(entity.getPayloadJson(), DeleteBrandResultFileDto.class);
|
String payloadJson = transientPayloadStorageService.resolvePayload(entity.getPayloadJson(), "failed to load delete-brand result chunks");
|
||||||
|
DeleteBrandResultFileDto dto = objectMapper.readValue(payloadJson, DeleteBrandResultFileDto.class);
|
||||||
grouped.computeIfAbsent(entity.getScopeKey(), ignored -> new ArrayList<>()).add(dto);
|
grouped.computeIfAbsent(entity.getScopeKey(), ignored -> new ArrayList<>()).add(dto);
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
throw new BusinessException("failed to load delete-brand result chunks");
|
throw new BusinessException("failed to load delete-brand result chunks");
|
||||||
@@ -228,12 +255,22 @@ public class DeleteBrandTaskStorageService {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||||
.select(TaskScopeStateEntity::getParsedPayloadJson)
|
.select(TaskScopeStateEntity::getParsedPayloadJson, TaskScopeStateEntity::getStateJson)
|
||||||
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
.eq(TaskScopeStateEntity::getTaskId, taskId)
|
||||||
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
|
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
|
||||||
if (states != null) {
|
if (states != null) {
|
||||||
for (TaskScopeStateEntity state : states) {
|
for (TaskScopeStateEntity state : states) {
|
||||||
deleteParsedPayloadIfNeeded(state.getParsedPayloadJson(), null);
|
transientPayloadStorageService.deletePayloadIfPresent(state.getParsedPayloadJson());
|
||||||
|
transientPayloadStorageService.deletePayloadIfPresent(state.getStateJson());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||||
|
.select(TaskChunkEntity::getPayloadJson)
|
||||||
|
.eq(TaskChunkEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
|
||||||
|
if (chunks != null) {
|
||||||
|
for (TaskChunkEntity chunk : chunks) {
|
||||||
|
transientPayloadStorageService.deletePayloadIfPresent(chunk.getPayloadJson());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
taskChunkMapper.delete(new LambdaQueryWrapper<TaskChunkEntity>()
|
taskChunkMapper.delete(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||||
@@ -322,56 +359,17 @@ public class DeleteBrandTaskStorageService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private String storeParsedPayload(Long taskId, String scopeHash, String payloadJson) {
|
private String storeParsedPayload(Long taskId, String scopeHash, String payloadJson) {
|
||||||
try {
|
return transientPayloadStorageService.storeParsedPayload(MODULE_TYPE, taskId, scopeHash, payloadJson, true);
|
||||||
String pointer = OSS_POINTER_PREFIX + ossStorageService.uploadTaskParsedPayload(MODULE_TYPE, taskId, scopeHash, payloadJson);
|
|
||||||
return objectMapper.writeValueAsString(pointer);
|
|
||||||
} catch (Exception ex) {
|
|
||||||
throw new BusinessException("failed to save delete-brand parsed payload");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private String resolveParsedPayload(String value) {
|
private String resolveParsedPayload(String value) {
|
||||||
if (isBlank(value)) {
|
|
||||||
return value;
|
|
||||||
}
|
|
||||||
String ossPointer = extractOssPointer(value);
|
|
||||||
if (ossPointer == null) {
|
|
||||||
return value;
|
|
||||||
}
|
|
||||||
try {
|
try {
|
||||||
return ossStorageService.readObjectAsString(ossPointer.substring(OSS_POINTER_PREFIX.length()));
|
return transientPayloadStorageService.resolvePayload(value, "failed to load delete-brand parsed payload");
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
throw new BusinessException("failed to load delete-brand parsed payload");
|
throw new BusinessException("failed to load delete-brand parsed payload");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void deleteParsedPayloadIfNeeded(String oldValue, String newValue) {
|
|
||||||
String oldPointer = extractOssPointer(oldValue);
|
|
||||||
String newPointer = extractOssPointer(newValue);
|
|
||||||
if (oldPointer == null || oldPointer.equals(newPointer)) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
ossStorageService.deleteObject(oldPointer.substring(OSS_POINTER_PREFIX.length()));
|
|
||||||
} catch (Exception ignored) {
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private String extractOssPointer(String value) {
|
|
||||||
if (isBlank(value)) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
if (value.startsWith(OSS_POINTER_PREFIX)) {
|
|
||||||
return value;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
String decoded = objectMapper.readValue(value, String.class);
|
|
||||||
return decoded != null && decoded.startsWith(OSS_POINTER_PREFIX) ? decoded : null;
|
|
||||||
} catch (Exception ignored) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private String writeJson(Object value, String message) {
|
private String writeJson(Object value, String message) {
|
||||||
try {
|
try {
|
||||||
return objectMapper.writeValueAsString(value);
|
return objectMapper.writeValueAsString(value);
|
||||||
|
|||||||
@@ -0,0 +1,85 @@
|
|||||||
|
package com.nanri.aiimage.modules.file.service.object;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.config.TransientStorageProperties;
|
||||||
|
import io.minio.GetObjectArgs;
|
||||||
|
import io.minio.MinioClient;
|
||||||
|
import io.minio.PutObjectArgs;
|
||||||
|
import io.minio.RemoveObjectArgs;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
|
import java.io.ByteArrayInputStream;
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.util.Objects;
|
||||||
|
|
||||||
|
@Service
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
public class RustfsObjectStorageService {
|
||||||
|
|
||||||
|
private final TransientStorageProperties properties;
|
||||||
|
|
||||||
|
public boolean isConfigured() {
|
||||||
|
return notBlank(properties.getEndpoint())
|
||||||
|
&& notBlank(properties.getBucket())
|
||||||
|
&& notBlank(properties.getAccessKeyId())
|
||||||
|
&& notBlank(properties.getAccessKeySecret());
|
||||||
|
}
|
||||||
|
|
||||||
|
public String uploadText(String objectKey, String content) {
|
||||||
|
if (!isConfigured()) {
|
||||||
|
throw new IllegalStateException("transient storage is not configured");
|
||||||
|
}
|
||||||
|
try (ByteArrayInputStream stream = new ByteArrayInputStream(
|
||||||
|
Objects.requireNonNullElse(content, "").getBytes(StandardCharsets.UTF_8))) {
|
||||||
|
buildClient().putObject(PutObjectArgs.builder()
|
||||||
|
.bucket(properties.getBucket())
|
||||||
|
.object(objectKey)
|
||||||
|
.stream(stream, stream.available(), -1)
|
||||||
|
.contentType("application/json")
|
||||||
|
.build());
|
||||||
|
return objectKey;
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new IllegalStateException("failed to upload payload to transient storage", ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public String readObjectAsString(String objectKey) {
|
||||||
|
if (!isConfigured()) {
|
||||||
|
throw new IllegalStateException("transient storage is not configured");
|
||||||
|
}
|
||||||
|
try (var stream = buildClient().getObject(GetObjectArgs.builder()
|
||||||
|
.bucket(properties.getBucket())
|
||||||
|
.object(objectKey)
|
||||||
|
.build())) {
|
||||||
|
return new String(stream.readAllBytes(), StandardCharsets.UTF_8);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new IllegalStateException("failed to read payload from transient storage", ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void deleteObject(String objectKey) {
|
||||||
|
if (!isConfigured() || !notBlank(objectKey)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
buildClient().removeObject(RemoveObjectArgs.builder()
|
||||||
|
.bucket(properties.getBucket())
|
||||||
|
.object(objectKey)
|
||||||
|
.build());
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new IllegalStateException("failed to delete payload from transient storage", ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private MinioClient buildClient() {
|
||||||
|
return MinioClient.builder()
|
||||||
|
.endpoint(properties.getEndpoint())
|
||||||
|
.credentials(properties.getAccessKeyId(), properties.getAccessKeySecret())
|
||||||
|
.region(properties.getRegion())
|
||||||
|
.build();
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean notBlank(String value) {
|
||||||
|
return value != null && !value.isBlank();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -43,6 +43,10 @@ public class PatrolDeleteResultItemVo {
|
|||||||
private LocalDateTime finishedAt;
|
private LocalDateTime finishedAt;
|
||||||
private String outputFilename;
|
private String outputFilename;
|
||||||
private String downloadUrl;
|
private String downloadUrl;
|
||||||
|
private Long fileJobId;
|
||||||
|
private String fileStatus;
|
||||||
|
private String fileError;
|
||||||
|
private Boolean fileReady;
|
||||||
|
|
||||||
@JsonProperty("countrySections")
|
@JsonProperty("countrySections")
|
||||||
private List<PatrolDeleteCountrySectionDto> countrySections = new ArrayList<>();
|
private List<PatrolDeleteCountrySectionDto> countrySections = new ArrayList<>();
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import com.nanri.aiimage.modules.patroldelete.model.dto.PatrolDeleteShopPayloadD
|
|||||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskScopePayloadStorageService;
|
import com.nanri.aiimage.modules.task.service.TaskScopePayloadStorageService;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
@@ -13,11 +14,13 @@ import java.time.Duration;
|
|||||||
import java.time.Instant;
|
import java.time.Instant;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.LinkedHashMap;
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.concurrent.ConcurrentHashMap;
|
import java.util.concurrent.ConcurrentHashMap;
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
|
@Slf4j
|
||||||
public class PatrolDeleteTaskCacheService {
|
public class PatrolDeleteTaskCacheService {
|
||||||
|
|
||||||
private static final String MODULE_TYPE = "PATROL_DELETE";
|
private static final String MODULE_TYPE = "PATROL_DELETE";
|
||||||
@@ -60,7 +63,13 @@ public class PatrolDeleteTaskCacheService {
|
|||||||
if (taskId == null || taskId <= 0) {
|
if (taskId == null || taskId <= 0) {
|
||||||
return 0L;
|
return 0L;
|
||||||
}
|
}
|
||||||
String raw = stringRedisTemplate.opsForValue().get(buildTaskHeartbeatKey(taskId));
|
String raw;
|
||||||
|
try {
|
||||||
|
raw = stringRedisTemplate.opsForValue().get(buildTaskHeartbeatKey(taskId));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[patrol-delete-cache] get heartbeat degraded taskId={} msg={}", taskId, ex.getMessage());
|
||||||
|
return 0L;
|
||||||
|
}
|
||||||
if (raw == null || raw.isBlank()) {
|
if (raw == null || raw.isBlank()) {
|
||||||
return 0L;
|
return 0L;
|
||||||
}
|
}
|
||||||
@@ -71,11 +80,50 @@ public class PatrolDeleteTaskCacheService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public Map<Long, Long> getTaskHeartbeatMillisBatch(List<Long> taskIds) {
|
||||||
|
Map<Long, Long> result = new LinkedHashMap<>();
|
||||||
|
if (taskIds == null || taskIds.isEmpty()) {
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
List<Long> normalized = taskIds.stream()
|
||||||
|
.filter(id -> id != null && id > 0)
|
||||||
|
.distinct()
|
||||||
|
.toList();
|
||||||
|
if (normalized.isEmpty()) {
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
List<String> keys = normalized.stream().map(this::buildTaskHeartbeatKey).toList();
|
||||||
|
List<String> values;
|
||||||
|
try {
|
||||||
|
values = stringRedisTemplate.opsForValue().multiGet(keys);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[patrol-delete-cache] batch get heartbeat degraded taskIds={} msg={}", normalized, ex.getMessage());
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
for (int i = 0; i < normalized.size(); i++) {
|
||||||
|
String raw = values != null && i < values.size() ? values.get(i) : null;
|
||||||
|
if (raw == null || raw.isBlank()) {
|
||||||
|
result.put(normalized.get(i), 0L);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
result.put(normalized.get(i), Long.parseLong(raw));
|
||||||
|
} catch (NumberFormatException ignored) {
|
||||||
|
result.put(normalized.get(i), 0L);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
public void touchTaskHeartbeat(Long taskId) {
|
public void touchTaskHeartbeat(Long taskId) {
|
||||||
stringRedisTemplate.opsForValue().set(
|
try {
|
||||||
buildTaskHeartbeatKey(taskId),
|
stringRedisTemplate.opsForValue().set(
|
||||||
String.valueOf(Instant.now().toEpochMilli()),
|
buildTaskHeartbeatKey(taskId),
|
||||||
Duration.ofHours(PAYLOAD_TTL_HOURS));
|
String.valueOf(Instant.now().toEpochMilli()),
|
||||||
|
Duration.ofHours(PAYLOAD_TTL_HOURS));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[patrol-delete-cache] touch heartbeat degraded taskId={} msg={}", taskId, ex.getMessage());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public void deleteTaskCache(Long taskId) {
|
public void deleteTaskCache(Long taskId) {
|
||||||
@@ -83,8 +131,12 @@ public class PatrolDeleteTaskCacheService {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
taskEntityLocalCache.remove(taskId);
|
taskEntityLocalCache.remove(taskId);
|
||||||
stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId));
|
try {
|
||||||
stringRedisTemplate.delete(buildTaskEntityKey(taskId));
|
stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId));
|
||||||
|
stringRedisTemplate.delete(buildTaskEntityKey(taskId));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[patrol-delete-cache] delete cache degraded taskId={} msg={}", taskId, ex.getMessage());
|
||||||
|
}
|
||||||
taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE);
|
taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -133,7 +185,13 @@ public class PatrolDeleteTaskCacheService {
|
|||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
java.util.List<String> keys = missingIds.stream().map(this::buildTaskEntityKey).toList();
|
java.util.List<String> keys = missingIds.stream().map(this::buildTaskEntityKey).toList();
|
||||||
java.util.List<String> values = stringRedisTemplate.opsForValue().multiGet(keys);
|
java.util.List<String> values;
|
||||||
|
try {
|
||||||
|
values = stringRedisTemplate.opsForValue().multiGet(keys);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[patrol-delete-cache] batch get task cache degraded taskIds={} msg={}", missingIds, ex.getMessage());
|
||||||
|
return result;
|
||||||
|
}
|
||||||
for (int i = 0; i < missingIds.size(); i++) {
|
for (int i = 0; i < missingIds.size(); i++) {
|
||||||
Long taskId = missingIds.get(i);
|
Long taskId = missingIds.get(i);
|
||||||
String val = values != null && i < values.size() ? values.get(i) : null;
|
String val = values != null && i < values.size() ? values.get(i) : null;
|
||||||
|
|||||||
@@ -24,6 +24,10 @@ import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
|||||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskResultItemService;
|
||||||
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
|
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
@@ -45,6 +49,9 @@ public class PatrolDeleteTaskService {
|
|||||||
|
|
||||||
private static final String MODULE_TYPE = "PATROL_DELETE";
|
private static final String MODULE_TYPE = "PATROL_DELETE";
|
||||||
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
||||||
|
private static final int RESULT_PENDING = -1;
|
||||||
|
private static final int RESULT_FAILED = 0;
|
||||||
|
private static final int RESULT_SUCCESS = 1;
|
||||||
private static final String INTERRUPTED_MESSAGE = "Python 在该店铺结果提交完成前中断";
|
private static final String INTERRUPTED_MESSAGE = "Python 在该店铺结果提交完成前中断";
|
||||||
|
|
||||||
private final FileTaskMapper fileTaskMapper;
|
private final FileTaskMapper fileTaskMapper;
|
||||||
@@ -56,6 +63,9 @@ public class PatrolDeleteTaskService {
|
|||||||
private final ZiniaoShopSwitchService ziniaoShopSwitchService;
|
private final ZiniaoShopSwitchService ziniaoShopSwitchService;
|
||||||
private final ObjectMapper objectMapper;
|
private final ObjectMapper objectMapper;
|
||||||
private final TaskPressureProperties taskPressureProperties;
|
private final TaskPressureProperties taskPressureProperties;
|
||||||
|
private final TaskFileJobService taskFileJobService;
|
||||||
|
private final TaskResultItemService taskResultItemService;
|
||||||
|
private final TaskProgressSnapshotService taskProgressSnapshotService;
|
||||||
|
|
||||||
private FileTaskEntity loadTaskForExecution(Long taskId) {
|
private FileTaskEntity loadTaskForExecution(Long taskId) {
|
||||||
Map<Long, FileTaskEntity> cachedTasks = taskCacheService.getTaskCacheBatch(List.of(taskId));
|
Map<Long, FileTaskEntity> cachedTasks = taskCacheService.getTaskCacheBatch(List.of(taskId));
|
||||||
@@ -185,8 +195,7 @@ public class PatrolDeleteTaskService {
|
|||||||
batch.getMissingTaskIds().add(taskId);
|
batch.getMissingTaskIds().add(taskId);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
List<PatrolDeleteResultItemVo> snapshots = buildSnapshotFromDb(task, taskRows);
|
batch.getItems().addAll(buildProgressItems(task, taskRows));
|
||||||
batch.getItems().addAll(snapshots);
|
|
||||||
}
|
}
|
||||||
return batch;
|
return batch;
|
||||||
}
|
}
|
||||||
@@ -258,12 +267,13 @@ public class PatrolDeleteTaskService {
|
|||||||
result.setSourceFilename(normalizedShopName);
|
result.setSourceFilename(normalizedShopName);
|
||||||
result.setSourceFileUrl(item.getShopId());
|
result.setSourceFileUrl(item.getShopId());
|
||||||
result.setUserId(request.getUserId());
|
result.setUserId(request.getUserId());
|
||||||
result.setSuccess(null);
|
result.setSuccess(RESULT_PENDING);
|
||||||
result.setCreatedAt(now);
|
result.setCreatedAt(now);
|
||||||
fileResultMapper.insert(result);
|
fileResultMapper.insert(result);
|
||||||
snapshots.add(toSnapshotVo(result, item, task.getStatus(), null));
|
snapshots.add(toSnapshotVo(result, item, task.getStatus(), null));
|
||||||
}
|
}
|
||||||
persistTaskJson(task, uniqueItems, snapshots);
|
persistTaskJson(task, uniqueItems, snapshots);
|
||||||
|
taskCacheService.saveTaskCache(task);
|
||||||
|
|
||||||
PatrolDeleteCreateTaskVo vo = new PatrolDeleteCreateTaskVo();
|
PatrolDeleteCreateTaskVo vo = new PatrolDeleteCreateTaskVo();
|
||||||
vo.setTaskId(task.getId());
|
vo.setTaskId(task.getId());
|
||||||
@@ -320,6 +330,7 @@ public class PatrolDeleteTaskService {
|
|||||||
|
|
||||||
persistSnapshotJson(task, snapshots);
|
persistSnapshotJson(task, snapshots);
|
||||||
fileTaskMapper.updateById(task);
|
fileTaskMapper.updateById(task);
|
||||||
|
taskCacheService.saveTaskCache(task);
|
||||||
tryFinalizeTask(taskId, false);
|
tryFinalizeTask(taskId, false);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -396,6 +407,7 @@ public class PatrolDeleteTaskService {
|
|||||||
|
|
||||||
persistSnapshotJson(task, snapshots);
|
persistSnapshotJson(task, snapshots);
|
||||||
fileTaskMapper.updateById(task);
|
fileTaskMapper.updateById(task);
|
||||||
|
taskCacheService.saveTaskCache(task);
|
||||||
return changed;
|
return changed;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -442,6 +454,7 @@ public class PatrolDeleteTaskService {
|
|||||||
updateTaskStatusFromRows(task, rows);
|
updateTaskStatusFromRows(task, rows);
|
||||||
persistSnapshotJson(task, buildSnapshotFromDb(task, rows));
|
persistSnapshotJson(task, buildSnapshotFromDb(task, rows));
|
||||||
fileTaskMapper.updateById(task);
|
fileTaskMapper.updateById(task);
|
||||||
|
taskCacheService.saveTaskCache(task);
|
||||||
}
|
}
|
||||||
|
|
||||||
private long countTasks(Long userId, List<String> statuses) {
|
private long countTasks(Long userId, List<String> statuses) {
|
||||||
@@ -496,7 +509,11 @@ public class PatrolDeleteTaskService {
|
|||||||
private Map<Long, Map<Long, PatrolDeleteResultItemVo>> buildSnapshotMap(Map<Long, FileTaskEntity> taskMap) {
|
private Map<Long, Map<Long, PatrolDeleteResultItemVo>> buildSnapshotMap(Map<Long, FileTaskEntity> taskMap) {
|
||||||
Map<Long, Map<Long, PatrolDeleteResultItemVo>> out = new LinkedHashMap<>();
|
Map<Long, Map<Long, PatrolDeleteResultItemVo>> out = new LinkedHashMap<>();
|
||||||
for (Map.Entry<Long, FileTaskEntity> entry : taskMap.entrySet()) {
|
for (Map.Entry<Long, FileTaskEntity> entry : taskMap.entrySet()) {
|
||||||
out.put(entry.getKey(), indexSnapshotByResultId(parseTaskSnapshots(entry.getValue().getResultJson())));
|
List<PatrolDeleteResultItemVo> snapshots = parseTaskSnapshots(entry.getValue().getResultJson());
|
||||||
|
if (snapshots.isEmpty()) {
|
||||||
|
snapshots = taskResultItemService.listResultSnapshots(entry.getKey(), MODULE_TYPE, PatrolDeleteResultItemVo.class);
|
||||||
|
}
|
||||||
|
out.put(entry.getKey(), indexSnapshotByResultId(snapshots));
|
||||||
}
|
}
|
||||||
return out;
|
return out;
|
||||||
}
|
}
|
||||||
@@ -508,14 +525,13 @@ public class PatrolDeleteTaskService {
|
|||||||
item.setShopName(firstNonBlank(item.getShopName(), entity.getSourceFilename()));
|
item.setShopName(firstNonBlank(item.getShopName(), entity.getSourceFilename()));
|
||||||
item.setShopId(firstNonBlank(item.getShopId(), entity.getSourceFileUrl()));
|
item.setShopId(firstNonBlank(item.getShopId(), entity.getSourceFileUrl()));
|
||||||
item.setTaskStatus(task != null ? task.getStatus() : item.getTaskStatus());
|
item.setTaskStatus(task != null ? task.getStatus() : item.getTaskStatus());
|
||||||
item.setSuccess(entity.getSuccess() == null ? item.getSuccess() : entity.getSuccess() == 1);
|
item.setSuccess(toSuccessFlag(entity.getSuccess(), item.getSuccess()));
|
||||||
item.setError(!blank(entity.getErrorMessage()) ? entity.getErrorMessage() : item.getError());
|
item.setError(!blank(entity.getErrorMessage()) ? entity.getErrorMessage() : item.getError());
|
||||||
item.setCreatedAt(entity.getCreatedAt());
|
item.setCreatedAt(entity.getCreatedAt());
|
||||||
item.setFinishedAt(task != null ? task.getFinishedAt() : item.getFinishedAt());
|
item.setFinishedAt(task != null ? task.getFinishedAt() : item.getFinishedAt());
|
||||||
item.setOutputFilename(firstNonBlank(item.getOutputFilename(), entity.getResultFilename()));
|
item.setOutputFilename(firstNonBlank(item.getOutputFilename(), entity.getResultFilename()));
|
||||||
item.setDownloadUrl(blank(entity.getResultFileUrl())
|
item.setDownloadUrl(null);
|
||||||
? item.getDownloadUrl()
|
attachFileJobState(item, entity);
|
||||||
: ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl()));
|
|
||||||
if (item.getCountrySections() == null) {
|
if (item.getCountrySections() == null) {
|
||||||
item.setCountrySections(new ArrayList<>());
|
item.setCountrySections(new ArrayList<>());
|
||||||
}
|
}
|
||||||
@@ -525,6 +541,18 @@ public class PatrolDeleteTaskService {
|
|||||||
return item;
|
return item;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void attachFileJobState(PatrolDeleteResultItemVo item, FileResultEntity entity) {
|
||||||
|
TaskFileJobEntity job = taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId());
|
||||||
|
item.setFileReady(!blank(entity.getResultFileUrl()));
|
||||||
|
if (job == null) {
|
||||||
|
item.setFileStatus(Boolean.TRUE.equals(item.getFileReady()) ? "SUCCESS" : null);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
item.setFileJobId(job.getId());
|
||||||
|
item.setFileStatus(job.getStatus());
|
||||||
|
item.setFileError(job.getErrorMessage());
|
||||||
|
}
|
||||||
|
|
||||||
private List<PatrolDeleteTaskItemDto> dedupeItems(List<PatrolDeleteTaskItemDto> items) {
|
private List<PatrolDeleteTaskItemDto> dedupeItems(List<PatrolDeleteTaskItemDto> items) {
|
||||||
LinkedHashMap<String, PatrolDeleteTaskItemDto> map = new LinkedHashMap<>();
|
LinkedHashMap<String, PatrolDeleteTaskItemDto> map = new LinkedHashMap<>();
|
||||||
for (PatrolDeleteTaskItemDto item : items) {
|
for (PatrolDeleteTaskItemDto item : items) {
|
||||||
@@ -554,7 +582,7 @@ public class PatrolDeleteTaskService {
|
|||||||
vo.setMatchStatus(item.getMatchStatus());
|
vo.setMatchStatus(item.getMatchStatus());
|
||||||
vo.setMatchMessage(item.getMatchMessage());
|
vo.setMatchMessage(item.getMatchMessage());
|
||||||
vo.setTaskStatus(taskStatus);
|
vo.setTaskStatus(taskStatus);
|
||||||
vo.setSuccess(result.getSuccess() == null ? null : result.getSuccess() == 1);
|
vo.setSuccess(toSuccessFlag(result.getSuccess(), null));
|
||||||
vo.setError(result.getErrorMessage());
|
vo.setError(result.getErrorMessage());
|
||||||
vo.setCreatedAt(result.getCreatedAt());
|
vo.setCreatedAt(result.getCreatedAt());
|
||||||
vo.setFinishedAt(finishedAt);
|
vo.setFinishedAt(finishedAt);
|
||||||
@@ -569,6 +597,7 @@ public class PatrolDeleteTaskService {
|
|||||||
try {
|
try {
|
||||||
task.setRequestJson(objectMapper.writeValueAsString(requestItems));
|
task.setRequestJson(objectMapper.writeValueAsString(requestItems));
|
||||||
task.setResultJson(objectMapper.writeValueAsString(snapshots));
|
task.setResultJson(objectMapper.writeValueAsString(snapshots));
|
||||||
|
syncSnapshotTables(task, snapshots);
|
||||||
fileTaskMapper.updateById(task);
|
fileTaskMapper.updateById(task);
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
throw new BusinessException("巡店删除任务快照保存失败");
|
throw new BusinessException("巡店删除任务快照保存失败");
|
||||||
@@ -585,6 +614,27 @@ public class PatrolDeleteTaskService {
|
|||||||
return list;
|
return list;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private List<PatrolDeleteResultItemVo> buildProgressItems(FileTaskEntity task, List<FileResultEntity> rows) {
|
||||||
|
List<PatrolDeleteResultItemVo> list = new ArrayList<>();
|
||||||
|
for (FileResultEntity row : rows) {
|
||||||
|
PatrolDeleteResultItemVo item = new PatrolDeleteResultItemVo();
|
||||||
|
item.setResultId(row.getId());
|
||||||
|
item.setTaskId(row.getTaskId());
|
||||||
|
item.setShopName(row.getSourceFilename());
|
||||||
|
item.setShopId(row.getSourceFileUrl());
|
||||||
|
item.setTaskStatus(task == null ? null : task.getStatus());
|
||||||
|
item.setSuccess(toSuccessFlag(row.getSuccess(), null));
|
||||||
|
item.setError(row.getErrorMessage());
|
||||||
|
item.setCreatedAt(row.getCreatedAt());
|
||||||
|
item.setFinishedAt(task == null ? null : task.getFinishedAt());
|
||||||
|
item.setOutputFilename(row.getResultFilename());
|
||||||
|
item.setDownloadUrl(null);
|
||||||
|
attachFileJobState(item, row);
|
||||||
|
list.add(item);
|
||||||
|
}
|
||||||
|
return list;
|
||||||
|
}
|
||||||
|
|
||||||
private Map<Long, PatrolDeleteResultItemVo> indexSnapshotByResultId(List<PatrolDeleteResultItemVo> snapshots) {
|
private Map<Long, PatrolDeleteResultItemVo> indexSnapshotByResultId(List<PatrolDeleteResultItemVo> snapshots) {
|
||||||
Map<Long, PatrolDeleteResultItemVo> map = new LinkedHashMap<>();
|
Map<Long, PatrolDeleteResultItemVo> map = new LinkedHashMap<>();
|
||||||
if (snapshots == null) {
|
if (snapshots == null) {
|
||||||
@@ -599,8 +649,8 @@ public class PatrolDeleteTaskService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private void updateTaskStatusFromRows(FileTaskEntity task, List<FileResultEntity> rows) {
|
private void updateTaskStatusFromRows(FileTaskEntity task, List<FileResultEntity> rows) {
|
||||||
long successCount = rows.stream().filter(row -> Integer.valueOf(1).equals(row.getSuccess())).count();
|
long successCount = rows.stream().filter(row -> Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess())).count();
|
||||||
long failedCount = rows.stream().filter(row -> Integer.valueOf(0).equals(row.getSuccess())).count();
|
long failedCount = rows.stream().filter(row -> Integer.valueOf(RESULT_FAILED).equals(row.getSuccess())).count();
|
||||||
long pendingCount = rows.stream().filter(row -> !isResultFinished(row)).count();
|
long pendingCount = rows.stream().filter(row -> !isResultFinished(row)).count();
|
||||||
task.setSuccessFileCount((int) successCount);
|
task.setSuccessFileCount((int) successCount);
|
||||||
task.setFailedFileCount((int) failedCount);
|
task.setFailedFileCount((int) failedCount);
|
||||||
@@ -753,27 +803,25 @@ public class PatrolDeleteTaskService {
|
|||||||
.filter(item -> Boolean.TRUE.equals(item.getSuccess()))
|
.filter(item -> Boolean.TRUE.equals(item.getSuccess()))
|
||||||
.toList();
|
.toList();
|
||||||
if (!successItems.isEmpty()) {
|
if (!successItems.isEmpty()) {
|
||||||
File workRoot = FileUtil.mkdir(FileUtil.file(System.getProperty("java.io.tmpdir"), "patrol-delete-result", String.valueOf(task.getId())));
|
String filename = buildTaskWorkbookFilename(task);
|
||||||
String filename = safeFileStem("巡店删除-" + task.getId()) + ".xlsx";
|
int rowCount = excelAssemblyService.countRows(successItems);
|
||||||
File xlsx = FileUtil.file(workRoot, filename);
|
FileResultEntity firstSuccessRow = null;
|
||||||
try {
|
|
||||||
excelAssemblyService.writeWorkbook(xlsx, successItems);
|
|
||||||
String objectKey = ossStorageService.uploadResultFile(xlsx, MODULE_TYPE);
|
|
||||||
long fileSize = xlsx.length();
|
|
||||||
int rowCount = excelAssemblyService.countRows(successItems);
|
|
||||||
for (FileResultEntity row : rows) {
|
for (FileResultEntity row : rows) {
|
||||||
if (Integer.valueOf(1).equals(row.getSuccess())) {
|
if (Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess())) {
|
||||||
row.setResultFilename(filename);
|
row.setResultFilename(filename);
|
||||||
row.setResultFileUrl(objectKey);
|
row.setResultFileUrl(null);
|
||||||
row.setResultFileSize(fileSize);
|
row.setResultFileSize(0L);
|
||||||
row.setResultContentType(CONTENT_TYPE_XLSX);
|
row.setResultContentType(CONTENT_TYPE_XLSX);
|
||||||
row.setRowCount(rowCount);
|
row.setRowCount(rowCount);
|
||||||
fileResultMapper.updateById(row);
|
fileResultMapper.updateById(row);
|
||||||
|
if (firstSuccessRow == null) {
|
||||||
|
firstSuccessRow = row;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if (firstSuccessRow != null) {
|
||||||
|
taskFileJobService.enqueueAssembleResult(task.getId(), MODULE_TYPE, firstSuccessRow.getId(), "task:" + task.getId());
|
||||||
snapshots = buildSnapshotFromDb(task, rows);
|
snapshots = buildSnapshotFromDb(task, rows);
|
||||||
} finally {
|
|
||||||
FileUtil.del(xlsx);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -782,14 +830,56 @@ public class PatrolDeleteTaskService {
|
|||||||
taskCacheService.deleteTaskCache(task.getId());
|
taskCacheService.deleteTaskCache(task.getId());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void processResultFileJob(TaskFileJobEntity job) {
|
||||||
|
if (job == null || job.getTaskId() == null) {
|
||||||
|
throw new BusinessException("结果文件任务参数不完整");
|
||||||
|
}
|
||||||
|
FileTaskEntity task = fileTaskMapper.selectById(job.getTaskId());
|
||||||
|
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
|
||||||
|
throw new BusinessException("任务不存在");
|
||||||
|
}
|
||||||
|
List<FileResultEntity> rows = listTaskRows(job.getTaskId());
|
||||||
|
List<PatrolDeleteResultItemVo> snapshots = taskResultItemService.listResultSnapshots(job.getTaskId(), MODULE_TYPE, PatrolDeleteResultItemVo.class);
|
||||||
|
if (snapshots.isEmpty()) {
|
||||||
|
snapshots = buildSnapshotFromDb(task, rows);
|
||||||
|
}
|
||||||
|
List<PatrolDeleteResultItemVo> successItems = snapshots.stream()
|
||||||
|
.filter(item -> Boolean.TRUE.equals(item.getSuccess()))
|
||||||
|
.toList();
|
||||||
|
if (successItems.isEmpty()) {
|
||||||
|
throw new BusinessException("没有可生成的巡店删除结果");
|
||||||
|
}
|
||||||
|
File workRoot = FileUtil.mkdir(FileUtil.file(System.getProperty("java.io.tmpdir"), "patrol-delete-result", String.valueOf(task.getId())));
|
||||||
|
String filename = buildTaskWorkbookFilename(task);
|
||||||
|
File xlsx = FileUtil.file(workRoot, filename);
|
||||||
|
try {
|
||||||
|
excelAssemblyService.writeWorkbook(xlsx, successItems);
|
||||||
|
String objectKey = ossStorageService.uploadResultFile(xlsx, MODULE_TYPE);
|
||||||
|
long fileSize = xlsx.length();
|
||||||
|
int rowCount = excelAssemblyService.countRows(successItems);
|
||||||
|
for (FileResultEntity row : rows) {
|
||||||
|
if (Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess())) {
|
||||||
|
row.setResultFilename(filename);
|
||||||
|
row.setResultFileUrl(objectKey);
|
||||||
|
row.setResultFileSize(fileSize);
|
||||||
|
row.setResultContentType(CONTENT_TYPE_XLSX);
|
||||||
|
row.setRowCount(rowCount);
|
||||||
|
fileResultMapper.updateById(row);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
FileUtil.del(xlsx);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private void markResultSuccess(FileResultEntity row) {
|
private void markResultSuccess(FileResultEntity row) {
|
||||||
row.setSuccess(1);
|
row.setSuccess(RESULT_SUCCESS);
|
||||||
row.setErrorMessage(null);
|
row.setErrorMessage(null);
|
||||||
fileResultMapper.updateById(row);
|
fileResultMapper.updateById(row);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void markResultFailed(FileResultEntity row, String message) {
|
private void markResultFailed(FileResultEntity row, String message) {
|
||||||
row.setSuccess(0);
|
row.setSuccess(RESULT_FAILED);
|
||||||
row.setErrorMessage(blankToNull(message));
|
row.setErrorMessage(blankToNull(message));
|
||||||
row.setResultFilename(null);
|
row.setResultFilename(null);
|
||||||
row.setResultFileUrl(null);
|
row.setResultFileUrl(null);
|
||||||
@@ -800,7 +890,14 @@ public class PatrolDeleteTaskService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private boolean isResultFinished(FileResultEntity row) {
|
private boolean isResultFinished(FileResultEntity row) {
|
||||||
return Integer.valueOf(1).equals(row.getSuccess()) || Integer.valueOf(0).equals(row.getSuccess());
|
return Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess()) || Integer.valueOf(RESULT_FAILED).equals(row.getSuccess());
|
||||||
|
}
|
||||||
|
|
||||||
|
private Boolean toSuccessFlag(Integer dbValue, Boolean fallback) {
|
||||||
|
if (dbValue == null || Integer.valueOf(RESULT_PENDING).equals(dbValue)) {
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
return Integer.valueOf(RESULT_SUCCESS).equals(dbValue);
|
||||||
}
|
}
|
||||||
|
|
||||||
private List<PatrolDeleteResultItemVo> parseTaskSnapshots(String json) {
|
private List<PatrolDeleteResultItemVo> parseTaskSnapshots(String json) {
|
||||||
@@ -818,11 +915,39 @@ public class PatrolDeleteTaskService {
|
|||||||
private void persistSnapshotJson(FileTaskEntity task, List<PatrolDeleteResultItemVo> snapshots) {
|
private void persistSnapshotJson(FileTaskEntity task, List<PatrolDeleteResultItemVo> snapshots) {
|
||||||
try {
|
try {
|
||||||
task.setResultJson(objectMapper.writeValueAsString(snapshots == null ? List.of() : snapshots));
|
task.setResultJson(objectMapper.writeValueAsString(snapshots == null ? List.of() : snapshots));
|
||||||
|
syncSnapshotTables(task, snapshots);
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
throw new BusinessException("巡店删除任务快照保存失败");
|
throw new BusinessException("巡店删除任务快照保存失败");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void syncSnapshotTables(FileTaskEntity task, List<PatrolDeleteResultItemVo> snapshots) {
|
||||||
|
List<PatrolDeleteResultItemVo> safe = snapshots == null ? List.of() : snapshots;
|
||||||
|
taskResultItemService.replaceTaskSnapshots(task.getId(), MODULE_TYPE, safe, new TaskResultItemService.SnapshotKeyResolver() {
|
||||||
|
@Override
|
||||||
|
public Long resultId(Object snapshot) {
|
||||||
|
return ((PatrolDeleteResultItemVo) snapshot).getResultId();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String scopeKey(Object snapshot) {
|
||||||
|
PatrolDeleteResultItemVo item = (PatrolDeleteResultItemVo) snapshot;
|
||||||
|
return firstNonBlank(item.getShopName(), "result:" + item.getResultId());
|
||||||
|
}
|
||||||
|
});
|
||||||
|
int successCount = 0;
|
||||||
|
int failedCount = 0;
|
||||||
|
for (PatrolDeleteResultItemVo item : safe) {
|
||||||
|
if (Boolean.TRUE.equals(item.getSuccess())) {
|
||||||
|
successCount++;
|
||||||
|
} else if (Boolean.FALSE.equals(item.getSuccess())) {
|
||||||
|
failedCount++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
taskProgressSnapshotService.save(task.getId(), MODULE_TYPE, firstNonBlank(task.getStatus(), "RUNNING"),
|
||||||
|
safe.size(), successCount, failedCount, null, task.getErrorMessage(), null);
|
||||||
|
}
|
||||||
|
|
||||||
private List<PatrolDeleteCountrySectionDto> copyCountrySections(List<PatrolDeleteCountrySectionDto> sections) {
|
private List<PatrolDeleteCountrySectionDto> copyCountrySections(List<PatrolDeleteCountrySectionDto> sections) {
|
||||||
List<PatrolDeleteCountrySectionDto> copy = new ArrayList<>();
|
List<PatrolDeleteCountrySectionDto> copy = new ArrayList<>();
|
||||||
if (sections == null) {
|
if (sections == null) {
|
||||||
@@ -861,6 +986,11 @@ public class PatrolDeleteTaskService {
|
|||||||
return !blank(first) ? first : second;
|
return !blank(first) ? first : second;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private String buildTaskWorkbookFilename(FileTaskEntity task) {
|
||||||
|
Long taskId = task == null ? null : task.getId();
|
||||||
|
return safeFileStem("巡店删除-" + (taskId == null ? "result" : taskId)) + ".xlsx";
|
||||||
|
}
|
||||||
|
|
||||||
private String safeFileStem(String value) {
|
private String safeFileStem(String value) {
|
||||||
String raw = value == null ? "result" : value.trim();
|
String raw = value == null ? "result" : value.trim();
|
||||||
String safe = raw.replaceAll("[\\\\/:*?\"<>|]+", "_");
|
String safe = raw.replaceAll("[\\\\/:*?\"<>|]+", "_");
|
||||||
|
|||||||
@@ -1,16 +1,20 @@
|
|||||||
package com.nanri.aiimage.modules.permission.service;
|
package com.nanri.aiimage.modules.permission.service;
|
||||||
|
|
||||||
import jakarta.annotation.PostConstruct;
|
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||||
|
import org.springframework.boot.context.event.ApplicationReadyEvent;
|
||||||
|
import org.springframework.context.event.EventListener;
|
||||||
import org.springframework.jdbc.core.JdbcTemplate;
|
import org.springframework.jdbc.core.JdbcTemplate;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import java.util.concurrent.CompletableFuture;
|
||||||
|
|
||||||
@Component
|
@Component
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
@Slf4j
|
@Slf4j
|
||||||
|
@ConditionalOnProperty(prefix = "aiimage.permission-schema-init", name = "enabled", havingValue = "true")
|
||||||
public class PermissionMenuSchemaInitializer {
|
public class PermissionMenuSchemaInitializer {
|
||||||
|
|
||||||
private final JdbcTemplate jdbcTemplate;
|
private final JdbcTemplate jdbcTemplate;
|
||||||
@@ -33,8 +37,12 @@ public class PermissionMenuSchemaInitializer {
|
|||||||
new DefaultAdminMenu("版本管理", "admin_version", "version", 80)
|
new DefaultAdminMenu("版本管理", "admin_version", "version", 80)
|
||||||
);
|
);
|
||||||
|
|
||||||
@PostConstruct
|
@EventListener(ApplicationReadyEvent.class)
|
||||||
public void initialize() {
|
public void initialize() {
|
||||||
|
CompletableFuture.runAsync(this::initializeInternal);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void initializeInternal() {
|
||||||
executeQuietly("""
|
executeQuietly("""
|
||||||
CREATE TABLE IF NOT EXISTS columns (
|
CREATE TABLE IF NOT EXISTS columns (
|
||||||
id INT AUTO_INCREMENT PRIMARY KEY,
|
id INT AUTO_INCREMENT PRIMARY KEY,
|
||||||
|
|||||||
@@ -215,7 +215,7 @@ public class PriceTrackController {
|
|||||||
+ "未完成/处理中:本次只传增量行数据,shop.success 不传或传 null,后端仅合并缓存继续等待。"
|
+ "未完成/处理中:本次只传增量行数据,shop.success 不传或传 null,后端仅合并缓存继续等待。"
|
||||||
+ "已完成:传最后一批数据并设置 shop.success=true,后端会使用该店铺累计后的完整数据出结果。"
|
+ "已完成:传最后一批数据并设置 shop.success=true,后端会使用该店铺累计后的完整数据出结果。"
|
||||||
+ "失败:传 shop.success=false 或传非空 shop.error。"
|
+ "失败:传 shop.success=false 或传非空 shop.error。"
|
||||||
+ "行级数据只保留表头字段:shopMallName、asin、price、recommendedPrice、minimumPrice、firstPlace、secondPlace、cartShopName、priceChangeStatus、modifyCount、status。"
|
+ "行级数据只保留表头字段:shopMallName、asin、price、recommendedPrice、shippingFee、minimumPrice、firstPlace、secondPlace、cartShopName、priceChangeStatus、modifyCount、status。"
|
||||||
)
|
)
|
||||||
public ApiResponse<Void> submitResult(
|
public ApiResponse<Void> submitResult(
|
||||||
@Parameter(
|
@Parameter(
|
||||||
|
|||||||
@@ -73,6 +73,9 @@ public class PriceTrackSubmitResultRequest {
|
|||||||
@Schema(description = "推荐价", example = "18.99")
|
@Schema(description = "推荐价", example = "18.99")
|
||||||
private String recommendedPrice;
|
private String recommendedPrice;
|
||||||
|
|
||||||
|
@Schema(description = "运费", example = "2.50")
|
||||||
|
private String shippingFee;
|
||||||
|
|
||||||
@Schema(description = "最低价", example = "18.50")
|
@Schema(description = "最低价", example = "18.50")
|
||||||
private String minimumPrice;
|
private String minimumPrice;
|
||||||
|
|
||||||
|
|||||||
@@ -52,6 +52,10 @@ public class PriceTrackResultItemVo {
|
|||||||
|
|
||||||
@Schema(description = "预签名下载 URL,列表可能带回,也可通过下载接口获取")
|
@Schema(description = "预签名下载 URL,列表可能带回,也可通过下载接口获取")
|
||||||
private String downloadUrl;
|
private String downloadUrl;
|
||||||
|
private Long fileJobId;
|
||||||
|
private String fileStatus;
|
||||||
|
private String fileError;
|
||||||
|
private Boolean fileReady;
|
||||||
|
|
||||||
@Schema(description = "所属循环任务 ID", example = "1")
|
@Schema(description = "所属循环任务 ID", example = "1")
|
||||||
private Long loopRunId;
|
private Long loopRunId;
|
||||||
|
|||||||
@@ -24,6 +24,7 @@ public class PriceTrackExcelAssemblyService {
|
|||||||
"ASIN",
|
"ASIN",
|
||||||
"价格",
|
"价格",
|
||||||
"推荐价",
|
"推荐价",
|
||||||
|
"运费",
|
||||||
"最低价",
|
"最低价",
|
||||||
"第一名",
|
"第一名",
|
||||||
"第二名",
|
"第二名",
|
||||||
@@ -56,13 +57,14 @@ public class PriceTrackExcelAssemblyService {
|
|||||||
row.createCell(1).setCellValue(valueOf(item.getAsin()));
|
row.createCell(1).setCellValue(valueOf(item.getAsin()));
|
||||||
row.createCell(2).setCellValue(valueOf(item.getPrice()));
|
row.createCell(2).setCellValue(valueOf(item.getPrice()));
|
||||||
row.createCell(3).setCellValue(valueOf(item.getRecommendedPrice()));
|
row.createCell(3).setCellValue(valueOf(item.getRecommendedPrice()));
|
||||||
row.createCell(4).setCellValue(valueOf(item.getMinimumPrice()));
|
row.createCell(4).setCellValue(valueOf(item.getShippingFee()));
|
||||||
row.createCell(5).setCellValue(valueOf(item.getFirstPlace()));
|
row.createCell(5).setCellValue(valueOf(item.getMinimumPrice()));
|
||||||
row.createCell(6).setCellValue(valueOf(item.getSecondPlace()));
|
row.createCell(6).setCellValue(valueOf(item.getFirstPlace()));
|
||||||
row.createCell(7).setCellValue(valueOf(item.getCartShopName()));
|
row.createCell(7).setCellValue(valueOf(item.getSecondPlace()));
|
||||||
row.createCell(8).setCellValue(valueOf(item.getPriceChangeStatus()));
|
row.createCell(8).setCellValue(valueOf(item.getCartShopName()));
|
||||||
row.createCell(9).setCellValue(valueOf(item.getModifyCount()));
|
row.createCell(9).setCellValue(valueOf(item.getPriceChangeStatus()));
|
||||||
row.createCell(10).setCellValue(valueOf(item.getStatus()));
|
row.createCell(10).setCellValue(valueOf(item.getModifyCount()));
|
||||||
|
row.createCell(11).setCellValue(valueOf(item.getStatus()));
|
||||||
}
|
}
|
||||||
applyDefaultColumnWidths(sheet);
|
applyDefaultColumnWidths(sheet);
|
||||||
}
|
}
|
||||||
@@ -112,7 +114,7 @@ public class PriceTrackExcelAssemblyService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private void applyDefaultColumnWidths(Sheet sheet) {
|
private void applyDefaultColumnWidths(Sheet sheet) {
|
||||||
int[] widths = {22, 18, 12, 12, 12, 20, 20, 22, 18, 12, 12};
|
int[] widths = {22, 18, 12, 12, 12, 12, 20, 20, 22, 18, 12, 12};
|
||||||
for (int i = 0; i < widths.length; i++) {
|
for (int i = 0; i < widths.length; i++) {
|
||||||
sheet.setColumnWidth(i, widths[i] * 256);
|
sheet.setColumnWidth(i, widths[i] * 256);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import com.nanri.aiimage.modules.pricetrack.model.dto.PriceTrackSubmitResultRequ
|
|||||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskScopePayloadStorageService;
|
import com.nanri.aiimage.modules.task.service.TaskScopePayloadStorageService;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
@@ -13,11 +14,13 @@ import java.time.Duration;
|
|||||||
import java.time.Instant;
|
import java.time.Instant;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.LinkedHashMap;
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.concurrent.ConcurrentHashMap;
|
import java.util.concurrent.ConcurrentHashMap;
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
|
@Slf4j
|
||||||
public class PriceTrackTaskCacheService {
|
public class PriceTrackTaskCacheService {
|
||||||
|
|
||||||
private static final String MODULE_TYPE = "PRICE_TRACK";
|
private static final String MODULE_TYPE = "PRICE_TRACK";
|
||||||
@@ -32,17 +35,27 @@ public class PriceTrackTaskCacheService {
|
|||||||
if (taskId == null || taskId <= 0) {
|
if (taskId == null || taskId <= 0) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
stringRedisTemplate.opsForValue().set(
|
try {
|
||||||
buildTaskHeartbeatKey(taskId),
|
stringRedisTemplate.opsForValue().set(
|
||||||
String.valueOf(Instant.now().toEpochMilli()),
|
buildTaskHeartbeatKey(taskId),
|
||||||
Duration.ofHours(HEARTBEAT_TTL_HOURS));
|
String.valueOf(Instant.now().toEpochMilli()),
|
||||||
|
Duration.ofHours(HEARTBEAT_TTL_HOURS));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[price-track-cache] touch heartbeat degraded taskId={} msg={}", taskId, ex.getMessage());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public long getTaskHeartbeatMillis(Long taskId) {
|
public long getTaskHeartbeatMillis(Long taskId) {
|
||||||
if (taskId == null || taskId <= 0) {
|
if (taskId == null || taskId <= 0) {
|
||||||
return 0L;
|
return 0L;
|
||||||
}
|
}
|
||||||
String raw = stringRedisTemplate.opsForValue().get(buildTaskHeartbeatKey(taskId));
|
String raw;
|
||||||
|
try {
|
||||||
|
raw = stringRedisTemplate.opsForValue().get(buildTaskHeartbeatKey(taskId));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[price-track-cache] get heartbeat degraded taskId={} msg={}", taskId, ex.getMessage());
|
||||||
|
return 0L;
|
||||||
|
}
|
||||||
if (raw == null || raw.isBlank()) {
|
if (raw == null || raw.isBlank()) {
|
||||||
return 0L;
|
return 0L;
|
||||||
}
|
}
|
||||||
@@ -53,6 +66,41 @@ public class PriceTrackTaskCacheService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public Map<Long, Long> getTaskHeartbeatMillisBatch(List<Long> taskIds) {
|
||||||
|
Map<Long, Long> result = new LinkedHashMap<>();
|
||||||
|
if (taskIds == null || taskIds.isEmpty()) {
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
List<Long> normalized = taskIds.stream()
|
||||||
|
.filter(id -> id != null && id > 0)
|
||||||
|
.distinct()
|
||||||
|
.toList();
|
||||||
|
if (normalized.isEmpty()) {
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
List<String> keys = normalized.stream().map(this::buildTaskHeartbeatKey).toList();
|
||||||
|
List<String> values;
|
||||||
|
try {
|
||||||
|
values = stringRedisTemplate.opsForValue().multiGet(keys);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[price-track-cache] batch get heartbeat degraded taskIds={} msg={}", normalized, ex.getMessage());
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
for (int i = 0; i < normalized.size(); i++) {
|
||||||
|
String raw = values != null && i < values.size() ? values.get(i) : null;
|
||||||
|
if (raw == null || raw.isBlank()) {
|
||||||
|
result.put(normalized.get(i), 0L);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
result.put(normalized.get(i), Long.parseLong(raw));
|
||||||
|
} catch (NumberFormatException ignored) {
|
||||||
|
result.put(normalized.get(i), 0L);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
public PriceTrackSubmitResultRequest.ShopResult getShopMergedPayload(Long taskId, String shopKey) {
|
public PriceTrackSubmitResultRequest.ShopResult getShopMergedPayload(Long taskId, String shopKey) {
|
||||||
return taskScopePayloadStorageService.getScopePayload(taskId, MODULE_TYPE, shopKey, PriceTrackSubmitResultRequest.ShopResult.class);
|
return taskScopePayloadStorageService.getScopePayload(taskId, MODULE_TYPE, shopKey, PriceTrackSubmitResultRequest.ShopResult.class);
|
||||||
}
|
}
|
||||||
@@ -82,8 +130,12 @@ public class PriceTrackTaskCacheService {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
taskEntityLocalCache.remove(taskId);
|
taskEntityLocalCache.remove(taskId);
|
||||||
stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId));
|
try {
|
||||||
stringRedisTemplate.delete(buildTaskEntityKey(taskId));
|
stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId));
|
||||||
|
stringRedisTemplate.delete(buildTaskEntityKey(taskId));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[price-track-cache] delete cache degraded taskId={} msg={}", taskId, ex.getMessage());
|
||||||
|
}
|
||||||
taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE);
|
taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -132,7 +184,13 @@ public class PriceTrackTaskCacheService {
|
|||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
java.util.List<String> keys = missingIds.stream().map(this::buildTaskEntityKey).toList();
|
java.util.List<String> keys = missingIds.stream().map(this::buildTaskEntityKey).toList();
|
||||||
java.util.List<String> values = stringRedisTemplate.opsForValue().multiGet(keys);
|
java.util.List<String> values;
|
||||||
|
try {
|
||||||
|
values = stringRedisTemplate.opsForValue().multiGet(keys);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[price-track-cache] batch get task cache degraded taskIds={} msg={}", missingIds, ex.getMessage());
|
||||||
|
return result;
|
||||||
|
}
|
||||||
for (int i = 0; i < missingIds.size(); i++) {
|
for (int i = 0; i < missingIds.size(); i++) {
|
||||||
Long taskId = missingIds.get(i);
|
Long taskId = missingIds.get(i);
|
||||||
String val = values != null && i < values.size() ? values.get(i) : null;
|
String val = values != null && i < values.size() ? values.get(i) : null;
|
||||||
|
|||||||
@@ -26,6 +26,9 @@ import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
|||||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskResultPayloadService;
|
||||||
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
|
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
|
||||||
import cn.hutool.core.util.IdUtil;
|
import cn.hutool.core.util.IdUtil;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
@@ -66,6 +69,8 @@ public class PriceTrackTaskService {
|
|||||||
private final PriceTrackTaskCacheService priceTrackTaskCacheService;
|
private final PriceTrackTaskCacheService priceTrackTaskCacheService;
|
||||||
private final PriceTrackLoopRunService priceTrackLoopRunService;
|
private final PriceTrackLoopRunService priceTrackLoopRunService;
|
||||||
private final TaskPressureProperties taskPressureProperties;
|
private final TaskPressureProperties taskPressureProperties;
|
||||||
|
private final TaskResultPayloadService taskResultPayloadService;
|
||||||
|
private final TaskFileJobService taskFileJobService;
|
||||||
|
|
||||||
private FileTaskEntity loadTaskForExecution(Long taskId) {
|
private FileTaskEntity loadTaskForExecution(Long taskId) {
|
||||||
Map<Long, FileTaskEntity> cachedTasks = priceTrackTaskCacheService.getTaskCacheBatch(List.of(taskId));
|
Map<Long, FileTaskEntity> cachedTasks = priceTrackTaskCacheService.getTaskCacheBatch(List.of(taskId));
|
||||||
@@ -435,7 +440,7 @@ public class PriceTrackTaskService {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
assembleShopResult(fr, shopKey, merged);
|
enqueueResultFileAssembly(fr, shopKey, merged);
|
||||||
priceTrackTaskCacheService.removeShopMergedPayload(taskId, shopKey);
|
priceTrackTaskCacheService.removeShopMergedPayload(taskId, shopKey);
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
markResultFailed(fr, ex.getMessage() == null ? "assemble price-track result failed" : ex.getMessage());
|
markResultFailed(fr, ex.getMessage() == null ? "assemble price-track result failed" : ex.getMessage());
|
||||||
@@ -509,7 +514,7 @@ public class PriceTrackTaskService {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
assembleShopResult(fr, shopKey, cachedPayload);
|
enqueueResultFileAssembly(fr, shopKey, cachedPayload);
|
||||||
priceTrackTaskCacheService.removeShopMergedPayload(taskId, shopKey);
|
priceTrackTaskCacheService.removeShopMergedPayload(taskId, shopKey);
|
||||||
changed = true;
|
changed = true;
|
||||||
log.warn("[price-track] stale finalize assembled taskId={} shop={} fromCompensation={} completedFlag={} rowCount={}",
|
log.warn("[price-track] stale finalize assembled taskId={} shop={} fromCompensation={} completedFlag={} rowCount={}",
|
||||||
@@ -618,6 +623,49 @@ public class PriceTrackTaskService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void processResultFileJob(TaskFileJobEntity job) {
|
||||||
|
if (job == null || job.getTaskId() == null || job.getResultId() == null || job.getScopeKey() == null) {
|
||||||
|
throw new BusinessException("结果文件任务参数不完整");
|
||||||
|
}
|
||||||
|
FileResultEntity result = fileResultMapper.selectById(job.getResultId());
|
||||||
|
if (result == null || !MODULE_TYPE.equals(result.getModuleType())) {
|
||||||
|
throw new BusinessException("结果记录不存在");
|
||||||
|
}
|
||||||
|
PriceTrackSubmitResultRequest.ShopResult payload = taskResultPayloadService.getLatest(
|
||||||
|
job.getTaskId(), MODULE_TYPE, job.getScopeKey(), PriceTrackSubmitResultRequest.ShopResult.class);
|
||||||
|
if (payload == null) {
|
||||||
|
throw new BusinessException("结果文件载荷不存在");
|
||||||
|
}
|
||||||
|
assembleShopResult(result, job.getScopeKey(), payload);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void enqueueResultFileAssembly(FileResultEntity result,
|
||||||
|
String shopKey,
|
||||||
|
PriceTrackSubmitResultRequest.ShopResult payload) {
|
||||||
|
taskResultPayloadService.saveLatest(result.getTaskId(), MODULE_TYPE, shopKey, payload);
|
||||||
|
markResultFilePending(result, shopKey, payload);
|
||||||
|
taskFileJobService.enqueueAssembleResult(result.getTaskId(), MODULE_TYPE, result.getId(), shopKey);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void markResultFilePending(FileResultEntity result,
|
||||||
|
String shopKey,
|
||||||
|
PriceTrackSubmitResultRequest.ShopResult payload) {
|
||||||
|
Map<String, List<PriceTrackSubmitResultRequest.AsinResult>> countries =
|
||||||
|
excelAssemblyService.normalizeCountriesMap(payload.getCountries());
|
||||||
|
String displayName = payload.getShopName() != null && !payload.getShopName().isBlank()
|
||||||
|
? payload.getShopName().trim()
|
||||||
|
: shopKey;
|
||||||
|
String stem = safeFileStem(displayName);
|
||||||
|
result.setSuccess(1);
|
||||||
|
result.setErrorMessage(null);
|
||||||
|
result.setResultFilename(stem + ".xlsx");
|
||||||
|
result.setResultFileUrl(null);
|
||||||
|
result.setResultFileSize(0L);
|
||||||
|
result.setResultContentType(CONTENT_TYPE_XLSX);
|
||||||
|
result.setRowCount(excelAssemblyService.countRows(countries));
|
||||||
|
fileResultMapper.updateById(result);
|
||||||
|
}
|
||||||
|
|
||||||
public Map<String, List<Map<String, String>>> parseMatchAsinRowsByCountry(List<String> asinFiles, List<String> countryCodes) {
|
public Map<String, List<Map<String, String>>> parseMatchAsinRowsByCountry(List<String> asinFiles, List<String> countryCodes) {
|
||||||
return parseAsinRowsByCountry(asinFiles, countryCodes);
|
return parseAsinRowsByCountry(asinFiles, countryCodes);
|
||||||
}
|
}
|
||||||
@@ -901,8 +949,8 @@ public class PriceTrackTaskService {
|
|||||||
vo.setSuccess(ok);
|
vo.setSuccess(ok);
|
||||||
vo.setError(entity.getErrorMessage());
|
vo.setError(entity.getErrorMessage());
|
||||||
vo.setOutputFilename(entity.getResultFilename());
|
vo.setOutputFilename(entity.getResultFilename());
|
||||||
vo.setDownloadUrl(entity.getResultFileUrl() == null || entity.getResultFileUrl().isBlank()
|
vo.setDownloadUrl(null);
|
||||||
? null : ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl()));
|
attachFileJobState(vo, entity);
|
||||||
vo.setMatched(true);
|
vo.setMatched(true);
|
||||||
if (entity.getTaskId() != null) {
|
if (entity.getTaskId() != null) {
|
||||||
mergeQueueFieldsFromRequest(vo, entity.getTaskId());
|
mergeQueueFieldsFromRequest(vo, entity.getTaskId());
|
||||||
@@ -910,6 +958,18 @@ public class PriceTrackTaskService {
|
|||||||
return vo;
|
return vo;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void attachFileJobState(PriceTrackResultItemVo vo, FileResultEntity entity) {
|
||||||
|
TaskFileJobEntity job = taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId());
|
||||||
|
vo.setFileReady(entity.getResultFileUrl() != null && !entity.getResultFileUrl().isBlank());
|
||||||
|
if (job == null) {
|
||||||
|
vo.setFileStatus(Boolean.TRUE.equals(vo.getFileReady()) ? "SUCCESS" : null);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
vo.setFileJobId(job.getId());
|
||||||
|
vo.setFileStatus(job.getStatus());
|
||||||
|
vo.setFileError(job.getErrorMessage());
|
||||||
|
}
|
||||||
|
|
||||||
private PriceTrackTaskDetailVo buildTaskDetail(FileTaskEntity task) {
|
private PriceTrackTaskDetailVo buildTaskDetail(FileTaskEntity task) {
|
||||||
PriceTrackTaskDetailVo detail = new PriceTrackTaskDetailVo();
|
PriceTrackTaskDetailVo detail = new PriceTrackTaskDetailVo();
|
||||||
detail.setTask(toTaskItemVo(task));
|
detail.setTask(toTaskItemVo(task));
|
||||||
@@ -1074,6 +1134,7 @@ public class PriceTrackTaskService {
|
|||||||
merged.setAsin(firstNonBlank(incoming.getAsin(), merged.getAsin()));
|
merged.setAsin(firstNonBlank(incoming.getAsin(), merged.getAsin()));
|
||||||
merged.setPrice(firstNonBlank(incoming.getPrice(), merged.getPrice()));
|
merged.setPrice(firstNonBlank(incoming.getPrice(), merged.getPrice()));
|
||||||
merged.setRecommendedPrice(firstNonBlank(incoming.getRecommendedPrice(), merged.getRecommendedPrice()));
|
merged.setRecommendedPrice(firstNonBlank(incoming.getRecommendedPrice(), merged.getRecommendedPrice()));
|
||||||
|
merged.setShippingFee(firstNonBlank(incoming.getShippingFee(), merged.getShippingFee()));
|
||||||
merged.setMinimumPrice(firstNonBlank(incoming.getMinimumPrice(), merged.getMinimumPrice()));
|
merged.setMinimumPrice(firstNonBlank(incoming.getMinimumPrice(), merged.getMinimumPrice()));
|
||||||
merged.setFirstPlace(firstNonBlank(incoming.getFirstPlace(), merged.getFirstPlace()));
|
merged.setFirstPlace(firstNonBlank(incoming.getFirstPlace(), merged.getFirstPlace()));
|
||||||
merged.setSecondPlace(firstNonBlank(incoming.getSecondPlace(), merged.getSecondPlace()));
|
merged.setSecondPlace(firstNonBlank(incoming.getSecondPlace(), merged.getSecondPlace()));
|
||||||
@@ -1097,6 +1158,7 @@ public class PriceTrackTaskService {
|
|||||||
out.setAsin(row.getAsin());
|
out.setAsin(row.getAsin());
|
||||||
out.setPrice(row.getPrice());
|
out.setPrice(row.getPrice());
|
||||||
out.setRecommendedPrice(row.getRecommendedPrice());
|
out.setRecommendedPrice(row.getRecommendedPrice());
|
||||||
|
out.setShippingFee(row.getShippingFee());
|
||||||
out.setMinimumPrice(row.getMinimumPrice());
|
out.setMinimumPrice(row.getMinimumPrice());
|
||||||
out.setFirstPlace(row.getFirstPlace());
|
out.setFirstPlace(row.getFirstPlace());
|
||||||
out.setSecondPlace(row.getSecondPlace());
|
out.setSecondPlace(row.getSecondPlace());
|
||||||
|
|||||||
@@ -57,6 +57,10 @@ public class ProductRiskResultItemVo {
|
|||||||
@JsonProperty("downloadUrl")
|
@JsonProperty("downloadUrl")
|
||||||
@Schema(description = "预签名下载 URL(列表里可能带;直链下载也可用 GET /results/{resultId}/download)")
|
@Schema(description = "预签名下载 URL(列表里可能带;直链下载也可用 GET /results/{resultId}/download)")
|
||||||
private String downloadUrl;
|
private String downloadUrl;
|
||||||
|
private Long fileJobId;
|
||||||
|
private String fileStatus;
|
||||||
|
private String fileError;
|
||||||
|
private Boolean fileReady;
|
||||||
|
|
||||||
@JsonProperty("scheduledAt")
|
@JsonProperty("scheduledAt")
|
||||||
@Schema(description = "定时执行时间,未设置时为空")
|
@Schema(description = "定时执行时间,未设置时为空")
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskShopPayloadDto
|
|||||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskScopePayloadStorageService;
|
import com.nanri.aiimage.modules.task.service.TaskScopePayloadStorageService;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
@@ -13,11 +14,13 @@ import java.time.Duration;
|
|||||||
import java.time.Instant;
|
import java.time.Instant;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.LinkedHashMap;
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.concurrent.ConcurrentHashMap;
|
import java.util.concurrent.ConcurrentHashMap;
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
|
@Slf4j
|
||||||
public class ProductRiskTaskCacheService {
|
public class ProductRiskTaskCacheService {
|
||||||
|
|
||||||
private static final String MODULE_TYPE = "PRODUCT_RISK_RESOLVE";
|
private static final String MODULE_TYPE = "PRODUCT_RISK_RESOLVE";
|
||||||
@@ -60,7 +63,13 @@ public class ProductRiskTaskCacheService {
|
|||||||
if (taskId == null || taskId <= 0) {
|
if (taskId == null || taskId <= 0) {
|
||||||
return 0L;
|
return 0L;
|
||||||
}
|
}
|
||||||
String raw = stringRedisTemplate.opsForValue().get(buildTaskHeartbeatKey(taskId));
|
String raw;
|
||||||
|
try {
|
||||||
|
raw = stringRedisTemplate.opsForValue().get(buildTaskHeartbeatKey(taskId));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[product-risk-cache] get heartbeat degraded taskId={} msg={}", taskId, ex.getMessage());
|
||||||
|
return 0L;
|
||||||
|
}
|
||||||
if (raw == null || raw.isBlank()) {
|
if (raw == null || raw.isBlank()) {
|
||||||
return 0L;
|
return 0L;
|
||||||
}
|
}
|
||||||
@@ -71,13 +80,52 @@ public class ProductRiskTaskCacheService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public Map<Long, Long> getTaskHeartbeatMillisBatch(List<Long> taskIds) {
|
||||||
|
Map<Long, Long> result = new LinkedHashMap<>();
|
||||||
|
if (taskIds == null || taskIds.isEmpty()) {
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
List<Long> normalized = taskIds.stream()
|
||||||
|
.filter(id -> id != null && id > 0)
|
||||||
|
.distinct()
|
||||||
|
.toList();
|
||||||
|
if (normalized.isEmpty()) {
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
List<String> keys = normalized.stream().map(this::buildTaskHeartbeatKey).toList();
|
||||||
|
List<String> values;
|
||||||
|
try {
|
||||||
|
values = stringRedisTemplate.opsForValue().multiGet(keys);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[product-risk-cache] batch get heartbeat degraded taskIds={} msg={}", normalized, ex.getMessage());
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
for (int i = 0; i < normalized.size(); i++) {
|
||||||
|
String raw = values != null && i < values.size() ? values.get(i) : null;
|
||||||
|
if (raw == null || raw.isBlank()) {
|
||||||
|
result.put(normalized.get(i), 0L);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
result.put(normalized.get(i), Long.parseLong(raw));
|
||||||
|
} catch (NumberFormatException ignored) {
|
||||||
|
result.put(normalized.get(i), 0L);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
public void deleteTaskCache(Long taskId) {
|
public void deleteTaskCache(Long taskId) {
|
||||||
if (taskId == null || taskId <= 0) {
|
if (taskId == null || taskId <= 0) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
taskEntityLocalCache.remove(taskId);
|
taskEntityLocalCache.remove(taskId);
|
||||||
stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId));
|
try {
|
||||||
stringRedisTemplate.delete(buildTaskEntityKey(taskId));
|
stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId));
|
||||||
|
stringRedisTemplate.delete(buildTaskEntityKey(taskId));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[product-risk-cache] delete cache degraded taskId={} msg={}", taskId, ex.getMessage());
|
||||||
|
}
|
||||||
taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE);
|
taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -126,7 +174,13 @@ public class ProductRiskTaskCacheService {
|
|||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
java.util.List<String> keys = missingIds.stream().map(this::buildTaskEntityKey).toList();
|
java.util.List<String> keys = missingIds.stream().map(this::buildTaskEntityKey).toList();
|
||||||
java.util.List<String> values = stringRedisTemplate.opsForValue().multiGet(keys);
|
java.util.List<String> values;
|
||||||
|
try {
|
||||||
|
values = stringRedisTemplate.opsForValue().multiGet(keys);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[product-risk-cache] batch get task cache degraded taskIds={} msg={}", missingIds, ex.getMessage());
|
||||||
|
return result;
|
||||||
|
}
|
||||||
for (int i = 0; i < missingIds.size(); i++) {
|
for (int i = 0; i < missingIds.size(); i++) {
|
||||||
Long taskId = missingIds.get(i);
|
Long taskId = missingIds.get(i);
|
||||||
String val = values != null && i < values.size() ? values.get(i) : null;
|
String val = values != null && i < values.size() ? values.get(i) : null;
|
||||||
@@ -144,10 +198,14 @@ public class ProductRiskTaskCacheService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public void touchTaskHeartbeat(Long taskId) {
|
public void touchTaskHeartbeat(Long taskId) {
|
||||||
stringRedisTemplate.opsForValue().set(
|
try {
|
||||||
buildTaskHeartbeatKey(taskId),
|
stringRedisTemplate.opsForValue().set(
|
||||||
String.valueOf(Instant.now().toEpochMilli()),
|
buildTaskHeartbeatKey(taskId),
|
||||||
Duration.ofHours(PAYLOAD_TTL_HOURS));
|
String.valueOf(Instant.now().toEpochMilli()),
|
||||||
|
Duration.ofHours(PAYLOAD_TTL_HOURS));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[product-risk-cache] touch heartbeat degraded taskId={} msg={}", taskId, ex.getMessage());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private String buildTaskHeartbeatKey(Long taskId) {
|
private String buildTaskHeartbeatKey(Long taskId) {
|
||||||
|
|||||||
@@ -24,6 +24,11 @@ import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
|||||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskResultItemService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskResultPayloadService;
|
||||||
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
|
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
|
||||||
import cn.hutool.core.io.FileUtil;
|
import cn.hutool.core.io.FileUtil;
|
||||||
import cn.hutool.core.util.IdUtil;
|
import cn.hutool.core.util.IdUtil;
|
||||||
@@ -59,6 +64,10 @@ public class ProductRiskTaskService {
|
|||||||
private final ObjectMapper objectMapper;
|
private final ObjectMapper objectMapper;
|
||||||
private final ProductRiskTaskCacheService productRiskTaskCacheService;
|
private final ProductRiskTaskCacheService productRiskTaskCacheService;
|
||||||
private final TaskPressureProperties taskPressureProperties;
|
private final TaskPressureProperties taskPressureProperties;
|
||||||
|
private final TaskResultPayloadService taskResultPayloadService;
|
||||||
|
private final TaskFileJobService taskFileJobService;
|
||||||
|
private final TaskResultItemService taskResultItemService;
|
||||||
|
private final TaskProgressSnapshotService taskProgressSnapshotService;
|
||||||
|
|
||||||
private FileTaskEntity loadTaskForExecution(Long taskId) {
|
private FileTaskEntity loadTaskForExecution(Long taskId) {
|
||||||
Map<Long, FileTaskEntity> cachedTasks = productRiskTaskCacheService.getTaskCacheBatch(List.of(taskId));
|
Map<Long, FileTaskEntity> cachedTasks = productRiskTaskCacheService.getTaskCacheBatch(List.of(taskId));
|
||||||
@@ -553,7 +562,7 @@ public class ProductRiskTaskService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
assembleShopResult(fr, shopKey, mergedPayload, workRoot);
|
enqueueResultFileAssembly(fr, shopKey, mergedPayload);
|
||||||
assembledCount++;
|
assembledCount++;
|
||||||
log.info("[product-risk] shop assembled taskId={} shop={} rowCount={} file={}",
|
log.info("[product-risk] shop assembled taskId={} shop={} rowCount={} file={}",
|
||||||
taskId, shopKey, fr.getRowCount(), fr.getResultFilename());
|
taskId, shopKey, fr.getRowCount(), fr.getResultFilename());
|
||||||
@@ -698,7 +707,7 @@ public class ProductRiskTaskService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
assembleShopResult(fr, shopKey, cachedPayload, workRoot);
|
enqueueResultFileAssembly(fr, shopKey, cachedPayload);
|
||||||
productRiskTaskCacheService.removeShopMergedPayload(taskId, shopKey);
|
productRiskTaskCacheService.removeShopMergedPayload(taskId, shopKey);
|
||||||
changed = true;
|
changed = true;
|
||||||
log.warn("[product-risk] stale finalize assembled taskId={} shop={} fromCompensation={} completedFlag={} rowCount={}",
|
log.warn("[product-risk] stale finalize assembled taskId={} shop={} fromCompensation={} completedFlag={} rowCount={}",
|
||||||
@@ -778,6 +787,49 @@ public class ProductRiskTaskService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void processResultFileJob(TaskFileJobEntity job) {
|
||||||
|
if (job == null || job.getTaskId() == null || job.getResultId() == null || job.getScopeKey() == null) {
|
||||||
|
throw new BusinessException("结果文件任务参数不完整");
|
||||||
|
}
|
||||||
|
FileResultEntity result = fileResultMapper.selectById(job.getResultId());
|
||||||
|
if (result == null || !MODULE_TYPE.equals(result.getModuleType())) {
|
||||||
|
throw new BusinessException("结果记录不存在");
|
||||||
|
}
|
||||||
|
ProductRiskShopPayloadDto payload = taskResultPayloadService.getLatest(
|
||||||
|
job.getTaskId(), MODULE_TYPE, job.getScopeKey(), ProductRiskShopPayloadDto.class);
|
||||||
|
if (payload == null) {
|
||||||
|
payload = taskResultItemService.getResultSnapshot(job.getTaskId(), MODULE_TYPE, job.getResultId(), ProductRiskShopPayloadDto.class);
|
||||||
|
}
|
||||||
|
if (payload == null) {
|
||||||
|
throw new BusinessException("结果文件载荷不存在");
|
||||||
|
}
|
||||||
|
File workRoot = FileUtil.mkdir(FileUtil.file(System.getProperty("java.io.tmpdir"), "product-risk-result", String.valueOf(job.getTaskId())));
|
||||||
|
assembleShopResult(result, job.getScopeKey(), payload, workRoot);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void enqueueResultFileAssembly(FileResultEntity result, String shopKey, ProductRiskShopPayloadDto payload) {
|
||||||
|
taskResultPayloadService.saveLatest(result.getTaskId(), MODULE_TYPE, shopKey, payload);
|
||||||
|
taskResultItemService.replaceResultSnapshot(result.getTaskId(), MODULE_TYPE, result.getId(), shopKey, payload);
|
||||||
|
markResultFilePending(result, shopKey, payload);
|
||||||
|
taskFileJobService.enqueueAssembleResult(result.getTaskId(), MODULE_TYPE, result.getId(), shopKey);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void markResultFilePending(FileResultEntity result, String shopKey, ProductRiskShopPayloadDto payload) {
|
||||||
|
Map<String, List<ProductRiskRowDto>> countries = excelAssemblyService.normalizeCountriesMap(payload.getCountries());
|
||||||
|
String displayName = payload.getShopName() != null && !payload.getShopName().isBlank()
|
||||||
|
? payload.getShopName().trim()
|
||||||
|
: shopKey;
|
||||||
|
String stem = safeFileStem(displayName);
|
||||||
|
result.setResultFilename(stem + ".zip");
|
||||||
|
result.setResultFileUrl(null);
|
||||||
|
result.setResultFileSize(0L);
|
||||||
|
result.setResultContentType(CONTENT_TYPE_ZIP);
|
||||||
|
result.setRowCount(excelAssemblyService.countRows(countries));
|
||||||
|
result.setSuccess(1);
|
||||||
|
result.setErrorMessage(null);
|
||||||
|
fileResultMapper.updateById(result);
|
||||||
|
}
|
||||||
|
|
||||||
private void updateTaskStatusFromLatestRows(FileTaskEntity task,
|
private void updateTaskStatusFromLatestRows(FileTaskEntity task,
|
||||||
List<FileResultEntity> latest,
|
List<FileResultEntity> latest,
|
||||||
List<String> batchErrors) {
|
List<String> batchErrors) {
|
||||||
@@ -826,6 +878,8 @@ public class ProductRiskTaskService {
|
|||||||
log.warn("[product-risk] compact result json failed: {}", ex.getMessage());
|
log.warn("[product-risk] compact result json failed: {}", ex.getMessage());
|
||||||
}
|
}
|
||||||
fileTaskMapper.updateById(task);
|
fileTaskMapper.updateById(task);
|
||||||
|
taskProgressSnapshotService.save(task.getId(), MODULE_TYPE, task.getStatus(),
|
||||||
|
latest.size(), ok, fail, null, task.getErrorMessage(), null);
|
||||||
log.warn("[product-risk] status updated from latest rows taskId={} oldStatus={} newStatus={} ok={} fail={} allDone={} batchErrors={} errorMessage={}",
|
log.warn("[product-risk] status updated from latest rows taskId={} oldStatus={} newStatus={} ok={} fail={} allDone={} batchErrors={} errorMessage={}",
|
||||||
task.getId(), oldStatus, task.getStatus(), ok, fail, allDone, batchErrors, task.getErrorMessage());
|
task.getId(), oldStatus, task.getStatus(), ok, fail, allDone, batchErrors, task.getErrorMessage());
|
||||||
cleanupTaskCacheIfTerminal(task.getId(), task.getStatus());
|
cleanupTaskCacheIfTerminal(task.getId(), task.getStatus());
|
||||||
@@ -910,9 +964,8 @@ public class ProductRiskTaskService {
|
|||||||
vo.setSuccess(ok);
|
vo.setSuccess(ok);
|
||||||
vo.setError(entity.getErrorMessage());
|
vo.setError(entity.getErrorMessage());
|
||||||
vo.setOutputFilename(entity.getResultFilename());
|
vo.setOutputFilename(entity.getResultFilename());
|
||||||
vo.setDownloadUrl(entity.getResultFileUrl() == null || entity.getResultFileUrl().isBlank()
|
vo.setDownloadUrl(null);
|
||||||
? null
|
attachFileJobState(vo, entity);
|
||||||
: ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl()));
|
|
||||||
vo.setMatched(true);
|
vo.setMatched(true);
|
||||||
if (entity.getTaskId() != null) {
|
if (entity.getTaskId() != null) {
|
||||||
mergeQueueFieldsFromRequest(vo, entity.getTaskId());
|
mergeQueueFieldsFromRequest(vo, entity.getTaskId());
|
||||||
@@ -920,6 +973,18 @@ public class ProductRiskTaskService {
|
|||||||
return vo;
|
return vo;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void attachFileJobState(ProductRiskResultItemVo vo, FileResultEntity entity) {
|
||||||
|
TaskFileJobEntity job = taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId());
|
||||||
|
vo.setFileReady(entity.getResultFileUrl() != null && !entity.getResultFileUrl().isBlank());
|
||||||
|
if (job == null) {
|
||||||
|
vo.setFileStatus(Boolean.TRUE.equals(vo.getFileReady()) ? "SUCCESS" : null);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
vo.setFileJobId(job.getId());
|
||||||
|
vo.setFileStatus(job.getStatus());
|
||||||
|
vo.setFileError(job.getErrorMessage());
|
||||||
|
}
|
||||||
|
|
||||||
private void mergeQueueFieldsFromRequest(ProductRiskResultItemVo vo, Long taskId) {
|
private void mergeQueueFieldsFromRequest(ProductRiskResultItemVo vo, Long taskId) {
|
||||||
FileTaskEntity t = loadTaskForExecution(taskId);
|
FileTaskEntity t = loadTaskForExecution(taskId);
|
||||||
if (t == null || t.getRequestJson() == null || t.getRequestJson().isBlank()) {
|
if (t == null || t.getRequestJson() == null || t.getRequestJson().isBlank()) {
|
||||||
|
|||||||
@@ -43,6 +43,10 @@ public class QueryAsinResultItemVo {
|
|||||||
private LocalDateTime finishedAt;
|
private LocalDateTime finishedAt;
|
||||||
private String outputFilename;
|
private String outputFilename;
|
||||||
private String downloadUrl;
|
private String downloadUrl;
|
||||||
|
private Long fileJobId;
|
||||||
|
private String fileStatus;
|
||||||
|
private String fileError;
|
||||||
|
private Boolean fileReady;
|
||||||
|
|
||||||
@JsonProperty("queryAsins")
|
@JsonProperty("queryAsins")
|
||||||
private List<QueryAsinCountryAsinsDto> queryAsins = new ArrayList<>();
|
private List<QueryAsinCountryAsinsDto> queryAsins = new ArrayList<>();
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinShopPayloadDto;
|
|||||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||||
import com.nanri.aiimage.modules.task.service.TaskScopePayloadStorageService;
|
import com.nanri.aiimage.modules.task.service.TaskScopePayloadStorageService;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
@@ -18,6 +19,7 @@ import java.util.concurrent.ConcurrentHashMap;
|
|||||||
|
|
||||||
@Service
|
@Service
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
|
@Slf4j
|
||||||
public class QueryAsinTaskCacheService {
|
public class QueryAsinTaskCacheService {
|
||||||
|
|
||||||
private static final String MODULE_TYPE = "QUERY_ASIN";
|
private static final String MODULE_TYPE = "QUERY_ASIN";
|
||||||
@@ -60,7 +62,13 @@ public class QueryAsinTaskCacheService {
|
|||||||
if (taskId == null || taskId <= 0) {
|
if (taskId == null || taskId <= 0) {
|
||||||
return 0L;
|
return 0L;
|
||||||
}
|
}
|
||||||
String raw = stringRedisTemplate.opsForValue().get(buildTaskHeartbeatKey(taskId));
|
String raw;
|
||||||
|
try {
|
||||||
|
raw = stringRedisTemplate.opsForValue().get(buildTaskHeartbeatKey(taskId));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[query-asin-cache] get heartbeat degraded taskId={} msg={}", taskId, ex.getMessage());
|
||||||
|
return 0L;
|
||||||
|
}
|
||||||
if (raw == null || raw.isBlank()) {
|
if (raw == null || raw.isBlank()) {
|
||||||
return 0L;
|
return 0L;
|
||||||
}
|
}
|
||||||
@@ -72,10 +80,14 @@ public class QueryAsinTaskCacheService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public void touchTaskHeartbeat(Long taskId) {
|
public void touchTaskHeartbeat(Long taskId) {
|
||||||
stringRedisTemplate.opsForValue().set(
|
try {
|
||||||
buildTaskHeartbeatKey(taskId),
|
stringRedisTemplate.opsForValue().set(
|
||||||
String.valueOf(Instant.now().toEpochMilli()),
|
buildTaskHeartbeatKey(taskId),
|
||||||
Duration.ofHours(PAYLOAD_TTL_HOURS));
|
String.valueOf(Instant.now().toEpochMilli()),
|
||||||
|
Duration.ofHours(PAYLOAD_TTL_HOURS));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[query-asin-cache] touch heartbeat degraded taskId={} msg={}", taskId, ex.getMessage());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public void deleteTaskCache(Long taskId) {
|
public void deleteTaskCache(Long taskId) {
|
||||||
@@ -83,8 +95,12 @@ public class QueryAsinTaskCacheService {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
taskEntityLocalCache.remove(taskId);
|
taskEntityLocalCache.remove(taskId);
|
||||||
stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId));
|
try {
|
||||||
stringRedisTemplate.delete(buildTaskEntityKey(taskId));
|
stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId));
|
||||||
|
stringRedisTemplate.delete(buildTaskEntityKey(taskId));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[query-asin-cache] delete cache degraded taskId={} msg={}", taskId, ex.getMessage());
|
||||||
|
}
|
||||||
taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE);
|
taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -133,7 +149,13 @@ public class QueryAsinTaskCacheService {
|
|||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
java.util.List<String> keys = missingIds.stream().map(this::buildTaskEntityKey).toList();
|
java.util.List<String> keys = missingIds.stream().map(this::buildTaskEntityKey).toList();
|
||||||
java.util.List<String> values = stringRedisTemplate.opsForValue().multiGet(keys);
|
java.util.List<String> values;
|
||||||
|
try {
|
||||||
|
values = stringRedisTemplate.opsForValue().multiGet(keys);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[query-asin-cache] batch get task cache degraded taskIds={} msg={}", missingIds, ex.getMessage());
|
||||||
|
return result;
|
||||||
|
}
|
||||||
for (int i = 0; i < missingIds.size(); i++) {
|
for (int i = 0; i < missingIds.size(); i++) {
|
||||||
Long taskId = missingIds.get(i);
|
Long taskId = missingIds.get(i);
|
||||||
String val = values != null && i < values.size() ? values.get(i) : null;
|
String val = values != null && i < values.size() ? values.get(i) : null;
|
||||||
|
|||||||
@@ -24,6 +24,10 @@ import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
|||||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskResultItemService;
|
||||||
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
|
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
@@ -45,6 +49,9 @@ public class QueryAsinTaskService {
|
|||||||
|
|
||||||
private static final String MODULE_TYPE = "QUERY_ASIN";
|
private static final String MODULE_TYPE = "QUERY_ASIN";
|
||||||
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
||||||
|
private static final int RESULT_PENDING = -1;
|
||||||
|
private static final int RESULT_FAILED = 0;
|
||||||
|
private static final int RESULT_SUCCESS = 1;
|
||||||
private static final String INTERRUPTED_MESSAGE = "Python 在该店铺结果提交完成前中断";
|
private static final String INTERRUPTED_MESSAGE = "Python 在该店铺结果提交完成前中断";
|
||||||
|
|
||||||
private final FileTaskMapper fileTaskMapper;
|
private final FileTaskMapper fileTaskMapper;
|
||||||
@@ -56,6 +63,9 @@ public class QueryAsinTaskService {
|
|||||||
private final ZiniaoShopSwitchService ziniaoShopSwitchService;
|
private final ZiniaoShopSwitchService ziniaoShopSwitchService;
|
||||||
private final ObjectMapper objectMapper;
|
private final ObjectMapper objectMapper;
|
||||||
private final TaskPressureProperties taskPressureProperties;
|
private final TaskPressureProperties taskPressureProperties;
|
||||||
|
private final TaskFileJobService taskFileJobService;
|
||||||
|
private final TaskResultItemService taskResultItemService;
|
||||||
|
private final TaskProgressSnapshotService taskProgressSnapshotService;
|
||||||
|
|
||||||
private FileTaskEntity loadTaskForExecution(Long taskId) {
|
private FileTaskEntity loadTaskForExecution(Long taskId) {
|
||||||
Map<Long, FileTaskEntity> cachedTasks = taskCacheService.getTaskCacheBatch(List.of(taskId));
|
Map<Long, FileTaskEntity> cachedTasks = taskCacheService.getTaskCacheBatch(List.of(taskId));
|
||||||
@@ -185,8 +195,7 @@ public class QueryAsinTaskService {
|
|||||||
batch.getMissingTaskIds().add(taskId);
|
batch.getMissingTaskIds().add(taskId);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
List<QueryAsinResultItemVo> snapshots = buildSnapshotFromDb(task, taskRows);
|
batch.getItems().addAll(buildProgressItems(task, taskRows));
|
||||||
batch.getItems().addAll(snapshots);
|
|
||||||
}
|
}
|
||||||
return batch;
|
return batch;
|
||||||
}
|
}
|
||||||
@@ -264,12 +273,13 @@ public class QueryAsinTaskService {
|
|||||||
result.setSourceFilename(normalizedShopName);
|
result.setSourceFilename(normalizedShopName);
|
||||||
result.setSourceFileUrl(item.getShopId());
|
result.setSourceFileUrl(item.getShopId());
|
||||||
result.setUserId(request.getUserId());
|
result.setUserId(request.getUserId());
|
||||||
result.setSuccess(null);
|
result.setSuccess(RESULT_PENDING);
|
||||||
result.setCreatedAt(now);
|
result.setCreatedAt(now);
|
||||||
fileResultMapper.insert(result);
|
fileResultMapper.insert(result);
|
||||||
snapshots.add(toSnapshotVo(result, item, task.getStatus(), null));
|
snapshots.add(toSnapshotVo(result, item, task.getStatus(), null));
|
||||||
}
|
}
|
||||||
persistTaskJson(task, uniqueItems, snapshots);
|
persistTaskJson(task, uniqueItems, snapshots);
|
||||||
|
taskCacheService.saveTaskCache(task);
|
||||||
|
|
||||||
QueryAsinCreateTaskVo vo = new QueryAsinCreateTaskVo();
|
QueryAsinCreateTaskVo vo = new QueryAsinCreateTaskVo();
|
||||||
vo.setTaskId(task.getId());
|
vo.setTaskId(task.getId());
|
||||||
@@ -326,6 +336,7 @@ public class QueryAsinTaskService {
|
|||||||
|
|
||||||
persistSnapshotJson(task, snapshots);
|
persistSnapshotJson(task, snapshots);
|
||||||
fileTaskMapper.updateById(task);
|
fileTaskMapper.updateById(task);
|
||||||
|
taskCacheService.saveTaskCache(task);
|
||||||
tryFinalizeTask(taskId, false);
|
tryFinalizeTask(taskId, false);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -402,6 +413,7 @@ public class QueryAsinTaskService {
|
|||||||
|
|
||||||
persistSnapshotJson(task, snapshots);
|
persistSnapshotJson(task, snapshots);
|
||||||
fileTaskMapper.updateById(task);
|
fileTaskMapper.updateById(task);
|
||||||
|
taskCacheService.saveTaskCache(task);
|
||||||
return changed;
|
return changed;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -448,6 +460,7 @@ public class QueryAsinTaskService {
|
|||||||
updateTaskStatusFromRows(task, rows);
|
updateTaskStatusFromRows(task, rows);
|
||||||
persistSnapshotJson(task, buildSnapshotFromDb(task, rows));
|
persistSnapshotJson(task, buildSnapshotFromDb(task, rows));
|
||||||
fileTaskMapper.updateById(task);
|
fileTaskMapper.updateById(task);
|
||||||
|
taskCacheService.saveTaskCache(task);
|
||||||
}
|
}
|
||||||
|
|
||||||
private long countTasks(Long userId, List<String> statuses) {
|
private long countTasks(Long userId, List<String> statuses) {
|
||||||
@@ -502,7 +515,11 @@ public class QueryAsinTaskService {
|
|||||||
private Map<Long, Map<Long, QueryAsinResultItemVo>> buildSnapshotMap(Map<Long, FileTaskEntity> taskMap) {
|
private Map<Long, Map<Long, QueryAsinResultItemVo>> buildSnapshotMap(Map<Long, FileTaskEntity> taskMap) {
|
||||||
Map<Long, Map<Long, QueryAsinResultItemVo>> out = new LinkedHashMap<>();
|
Map<Long, Map<Long, QueryAsinResultItemVo>> out = new LinkedHashMap<>();
|
||||||
for (Map.Entry<Long, FileTaskEntity> entry : taskMap.entrySet()) {
|
for (Map.Entry<Long, FileTaskEntity> entry : taskMap.entrySet()) {
|
||||||
out.put(entry.getKey(), indexSnapshotByResultId(parseTaskSnapshots(entry.getValue().getResultJson())));
|
List<QueryAsinResultItemVo> snapshots = parseTaskSnapshots(entry.getValue().getResultJson());
|
||||||
|
if (snapshots.isEmpty()) {
|
||||||
|
snapshots = taskResultItemService.listResultSnapshots(entry.getKey(), MODULE_TYPE, QueryAsinResultItemVo.class);
|
||||||
|
}
|
||||||
|
out.put(entry.getKey(), indexSnapshotByResultId(snapshots));
|
||||||
}
|
}
|
||||||
return out;
|
return out;
|
||||||
}
|
}
|
||||||
@@ -514,14 +531,13 @@ public class QueryAsinTaskService {
|
|||||||
item.setShopName(firstNonBlank(item.getShopName(), entity.getSourceFilename()));
|
item.setShopName(firstNonBlank(item.getShopName(), entity.getSourceFilename()));
|
||||||
item.setShopId(firstNonBlank(item.getShopId(), entity.getSourceFileUrl()));
|
item.setShopId(firstNonBlank(item.getShopId(), entity.getSourceFileUrl()));
|
||||||
item.setTaskStatus(task != null ? task.getStatus() : item.getTaskStatus());
|
item.setTaskStatus(task != null ? task.getStatus() : item.getTaskStatus());
|
||||||
item.setSuccess(entity.getSuccess() == null ? item.getSuccess() : entity.getSuccess() == 1);
|
item.setSuccess(toSuccessFlag(entity.getSuccess(), item.getSuccess()));
|
||||||
item.setError(!blank(entity.getErrorMessage()) ? entity.getErrorMessage() : item.getError());
|
item.setError(!blank(entity.getErrorMessage()) ? entity.getErrorMessage() : item.getError());
|
||||||
item.setCreatedAt(entity.getCreatedAt());
|
item.setCreatedAt(entity.getCreatedAt());
|
||||||
item.setFinishedAt(task != null ? task.getFinishedAt() : item.getFinishedAt());
|
item.setFinishedAt(task != null ? task.getFinishedAt() : item.getFinishedAt());
|
||||||
item.setOutputFilename(firstNonBlank(item.getOutputFilename(), entity.getResultFilename()));
|
item.setOutputFilename(firstNonBlank(item.getOutputFilename(), entity.getResultFilename()));
|
||||||
item.setDownloadUrl(blank(entity.getResultFileUrl())
|
item.setDownloadUrl(null);
|
||||||
? item.getDownloadUrl()
|
attachFileJobState(item, entity);
|
||||||
: ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl()));
|
|
||||||
if (item.getCountryResults() == null) {
|
if (item.getCountryResults() == null) {
|
||||||
item.setCountryResults(new ArrayList<>());
|
item.setCountryResults(new ArrayList<>());
|
||||||
}
|
}
|
||||||
@@ -531,6 +547,18 @@ public class QueryAsinTaskService {
|
|||||||
return item;
|
return item;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void attachFileJobState(QueryAsinResultItemVo item, FileResultEntity entity) {
|
||||||
|
TaskFileJobEntity job = taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId());
|
||||||
|
item.setFileReady(!blank(entity.getResultFileUrl()));
|
||||||
|
if (job == null) {
|
||||||
|
item.setFileStatus(Boolean.TRUE.equals(item.getFileReady()) ? "SUCCESS" : null);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
item.setFileJobId(job.getId());
|
||||||
|
item.setFileStatus(job.getStatus());
|
||||||
|
item.setFileError(job.getErrorMessage());
|
||||||
|
}
|
||||||
|
|
||||||
private List<QueryAsinTaskItemDto> dedupeItems(List<QueryAsinTaskItemDto> items) {
|
private List<QueryAsinTaskItemDto> dedupeItems(List<QueryAsinTaskItemDto> items) {
|
||||||
LinkedHashMap<String, QueryAsinTaskItemDto> map = new LinkedHashMap<>();
|
LinkedHashMap<String, QueryAsinTaskItemDto> map = new LinkedHashMap<>();
|
||||||
for (QueryAsinTaskItemDto item : items) {
|
for (QueryAsinTaskItemDto item : items) {
|
||||||
@@ -560,7 +588,7 @@ public class QueryAsinTaskService {
|
|||||||
vo.setMatchStatus(item.getMatchStatus());
|
vo.setMatchStatus(item.getMatchStatus());
|
||||||
vo.setMatchMessage(item.getMatchMessage());
|
vo.setMatchMessage(item.getMatchMessage());
|
||||||
vo.setTaskStatus(taskStatus);
|
vo.setTaskStatus(taskStatus);
|
||||||
vo.setSuccess(result.getSuccess() == null ? null : result.getSuccess() == 1);
|
vo.setSuccess(toSuccessFlag(result.getSuccess(), null));
|
||||||
vo.setError(result.getErrorMessage());
|
vo.setError(result.getErrorMessage());
|
||||||
vo.setCreatedAt(result.getCreatedAt());
|
vo.setCreatedAt(result.getCreatedAt());
|
||||||
vo.setFinishedAt(finishedAt);
|
vo.setFinishedAt(finishedAt);
|
||||||
@@ -575,6 +603,7 @@ public class QueryAsinTaskService {
|
|||||||
try {
|
try {
|
||||||
task.setRequestJson(objectMapper.writeValueAsString(requestItems));
|
task.setRequestJson(objectMapper.writeValueAsString(requestItems));
|
||||||
task.setResultJson(objectMapper.writeValueAsString(snapshots));
|
task.setResultJson(objectMapper.writeValueAsString(snapshots));
|
||||||
|
syncSnapshotTables(task, snapshots);
|
||||||
fileTaskMapper.updateById(task);
|
fileTaskMapper.updateById(task);
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
throw new BusinessException("查询ASIN任务快照保存失败");
|
throw new BusinessException("查询ASIN任务快照保存失败");
|
||||||
@@ -591,6 +620,27 @@ public class QueryAsinTaskService {
|
|||||||
return list;
|
return list;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private List<QueryAsinResultItemVo> buildProgressItems(FileTaskEntity task, List<FileResultEntity> rows) {
|
||||||
|
List<QueryAsinResultItemVo> list = new ArrayList<>();
|
||||||
|
for (FileResultEntity row : rows) {
|
||||||
|
QueryAsinResultItemVo item = new QueryAsinResultItemVo();
|
||||||
|
item.setResultId(row.getId());
|
||||||
|
item.setTaskId(row.getTaskId());
|
||||||
|
item.setShopName(row.getSourceFilename());
|
||||||
|
item.setShopId(row.getSourceFileUrl());
|
||||||
|
item.setTaskStatus(task == null ? null : task.getStatus());
|
||||||
|
item.setSuccess(toSuccessFlag(row.getSuccess(), null));
|
||||||
|
item.setError(row.getErrorMessage());
|
||||||
|
item.setCreatedAt(row.getCreatedAt());
|
||||||
|
item.setFinishedAt(task == null ? null : task.getFinishedAt());
|
||||||
|
item.setOutputFilename(row.getResultFilename());
|
||||||
|
item.setDownloadUrl(null);
|
||||||
|
attachFileJobState(item, row);
|
||||||
|
list.add(item);
|
||||||
|
}
|
||||||
|
return list;
|
||||||
|
}
|
||||||
|
|
||||||
private Map<Long, QueryAsinResultItemVo> indexSnapshotByResultId(List<QueryAsinResultItemVo> snapshots) {
|
private Map<Long, QueryAsinResultItemVo> indexSnapshotByResultId(List<QueryAsinResultItemVo> snapshots) {
|
||||||
Map<Long, QueryAsinResultItemVo> map = new LinkedHashMap<>();
|
Map<Long, QueryAsinResultItemVo> map = new LinkedHashMap<>();
|
||||||
if (snapshots == null) {
|
if (snapshots == null) {
|
||||||
@@ -605,8 +655,8 @@ public class QueryAsinTaskService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private void updateTaskStatusFromRows(FileTaskEntity task, List<FileResultEntity> rows) {
|
private void updateTaskStatusFromRows(FileTaskEntity task, List<FileResultEntity> rows) {
|
||||||
long successCount = rows.stream().filter(row -> Integer.valueOf(1).equals(row.getSuccess())).count();
|
long successCount = rows.stream().filter(row -> Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess())).count();
|
||||||
long failedCount = rows.stream().filter(row -> Integer.valueOf(0).equals(row.getSuccess())).count();
|
long failedCount = rows.stream().filter(row -> Integer.valueOf(RESULT_FAILED).equals(row.getSuccess())).count();
|
||||||
long pendingCount = rows.stream().filter(row -> !isResultFinished(row)).count();
|
long pendingCount = rows.stream().filter(row -> !isResultFinished(row)).count();
|
||||||
task.setSuccessFileCount((int) successCount);
|
task.setSuccessFileCount((int) successCount);
|
||||||
task.setFailedFileCount((int) failedCount);
|
task.setFailedFileCount((int) failedCount);
|
||||||
@@ -773,27 +823,25 @@ public class QueryAsinTaskService {
|
|||||||
.filter(item -> Boolean.TRUE.equals(item.getSuccess()))
|
.filter(item -> Boolean.TRUE.equals(item.getSuccess()))
|
||||||
.toList();
|
.toList();
|
||||||
if (!successItems.isEmpty()) {
|
if (!successItems.isEmpty()) {
|
||||||
File workRoot = FileUtil.mkdir(FileUtil.file(System.getProperty("java.io.tmpdir"), "query-asin-result", String.valueOf(task.getId())));
|
String filename = buildTaskWorkbookFilename(task);
|
||||||
String filename = safeFileStem("查询ASIN-" + task.getId()) + ".xlsx";
|
int rowCount = excelAssemblyService.countRows(successItems);
|
||||||
File xlsx = FileUtil.file(workRoot, filename);
|
FileResultEntity firstSuccessRow = null;
|
||||||
try {
|
|
||||||
excelAssemblyService.writeWorkbook(xlsx, successItems);
|
|
||||||
String objectKey = ossStorageService.uploadResultFile(xlsx, MODULE_TYPE);
|
|
||||||
long fileSize = xlsx.length();
|
|
||||||
int rowCount = excelAssemblyService.countRows(successItems);
|
|
||||||
for (FileResultEntity row : rows) {
|
for (FileResultEntity row : rows) {
|
||||||
if (Integer.valueOf(1).equals(row.getSuccess())) {
|
if (Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess())) {
|
||||||
row.setResultFilename(filename);
|
row.setResultFilename(filename);
|
||||||
row.setResultFileUrl(objectKey);
|
row.setResultFileUrl(null);
|
||||||
row.setResultFileSize(fileSize);
|
row.setResultFileSize(0L);
|
||||||
row.setResultContentType(CONTENT_TYPE_XLSX);
|
row.setResultContentType(CONTENT_TYPE_XLSX);
|
||||||
row.setRowCount(rowCount);
|
row.setRowCount(rowCount);
|
||||||
fileResultMapper.updateById(row);
|
fileResultMapper.updateById(row);
|
||||||
|
if (firstSuccessRow == null) {
|
||||||
|
firstSuccessRow = row;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if (firstSuccessRow != null) {
|
||||||
|
taskFileJobService.enqueueAssembleResult(task.getId(), MODULE_TYPE, firstSuccessRow.getId(), "task:" + task.getId());
|
||||||
snapshots = buildSnapshotFromDb(task, rows);
|
snapshots = buildSnapshotFromDb(task, rows);
|
||||||
} finally {
|
|
||||||
FileUtil.del(xlsx);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -802,14 +850,56 @@ public class QueryAsinTaskService {
|
|||||||
taskCacheService.deleteTaskCache(task.getId());
|
taskCacheService.deleteTaskCache(task.getId());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void processResultFileJob(TaskFileJobEntity job) {
|
||||||
|
if (job == null || job.getTaskId() == null) {
|
||||||
|
throw new BusinessException("结果文件任务参数不完整");
|
||||||
|
}
|
||||||
|
FileTaskEntity task = fileTaskMapper.selectById(job.getTaskId());
|
||||||
|
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
|
||||||
|
throw new BusinessException("任务不存在");
|
||||||
|
}
|
||||||
|
List<FileResultEntity> rows = listTaskRows(job.getTaskId());
|
||||||
|
List<QueryAsinResultItemVo> snapshots = taskResultItemService.listResultSnapshots(job.getTaskId(), MODULE_TYPE, QueryAsinResultItemVo.class);
|
||||||
|
if (snapshots.isEmpty()) {
|
||||||
|
snapshots = buildSnapshotFromDb(task, rows);
|
||||||
|
}
|
||||||
|
List<QueryAsinResultItemVo> successItems = snapshots.stream()
|
||||||
|
.filter(item -> Boolean.TRUE.equals(item.getSuccess()))
|
||||||
|
.toList();
|
||||||
|
if (successItems.isEmpty()) {
|
||||||
|
throw new BusinessException("没有可生成的查询ASIN结果");
|
||||||
|
}
|
||||||
|
File workRoot = FileUtil.mkdir(FileUtil.file(System.getProperty("java.io.tmpdir"), "query-asin-result", String.valueOf(task.getId())));
|
||||||
|
String filename = buildTaskWorkbookFilename(task);
|
||||||
|
File xlsx = FileUtil.file(workRoot, filename);
|
||||||
|
try {
|
||||||
|
excelAssemblyService.writeWorkbook(xlsx, successItems);
|
||||||
|
String objectKey = ossStorageService.uploadResultFile(xlsx, MODULE_TYPE);
|
||||||
|
long fileSize = xlsx.length();
|
||||||
|
int rowCount = excelAssemblyService.countRows(successItems);
|
||||||
|
for (FileResultEntity row : rows) {
|
||||||
|
if (Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess())) {
|
||||||
|
row.setResultFilename(filename);
|
||||||
|
row.setResultFileUrl(objectKey);
|
||||||
|
row.setResultFileSize(fileSize);
|
||||||
|
row.setResultContentType(CONTENT_TYPE_XLSX);
|
||||||
|
row.setRowCount(rowCount);
|
||||||
|
fileResultMapper.updateById(row);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
FileUtil.del(xlsx);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private void markResultSuccess(FileResultEntity row) {
|
private void markResultSuccess(FileResultEntity row) {
|
||||||
row.setSuccess(1);
|
row.setSuccess(RESULT_SUCCESS);
|
||||||
row.setErrorMessage(null);
|
row.setErrorMessage(null);
|
||||||
fileResultMapper.updateById(row);
|
fileResultMapper.updateById(row);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void markResultFailed(FileResultEntity row, String message) {
|
private void markResultFailed(FileResultEntity row, String message) {
|
||||||
row.setSuccess(0);
|
row.setSuccess(RESULT_FAILED);
|
||||||
row.setErrorMessage(blankToNull(message));
|
row.setErrorMessage(blankToNull(message));
|
||||||
row.setResultFilename(null);
|
row.setResultFilename(null);
|
||||||
row.setResultFileUrl(null);
|
row.setResultFileUrl(null);
|
||||||
@@ -820,7 +910,14 @@ public class QueryAsinTaskService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private boolean isResultFinished(FileResultEntity row) {
|
private boolean isResultFinished(FileResultEntity row) {
|
||||||
return Integer.valueOf(1).equals(row.getSuccess()) || Integer.valueOf(0).equals(row.getSuccess());
|
return Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess()) || Integer.valueOf(RESULT_FAILED).equals(row.getSuccess());
|
||||||
|
}
|
||||||
|
|
||||||
|
private Boolean toSuccessFlag(Integer dbValue, Boolean fallback) {
|
||||||
|
if (dbValue == null || Integer.valueOf(RESULT_PENDING).equals(dbValue)) {
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
return Integer.valueOf(RESULT_SUCCESS).equals(dbValue);
|
||||||
}
|
}
|
||||||
|
|
||||||
private List<QueryAsinResultItemVo> parseTaskSnapshots(String json) {
|
private List<QueryAsinResultItemVo> parseTaskSnapshots(String json) {
|
||||||
@@ -838,11 +935,39 @@ public class QueryAsinTaskService {
|
|||||||
private void persistSnapshotJson(FileTaskEntity task, List<QueryAsinResultItemVo> snapshots) {
|
private void persistSnapshotJson(FileTaskEntity task, List<QueryAsinResultItemVo> snapshots) {
|
||||||
try {
|
try {
|
||||||
task.setResultJson(objectMapper.writeValueAsString(snapshots == null ? List.of() : snapshots));
|
task.setResultJson(objectMapper.writeValueAsString(snapshots == null ? List.of() : snapshots));
|
||||||
|
syncSnapshotTables(task, snapshots);
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
throw new BusinessException("查询ASIN任务快照保存失败");
|
throw new BusinessException("查询ASIN任务快照保存失败");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void syncSnapshotTables(FileTaskEntity task, List<QueryAsinResultItemVo> snapshots) {
|
||||||
|
List<QueryAsinResultItemVo> safe = snapshots == null ? List.of() : snapshots;
|
||||||
|
taskResultItemService.replaceTaskSnapshots(task.getId(), MODULE_TYPE, safe, new TaskResultItemService.SnapshotKeyResolver() {
|
||||||
|
@Override
|
||||||
|
public Long resultId(Object snapshot) {
|
||||||
|
return ((QueryAsinResultItemVo) snapshot).getResultId();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String scopeKey(Object snapshot) {
|
||||||
|
QueryAsinResultItemVo item = (QueryAsinResultItemVo) snapshot;
|
||||||
|
return firstNonBlank(item.getShopName(), "result:" + item.getResultId());
|
||||||
|
}
|
||||||
|
});
|
||||||
|
int successCount = 0;
|
||||||
|
int failedCount = 0;
|
||||||
|
for (QueryAsinResultItemVo item : safe) {
|
||||||
|
if (Boolean.TRUE.equals(item.getSuccess())) {
|
||||||
|
successCount++;
|
||||||
|
} else if (Boolean.FALSE.equals(item.getSuccess())) {
|
||||||
|
failedCount++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
taskProgressSnapshotService.save(task.getId(), MODULE_TYPE, firstNonBlank(task.getStatus(), "RUNNING"),
|
||||||
|
safe.size(), successCount, failedCount, null, task.getErrorMessage(), null);
|
||||||
|
}
|
||||||
|
|
||||||
private List<QueryAsinCountryResultDto> copyCountryResults(List<QueryAsinCountryResultDto> results) {
|
private List<QueryAsinCountryResultDto> copyCountryResults(List<QueryAsinCountryResultDto> results) {
|
||||||
List<QueryAsinCountryResultDto> copy = new ArrayList<>();
|
List<QueryAsinCountryResultDto> copy = new ArrayList<>();
|
||||||
if (results == null) {
|
if (results == null) {
|
||||||
@@ -923,6 +1048,11 @@ public class QueryAsinTaskService {
|
|||||||
return asin == null ? "" : asin.trim().toUpperCase();
|
return asin == null ? "" : asin.trim().toUpperCase();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private String buildTaskWorkbookFilename(FileTaskEntity task) {
|
||||||
|
Long taskId = task == null ? null : task.getId();
|
||||||
|
return safeFileStem("查询ASIN-" + (taskId == null ? "result" : taskId)) + ".xlsx";
|
||||||
|
}
|
||||||
|
|
||||||
private String safeFileStem(String value) {
|
private String safeFileStem(String value) {
|
||||||
String raw = value == null ? "result" : value.trim();
|
String raw = value == null ? "result" : value.trim();
|
||||||
String safe = raw.replaceAll("[\\\\/:*?\"<>|]+", "_");
|
String safe = raw.replaceAll("[\\\\/:*?\"<>|]+", "_");
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ import com.nanri.aiimage.modules.shopkey.model.vo.QueryAsinItemVo;
|
|||||||
import com.nanri.aiimage.modules.shopkey.model.vo.QueryAsinPageVo;
|
import com.nanri.aiimage.modules.shopkey.model.vo.QueryAsinPageVo;
|
||||||
import com.nanri.aiimage.modules.shopkey.model.vo.ShopManageGroupItemVo;
|
import com.nanri.aiimage.modules.shopkey.model.vo.ShopManageGroupItemVo;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
import org.springframework.transaction.annotation.Transactional;
|
import org.springframework.transaction.annotation.Transactional;
|
||||||
import org.springframework.web.multipart.MultipartFile;
|
import org.springframework.web.multipart.MultipartFile;
|
||||||
@@ -30,9 +31,11 @@ import java.util.Locale;
|
|||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.concurrent.ConcurrentHashMap;
|
import java.util.concurrent.ConcurrentHashMap;
|
||||||
|
import java.util.function.Predicate;
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
|
@Slf4j
|
||||||
public class QueryAsinService {
|
public class QueryAsinService {
|
||||||
|
|
||||||
private static final List<String> SUPPORTED_COUNTRIES = List.of("DE", "UK", "FR", "IT", "ES");
|
private static final List<String> SUPPORTED_COUNTRIES = List.of("DE", "UK", "FR", "IT", "ES");
|
||||||
@@ -364,13 +367,32 @@ public class QueryAsinService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private String normalizeCountry(String country) {
|
private String normalizeCountry(String country) {
|
||||||
String normalized = normalizeBlank(country).toUpperCase(Locale.ROOT);
|
String normalized = normalizeCountryAlias(country);
|
||||||
if (!SUPPORTED_COUNTRIES.contains(normalized)) {
|
if (!SUPPORTED_COUNTRIES.contains(normalized)) {
|
||||||
throw new BusinessException("国家参数不支持: " + country);
|
throw new BusinessException("国家参数不支持: " + country);
|
||||||
}
|
}
|
||||||
return normalized;
|
return normalized;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private String normalizeCountryAlias(String country) {
|
||||||
|
String normalized = normalizeBlank(country);
|
||||||
|
if (normalized.isEmpty()) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
String upper = normalized.toUpperCase(Locale.ROOT);
|
||||||
|
if (SUPPORTED_COUNTRIES.contains(upper)) {
|
||||||
|
return upper;
|
||||||
|
}
|
||||||
|
return switch (normalizeHeaderKey(normalized)) {
|
||||||
|
case "\u5fb7\u56fd", "\u5fb7", "de", "germany", "german", "deutschland" -> "DE";
|
||||||
|
case "\u82f1\u56fd", "\u82f1", "uk", "gb", "greatbritain", "britain", "unitedkingdom" -> "UK";
|
||||||
|
case "\u6cd5\u56fd", "\u6cd5", "fr", "france", "french" -> "FR";
|
||||||
|
case "\u610f\u5927\u5229", "\u610f", "it", "italy", "italian" -> "IT";
|
||||||
|
case "\u897f\u73ed\u7259", "\u897f", "es", "spain", "spanish", "espana" -> "ES";
|
||||||
|
default -> "";
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
private String normalizeAsin(String asin) {
|
private String normalizeAsin(String asin) {
|
||||||
String normalized = normalizeRequired(asin, "ASIN 不能为空").toUpperCase(Locale.ROOT);
|
String normalized = normalizeRequired(asin, "ASIN 不能为空").toUpperCase(Locale.ROOT);
|
||||||
if (normalized.length() > 64) {
|
if (normalized.length() > 64) {
|
||||||
@@ -505,13 +527,19 @@ public class QueryAsinService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
headerLoaded = true;
|
headerLoaded = true;
|
||||||
if (findGroupIndex() == null && (fallbackGroupId == null || fallbackGroupId <= 0)) {
|
Integer resolvedGroupIndex = resolveGroupHeaderIndex();
|
||||||
|
Integer resolvedShopIndex = resolveShopHeaderIndex();
|
||||||
|
boolean hasWideCountries = hasWideCountryColumns();
|
||||||
|
boolean hasSingleCountryAsin = hasSingleCountryAsinColumns();
|
||||||
|
log.info("[query-asin-import] headerMap={} normalizedKeys={} groupIndex={} shopIndex={} hasWideCountries={} hasSingleCountryAsin={}",
|
||||||
|
headerMap, headerIndexByKey.keySet(), resolvedGroupIndex, resolvedShopIndex, hasWideCountries, hasSingleCountryAsin);
|
||||||
|
if (resolvedGroupIndex == null && (fallbackGroupId == null || fallbackGroupId <= 0)) {
|
||||||
throw new BusinessException("Excel 缺少分组列,请在页面选择分组或提供分组列");
|
throw new BusinessException("Excel 缺少分组列,请在页面选择分组或提供分组列");
|
||||||
}
|
}
|
||||||
if (findShopIndex() == null) {
|
if (resolvedShopIndex == null) {
|
||||||
throw new BusinessException("Excel 缺少店铺名列");
|
throw new BusinessException("Excel 缺少店铺名列");
|
||||||
}
|
}
|
||||||
if (SUPPORTED_COUNTRIES.stream().noneMatch(country -> findCountryIndex(country) != null)) {
|
if (!hasWideCountries && !hasSingleCountryAsin) {
|
||||||
throw new BusinessException("Excel 至少需要包含一个国家 ASIN 列");
|
throw new BusinessException("Excel 至少需要包含一个国家 ASIN 列");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -524,7 +552,7 @@ public class QueryAsinService {
|
|||||||
progress.setProcessedRows(rowIndex);
|
progress.setProcessedRows(rowIndex);
|
||||||
|
|
||||||
Long groupId = resolveGroupId(rowMap);
|
Long groupId = resolveGroupId(rowMap);
|
||||||
String shopName = cell(rowMap, findShopIndex());
|
String shopName = cell(rowMap, resolveShopHeaderIndex());
|
||||||
if (groupId == null || shopName.isEmpty()) {
|
if (groupId == null || shopName.isEmpty()) {
|
||||||
skippedCount++;
|
skippedCount++;
|
||||||
updateProgress();
|
updateProgress();
|
||||||
@@ -532,27 +560,8 @@ public class QueryAsinService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
QueryAsinEntity row = newImportEntity(groupId, shopName);
|
QueryAsinEntity row = newImportEntity(groupId, shopName);
|
||||||
int rowAsinCount = 0;
|
int rowAcceptedCount = collectRowCountryAsins(row, rowMap);
|
||||||
int rowAcceptedCount = 0;
|
if (countCountryCells(row) == 0) {
|
||||||
for (String country : SUPPORTED_COUNTRIES) {
|
|
||||||
Integer index = findCountryIndex(country);
|
|
||||||
if (index == null) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
String asin = normalizeBlank(cell(rowMap, index)).toUpperCase(Locale.ROOT);
|
|
||||||
if (asin.isEmpty()) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (asin.length() > 64) {
|
|
||||||
skippedCount++;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
asinCount++;
|
|
||||||
rowAsinCount++;
|
|
||||||
setCountryAsin(row, country, asin);
|
|
||||||
rowAcceptedCount++;
|
|
||||||
}
|
|
||||||
if (rowAsinCount == 0) {
|
|
||||||
skippedCount++;
|
skippedCount++;
|
||||||
}
|
}
|
||||||
if (rowAcceptedCount > 0) {
|
if (rowAcceptedCount > 0) {
|
||||||
@@ -754,7 +763,7 @@ public class QueryAsinService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private Long resolveGroupId(Map<Integer, String> rowMap) {
|
private Long resolveGroupId(Map<Integer, String> rowMap) {
|
||||||
Integer groupIndex = findGroupIndex();
|
Integer groupIndex = resolveGroupHeaderIndex();
|
||||||
String groupText = cell(rowMap, groupIndex);
|
String groupText = cell(rowMap, groupIndex);
|
||||||
if (groupText.isEmpty()) {
|
if (groupText.isEmpty()) {
|
||||||
return fallbackGroupId != null && fallbackGroupId > 0 ? fallbackGroupId : null;
|
return fallbackGroupId != null && fallbackGroupId > 0 ? fallbackGroupId : null;
|
||||||
@@ -767,6 +776,51 @@ public class QueryAsinService {
|
|||||||
return id;
|
return id;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private int collectRowCountryAsins(QueryAsinEntity row, Map<Integer, String> rowMap) {
|
||||||
|
int acceptedCount = 0;
|
||||||
|
for (String country : SUPPORTED_COUNTRIES) {
|
||||||
|
Integer index = resolveCountryHeaderIndex(country);
|
||||||
|
if (index == null) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
acceptedCount += acceptCountryAsin(row, country, cell(rowMap, index));
|
||||||
|
}
|
||||||
|
Integer singleCountryIndex = findSingleCountryIndex();
|
||||||
|
Integer singleAsinIndex = findSingleAsinIndex();
|
||||||
|
if (singleCountryIndex != null && singleAsinIndex != null) {
|
||||||
|
acceptedCount += acceptCountryAsin(
|
||||||
|
row,
|
||||||
|
normalizeCountryAlias(cell(rowMap, singleCountryIndex)),
|
||||||
|
cell(rowMap, singleAsinIndex));
|
||||||
|
}
|
||||||
|
return acceptedCount;
|
||||||
|
}
|
||||||
|
|
||||||
|
private int acceptCountryAsin(QueryAsinEntity row, String country, String asin) {
|
||||||
|
String normalizedAsin = normalizeBlank(asin).toUpperCase(Locale.ROOT);
|
||||||
|
if (normalizedAsin.isEmpty()) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
asinCount++;
|
||||||
|
if (country == null || country.isEmpty() || !SUPPORTED_COUNTRIES.contains(country)) {
|
||||||
|
skippedCount++;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
if (normalizedAsin.length() > 64) {
|
||||||
|
skippedCount++;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
String existing = getCountryAsin(row, country);
|
||||||
|
if (!existing.isEmpty()) {
|
||||||
|
if (!existing.equals(normalizedAsin)) {
|
||||||
|
skippedCount++;
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
setCountryAsin(row, country, normalizedAsin);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
private void loadGroups() {
|
private void loadGroups() {
|
||||||
if (groupsLoaded) {
|
if (groupsLoaded) {
|
||||||
return;
|
return;
|
||||||
@@ -801,6 +855,88 @@ public class QueryAsinService {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private Integer resolveGroupHeaderIndex() {
|
||||||
|
Integer index = findGroupIndex();
|
||||||
|
if (index != null) {
|
||||||
|
return index;
|
||||||
|
}
|
||||||
|
index = firstHeader("groupname", "\u5206\u7ec4id", "\u5206\u7ec4");
|
||||||
|
if (index != null) {
|
||||||
|
return index;
|
||||||
|
}
|
||||||
|
return findHeaderByPredicate(key -> key.contains("\u5206\u7ec4") || key.contains("group"));
|
||||||
|
}
|
||||||
|
|
||||||
|
private Integer resolveShopHeaderIndex() {
|
||||||
|
Integer index = findShopIndex();
|
||||||
|
if (index != null) {
|
||||||
|
return index;
|
||||||
|
}
|
||||||
|
index = firstHeader(
|
||||||
|
"\u5e97\u94fa\u540d", "\u5e97\u94fa\u540d\u79f0", "\u5e97\u94fa\u8d26\u53f7", "\u8d26\u53f7",
|
||||||
|
"account", "selleraccount", "storeaccount", "storename");
|
||||||
|
if (index != null) {
|
||||||
|
return index;
|
||||||
|
}
|
||||||
|
return findHeaderByPredicate(key ->
|
||||||
|
(key.contains("\u5e97\u94fa") && (key.contains("\u540d") || key.contains("\u8d26\u53f7")))
|
||||||
|
|| (key.contains("shop") && (key.contains("name") || key.contains("account")))
|
||||||
|
|| key.contains("selleraccount")
|
||||||
|
|| key.contains("storeaccount"));
|
||||||
|
}
|
||||||
|
|
||||||
|
private Integer resolveCountryHeaderIndex(String country) {
|
||||||
|
Integer index = findCountryIndex(country);
|
||||||
|
if (index != null) {
|
||||||
|
return index;
|
||||||
|
}
|
||||||
|
return switch (country) {
|
||||||
|
case "DE" -> firstHeader("\u5fb7\u56fd", "\u5fb7", "\u5fb7\u56fdasin");
|
||||||
|
case "UK" -> firstHeader("\u82f1\u56fd", "\u82f1", "\u82f1\u56fdasin");
|
||||||
|
case "FR" -> firstHeader("\u6cd5\u56fd", "\u6cd5", "\u6cd5\u56fdasin");
|
||||||
|
case "IT" -> firstHeader("\u610f\u5927\u5229", "\u610f", "\u610f\u5927\u5229asin");
|
||||||
|
case "ES" -> firstHeader("\u897f\u73ed\u7259", "\u897f", "\u897f\u73ed\u7259asin");
|
||||||
|
default -> null;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private Integer findSingleCountryIndex() {
|
||||||
|
Integer index = firstHeader(
|
||||||
|
"\u4e2d\u6587\u56fd\u5bb6\u540d", "\u56fd\u5bb6", "\u56fd\u5bb6\u540d", "\u56fd\u5bb6\u7ad9\u70b9",
|
||||||
|
"\u7ad9\u70b9", "country", "countryname", "marketplace", "site");
|
||||||
|
if (index != null) {
|
||||||
|
return index;
|
||||||
|
}
|
||||||
|
return findHeaderByPredicate(key ->
|
||||||
|
key.contains("\u56fd\u5bb6")
|
||||||
|
|| key.contains("\u7ad9\u70b9")
|
||||||
|
|| key.contains("country")
|
||||||
|
|| key.contains("marketplace")
|
||||||
|
|| key.equals("site"));
|
||||||
|
}
|
||||||
|
|
||||||
|
private Integer findSingleAsinIndex() {
|
||||||
|
return firstHeader("asin", "\u67e5\u8be2asin", "\u4ea7\u54c1asin", "\u94fe\u63a5asin");
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean hasWideCountryColumns() {
|
||||||
|
return SUPPORTED_COUNTRIES.stream().anyMatch(country -> resolveCountryHeaderIndex(country) != null);
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean hasSingleCountryAsinColumns() {
|
||||||
|
return findSingleCountryIndex() != null && findSingleAsinIndex() != null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private Integer findHeaderByPredicate(Predicate<String> predicate) {
|
||||||
|
for (Map.Entry<String, Integer> entry : headerIndexByKey.entrySet()) {
|
||||||
|
String key = entry.getKey();
|
||||||
|
if (key != null && predicate.test(key)) {
|
||||||
|
return entry.getValue();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
private Integer firstHeader(String... names) {
|
private Integer firstHeader(String... names) {
|
||||||
for (String name : names) {
|
for (String name : names) {
|
||||||
Integer index = headerIndexByKey.get(normalizeHeaderKey(name));
|
Integer index = headerIndexByKey.get(normalizeHeaderKey(name));
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ import java.time.Duration;
|
|||||||
import java.time.Instant;
|
import java.time.Instant;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.LinkedHashMap;
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.concurrent.ConcurrentHashMap;
|
import java.util.concurrent.ConcurrentHashMap;
|
||||||
|
|
||||||
@@ -38,17 +39,27 @@ public class ShopMatchTaskCacheService {
|
|||||||
if (taskId == null || taskId <= 0) {
|
if (taskId == null || taskId <= 0) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
stringRedisTemplate.opsForValue().set(
|
try {
|
||||||
buildTaskHeartbeatKey(taskId),
|
stringRedisTemplate.opsForValue().set(
|
||||||
String.valueOf(Instant.now().toEpochMilli()),
|
buildTaskHeartbeatKey(taskId),
|
||||||
Duration.ofHours(PAYLOAD_TTL_HOURS));
|
String.valueOf(Instant.now().toEpochMilli()),
|
||||||
|
Duration.ofHours(PAYLOAD_TTL_HOURS));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[shop-match-cache] touch heartbeat degraded taskId={} msg={}", taskId, ex.getMessage());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public long getTaskHeartbeatMillis(Long taskId) {
|
public long getTaskHeartbeatMillis(Long taskId) {
|
||||||
if (taskId == null || taskId <= 0) {
|
if (taskId == null || taskId <= 0) {
|
||||||
return 0L;
|
return 0L;
|
||||||
}
|
}
|
||||||
String raw = stringRedisTemplate.opsForValue().get(buildTaskHeartbeatKey(taskId));
|
String raw;
|
||||||
|
try {
|
||||||
|
raw = stringRedisTemplate.opsForValue().get(buildTaskHeartbeatKey(taskId));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[shop-match-cache] get heartbeat degraded taskId={} msg={}", taskId, ex.getMessage());
|
||||||
|
return 0L;
|
||||||
|
}
|
||||||
if (raw == null || raw.isBlank()) {
|
if (raw == null || raw.isBlank()) {
|
||||||
return 0L;
|
return 0L;
|
||||||
}
|
}
|
||||||
@@ -59,6 +70,41 @@ public class ShopMatchTaskCacheService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public Map<Long, Long> getTaskHeartbeatMillisBatch(List<Long> taskIds) {
|
||||||
|
Map<Long, Long> result = new LinkedHashMap<>();
|
||||||
|
if (taskIds == null || taskIds.isEmpty()) {
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
List<Long> normalized = taskIds.stream()
|
||||||
|
.filter(id -> id != null && id > 0)
|
||||||
|
.distinct()
|
||||||
|
.toList();
|
||||||
|
if (normalized.isEmpty()) {
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
List<String> keys = normalized.stream().map(this::buildTaskHeartbeatKey).toList();
|
||||||
|
List<String> values;
|
||||||
|
try {
|
||||||
|
values = stringRedisTemplate.opsForValue().multiGet(keys);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[shop-match-cache] batch get heartbeat degraded taskIds={} msg={}", normalized, ex.getMessage());
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
for (int i = 0; i < normalized.size(); i++) {
|
||||||
|
String raw = values != null && i < values.size() ? values.get(i) : null;
|
||||||
|
if (raw == null || raw.isBlank()) {
|
||||||
|
result.put(normalized.get(i), 0L);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
result.put(normalized.get(i), Long.parseLong(raw));
|
||||||
|
} catch (NumberFormatException ignored) {
|
||||||
|
result.put(normalized.get(i), 0L);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
public ShopMatchShopPayloadDto getShopMergedPayload(Long taskId, String shopKey) {
|
public ShopMatchShopPayloadDto getShopMergedPayload(Long taskId, String shopKey) {
|
||||||
return taskScopePayloadStorageService.getScopePayload(taskId, MODULE_TYPE, shopKey, ShopMatchShopPayloadDto.class);
|
return taskScopePayloadStorageService.getScopePayload(taskId, MODULE_TYPE, shopKey, ShopMatchShopPayloadDto.class);
|
||||||
}
|
}
|
||||||
@@ -88,7 +134,11 @@ public class ShopMatchTaskCacheService {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
taskEntityLocalCache.remove(taskId);
|
taskEntityLocalCache.remove(taskId);
|
||||||
stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId));
|
try {
|
||||||
|
stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[shop-match-cache] delete heartbeat degraded taskId={} msg={}", taskId, ex.getMessage());
|
||||||
|
}
|
||||||
taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE);
|
taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE);
|
||||||
try {
|
try {
|
||||||
Path taskDir = buildTaskDir(taskId);
|
Path taskDir = buildTaskDir(taskId);
|
||||||
|
|||||||
@@ -28,6 +28,9 @@ import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
|||||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskResultPayloadService;
|
||||||
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
|
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
@@ -62,6 +65,8 @@ public class ShopMatchTaskService {
|
|||||||
private final ObjectMapper objectMapper;
|
private final ObjectMapper objectMapper;
|
||||||
private final ShopMatchTaskCacheService shopMatchTaskCacheService;
|
private final ShopMatchTaskCacheService shopMatchTaskCacheService;
|
||||||
private final TaskPressureProperties taskPressureProperties;
|
private final TaskPressureProperties taskPressureProperties;
|
||||||
|
private final TaskResultPayloadService taskResultPayloadService;
|
||||||
|
private final TaskFileJobService taskFileJobService;
|
||||||
|
|
||||||
private FileTaskEntity loadTaskForExecution(Long taskId) {
|
private FileTaskEntity loadTaskForExecution(Long taskId) {
|
||||||
Map<Long, FileTaskEntity> cachedTasks = shopMatchTaskCacheService.getTaskCacheBatch(List.of(taskId));
|
Map<Long, FileTaskEntity> cachedTasks = shopMatchTaskCacheService.getTaskCacheBatch(List.of(taskId));
|
||||||
@@ -409,6 +414,12 @@ public class ShopMatchTaskService {
|
|||||||
if (state.getScheduleTimes() == null || stageIndex >= state.getScheduleTimes().size()) {
|
if (state.getScheduleTimes() == null || stageIndex >= state.getScheduleTimes().size()) {
|
||||||
throw new BusinessException("轮次配置缺失");
|
throw new BusinessException("轮次配置缺失");
|
||||||
}
|
}
|
||||||
|
FileTaskEntity runningTask = findOtherRunningTask(userId, taskId);
|
||||||
|
if (runningTask != null) {
|
||||||
|
log.warn("[shop-match] activate serialized taskId={} stageIndex={} blockedByTaskId={} blockedUpdatedAt={}",
|
||||||
|
taskId, stageIndex, runningTask.getId(), runningTask.getUpdatedAt());
|
||||||
|
throw new BusinessException("当前已有定时匹配任务执行中,请等待上一任务执行结束后再重试");
|
||||||
|
}
|
||||||
state.setActiveStageIndex(stageIndex);
|
state.setActiveStageIndex(stageIndex);
|
||||||
task.setStatus("RUNNING");
|
task.setStatus("RUNNING");
|
||||||
task.setUpdatedAt(now);
|
task.setUpdatedAt(now);
|
||||||
@@ -511,7 +522,7 @@ public class ShopMatchTaskService {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
assembleShopResult(result, shopKey, merged, workRoot);
|
enqueueResultFileAssembly(result, shopKey, merged);
|
||||||
shopMatchTaskCacheService.removeShopMergedPayload(taskId, shopKey);
|
shopMatchTaskCacheService.removeShopMergedPayload(taskId, shopKey);
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
markResultFailed(result, ex.getMessage() == null ? "结果组装失败" : ex.getMessage());
|
markResultFailed(result, ex.getMessage() == null ? "结果组装失败" : ex.getMessage());
|
||||||
@@ -603,7 +614,7 @@ public class ShopMatchTaskService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
assembleShopResult(result, shopKey, cachedPayload, workRoot);
|
enqueueResultFileAssembly(result, shopKey, cachedPayload);
|
||||||
shopMatchTaskCacheService.removeShopMergedPayload(taskId, shopKey);
|
shopMatchTaskCacheService.removeShopMergedPayload(taskId, shopKey);
|
||||||
changed = true;
|
changed = true;
|
||||||
log.warn("[shop-match] stale finalize assembled taskId={} shop={} fromCompensation={} completedFlag={} rowCount={}",
|
log.warn("[shop-match] stale finalize assembled taskId={} shop={} fromCompensation={} completedFlag={} rowCount={}",
|
||||||
@@ -651,6 +662,43 @@ public class ShopMatchTaskService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void processResultFileJob(TaskFileJobEntity job) {
|
||||||
|
if (job == null || job.getTaskId() == null || job.getResultId() == null || job.getScopeKey() == null) {
|
||||||
|
throw new BusinessException("结果文件任务参数不完整");
|
||||||
|
}
|
||||||
|
FileResultEntity result = fileResultMapper.selectById(job.getResultId());
|
||||||
|
if (result == null || !MODULE_TYPE.equals(result.getModuleType())) {
|
||||||
|
throw new BusinessException("结果记录不存在");
|
||||||
|
}
|
||||||
|
ShopMatchShopPayloadDto payload = taskResultPayloadService.getLatest(
|
||||||
|
job.getTaskId(), MODULE_TYPE, job.getScopeKey(), ShopMatchShopPayloadDto.class);
|
||||||
|
if (payload == null) {
|
||||||
|
throw new BusinessException("结果文件载荷不存在");
|
||||||
|
}
|
||||||
|
File workRoot = FileUtil.mkdir(FileUtil.file(System.getProperty("java.io.tmpdir"), "shop-match-result", String.valueOf(job.getTaskId())));
|
||||||
|
assembleShopResult(result, job.getScopeKey(), payload, workRoot);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void enqueueResultFileAssembly(FileResultEntity result, String shopKey, ShopMatchShopPayloadDto payload) {
|
||||||
|
taskResultPayloadService.saveLatest(result.getTaskId(), MODULE_TYPE, shopKey, payload);
|
||||||
|
markResultFilePending(result, shopKey, payload);
|
||||||
|
taskFileJobService.enqueueAssembleResult(result.getTaskId(), MODULE_TYPE, result.getId(), shopKey);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void markResultFilePending(FileResultEntity result, String shopKey, ShopMatchShopPayloadDto payload) {
|
||||||
|
Map<String, List<ShopMatchRowDto>> countries = excelAssemblyService.normalizeCountriesMap(payload.getCountries());
|
||||||
|
String displayName = payload.getShopName() != null && !payload.getShopName().isBlank() ? payload.getShopName().trim() : shopKey;
|
||||||
|
String stem = safeFileStem(displayName);
|
||||||
|
result.setResultFilename(stem + ".xlsx");
|
||||||
|
result.setResultFileUrl(null);
|
||||||
|
result.setResultFileSize(0L);
|
||||||
|
result.setResultContentType(CONTENT_TYPE_XLSX);
|
||||||
|
result.setRowCount(excelAssemblyService.countRows(countries));
|
||||||
|
result.setSuccess(1);
|
||||||
|
result.setErrorMessage(null);
|
||||||
|
fileResultMapper.updateById(result);
|
||||||
|
}
|
||||||
|
|
||||||
private void markResultFailed(FileResultEntity result, String message) {
|
private void markResultFailed(FileResultEntity result, String message) {
|
||||||
result.setSuccess(0);
|
result.setSuccess(0);
|
||||||
result.setErrorMessage(message);
|
result.setErrorMessage(message);
|
||||||
@@ -816,9 +864,8 @@ public class ShopMatchTaskService {
|
|||||||
vo.setSuccess(success);
|
vo.setSuccess(success);
|
||||||
vo.setError(entity.getErrorMessage());
|
vo.setError(entity.getErrorMessage());
|
||||||
vo.setOutputFilename(entity.getResultFilename());
|
vo.setOutputFilename(entity.getResultFilename());
|
||||||
vo.setDownloadUrl(entity.getResultFileUrl() == null || entity.getResultFileUrl().isBlank()
|
vo.setDownloadUrl(null);
|
||||||
? null
|
attachFileJobState(vo, entity);
|
||||||
: ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl()));
|
|
||||||
vo.setMatched(true);
|
vo.setMatched(true);
|
||||||
if (entity.getTaskId() != null) {
|
if (entity.getTaskId() != null) {
|
||||||
mergeQueueFieldsFromRequest(vo, entity.getTaskId());
|
mergeQueueFieldsFromRequest(vo, entity.getTaskId());
|
||||||
@@ -826,6 +873,18 @@ public class ShopMatchTaskService {
|
|||||||
return vo;
|
return vo;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void attachFileJobState(ProductRiskResultItemVo vo, FileResultEntity entity) {
|
||||||
|
TaskFileJobEntity job = taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId());
|
||||||
|
vo.setFileReady(entity.getResultFileUrl() != null && !entity.getResultFileUrl().isBlank());
|
||||||
|
if (job == null) {
|
||||||
|
vo.setFileStatus(Boolean.TRUE.equals(vo.getFileReady()) ? "SUCCESS" : null);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
vo.setFileJobId(job.getId());
|
||||||
|
vo.setFileStatus(job.getStatus());
|
||||||
|
vo.setFileError(job.getErrorMessage());
|
||||||
|
}
|
||||||
|
|
||||||
private List<ShopMatchTaskStageVo> buildStageVos(FileTaskEntity task, ShopMatchCreateTaskRequest request) {
|
private List<ShopMatchTaskStageVo> buildStageVos(FileTaskEntity task, ShopMatchCreateTaskRequest request) {
|
||||||
List<ShopMatchTaskStageVo> stages = new ArrayList<>();
|
List<ShopMatchTaskStageVo> stages = new ArrayList<>();
|
||||||
List<LocalDateTime> scheduleTimes = request.getScheduleTimes() == null ? List.of() : request.getScheduleTimes();
|
List<LocalDateTime> scheduleTimes = request.getScheduleTimes() == null ? List.of() : request.getScheduleTimes();
|
||||||
@@ -942,6 +1001,19 @@ public class ShopMatchTaskService {
|
|||||||
shopMatchTaskCacheService.saveTaskCache(task);
|
shopMatchTaskCacheService.saveTaskCache(task);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private FileTaskEntity findOtherRunningTask(Long userId, Long taskId) {
|
||||||
|
if (userId == null || userId <= 0) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return fileTaskMapper.selectOne(new LambdaQueryWrapper<FileTaskEntity>()
|
||||||
|
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
|
||||||
|
.eq(FileTaskEntity::getUserId, userId)
|
||||||
|
.eq(FileTaskEntity::getStatus, "RUNNING")
|
||||||
|
.ne(taskId != null, FileTaskEntity::getId, taskId)
|
||||||
|
.orderByAsc(FileTaskEntity::getUpdatedAt)
|
||||||
|
.last("limit 1"));
|
||||||
|
}
|
||||||
|
|
||||||
private static String fmt(LocalDateTime time) {
|
private static String fmt(LocalDateTime time) {
|
||||||
return time == null ? null : time.toString();
|
return time == null ? null : time.toString();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -173,9 +173,7 @@ public class SplitRunService {
|
|||||||
vo.setResultId(entity.getId());
|
vo.setResultId(entity.getId());
|
||||||
vo.setSourceFilename(entity.getSourceFilename());
|
vo.setSourceFilename(entity.getSourceFilename());
|
||||||
vo.setOutputFilename(entity.getResultFilename());
|
vo.setOutputFilename(entity.getResultFilename());
|
||||||
vo.setDownloadUrl(entity.getSuccess() != null && entity.getSuccess() == 1
|
vo.setDownloadUrl(null);
|
||||||
? ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl())
|
|
||||||
: null);
|
|
||||||
vo.setRowCount(entity.getRowCount());
|
vo.setRowCount(entity.getRowCount());
|
||||||
vo.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1);
|
vo.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1);
|
||||||
vo.setError(entity.getErrorMessage());
|
vo.setError(entity.getErrorMessage());
|
||||||
|
|||||||
@@ -0,0 +1,52 @@
|
|||||||
|
package com.nanri.aiimage.modules.task.controller;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.common.api.ApiResponse;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||||
|
import io.swagger.v3.oas.annotations.Operation;
|
||||||
|
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import org.springframework.web.bind.annotation.GetMapping;
|
||||||
|
import org.springframework.web.bind.annotation.PathVariable;
|
||||||
|
import org.springframework.web.bind.annotation.PostMapping;
|
||||||
|
import org.springframework.web.bind.annotation.RequestMapping;
|
||||||
|
import org.springframework.web.bind.annotation.RequestParam;
|
||||||
|
import org.springframework.web.bind.annotation.RestController;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
@RestController
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
@RequestMapping("/api/task-file-jobs")
|
||||||
|
@Tag(name = "任务结果文件 Job")
|
||||||
|
public class TaskFileJobController {
|
||||||
|
|
||||||
|
private final TaskFileJobService taskFileJobService;
|
||||||
|
|
||||||
|
@GetMapping
|
||||||
|
@Operation(summary = "查询结果文件生成 Job")
|
||||||
|
public ApiResponse<List<TaskFileJobEntity>> list(
|
||||||
|
@RequestParam(value = "status", required = false) String status,
|
||||||
|
@RequestParam(value = "module_type", required = false) String moduleType,
|
||||||
|
@RequestParam(value = "task_id", required = false) Long taskId,
|
||||||
|
@RequestParam(value = "limit", defaultValue = "100") int limit) {
|
||||||
|
return ApiResponse.success(taskFileJobService.listJobs(status, moduleType, taskId, limit));
|
||||||
|
}
|
||||||
|
|
||||||
|
@PostMapping("/{jobId}/retry")
|
||||||
|
@Operation(summary = "手动重试结果文件生成 Job")
|
||||||
|
public ApiResponse<Map<String, Object>> retry(@PathVariable Long jobId) {
|
||||||
|
boolean updated = taskFileJobService.retry(jobId);
|
||||||
|
return ApiResponse.success(Map.of("jobId", jobId, "updated", updated));
|
||||||
|
}
|
||||||
|
|
||||||
|
@PostMapping("/stuck/reset")
|
||||||
|
@Operation(summary = "手动扫描并复位卡住的 RUNNING Job")
|
||||||
|
public ApiResponse<Map<String, Object>> resetStuck(
|
||||||
|
@RequestParam(value = "timeout_minutes", defaultValue = "30") int timeoutMinutes,
|
||||||
|
@RequestParam(value = "limit", defaultValue = "100") int limit) {
|
||||||
|
int reset = taskFileJobService.resetStuckRunningJobs(timeoutMinutes, limit);
|
||||||
|
return ApiResponse.success(Map.of("reset", reset));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
package com.nanri.aiimage.modules.task.mapper;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||||
|
import org.apache.ibatis.annotations.Mapper;
|
||||||
|
|
||||||
|
@Mapper
|
||||||
|
public interface TaskFileJobMapper extends BaseMapper<TaskFileJobEntity> {
|
||||||
|
}
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
package com.nanri.aiimage.modules.task.mapper;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskProgressSnapshotEntity;
|
||||||
|
import org.apache.ibatis.annotations.Mapper;
|
||||||
|
|
||||||
|
@Mapper
|
||||||
|
public interface TaskProgressSnapshotMapper extends BaseMapper<TaskProgressSnapshotEntity> {
|
||||||
|
}
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
package com.nanri.aiimage.modules.task.mapper;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskResultItemEntity;
|
||||||
|
import org.apache.ibatis.annotations.Mapper;
|
||||||
|
|
||||||
|
@Mapper
|
||||||
|
public interface TaskResultItemMapper extends BaseMapper<TaskResultItemEntity> {
|
||||||
|
}
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
package com.nanri.aiimage.modules.task.mapper;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskResultPayloadEntity;
|
||||||
|
import org.apache.ibatis.annotations.Mapper;
|
||||||
|
|
||||||
|
@Mapper
|
||||||
|
public interface TaskResultPayloadMapper extends BaseMapper<TaskResultPayloadEntity> {
|
||||||
|
}
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
package com.nanri.aiimage.modules.task.model.dto;
|
||||||
|
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
|
||||||
|
public record TaskFileJobDispatchEvent(
|
||||||
|
Long jobId,
|
||||||
|
Long taskId,
|
||||||
|
String moduleType,
|
||||||
|
Long resultId,
|
||||||
|
String scopeKey,
|
||||||
|
String jobType,
|
||||||
|
LocalDateTime createdAt
|
||||||
|
) {
|
||||||
|
}
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
package com.nanri.aiimage.modules.task.model.dto;
|
||||||
|
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
public class TaskFileJobMessage {
|
||||||
|
private Long jobId;
|
||||||
|
private Long taskId;
|
||||||
|
private String moduleType;
|
||||||
|
private Long resultId;
|
||||||
|
private String scopeKey;
|
||||||
|
private String jobType;
|
||||||
|
private LocalDateTime createdAt;
|
||||||
|
}
|
||||||
@@ -0,0 +1,28 @@
|
|||||||
|
package com.nanri.aiimage.modules.task.model.entity;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.annotation.IdType;
|
||||||
|
import com.baomidou.mybatisplus.annotation.TableId;
|
||||||
|
import com.baomidou.mybatisplus.annotation.TableName;
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@TableName("biz_task_file_job")
|
||||||
|
public class TaskFileJobEntity {
|
||||||
|
|
||||||
|
@TableId(type = IdType.AUTO)
|
||||||
|
private Long id;
|
||||||
|
private Long taskId;
|
||||||
|
private String moduleType;
|
||||||
|
private Long resultId;
|
||||||
|
private String scopeKey;
|
||||||
|
private String jobType;
|
||||||
|
private String status;
|
||||||
|
private Integer retryCount;
|
||||||
|
private String errorMessage;
|
||||||
|
private String resultFileUrl;
|
||||||
|
private LocalDateTime createdAt;
|
||||||
|
private LocalDateTime updatedAt;
|
||||||
|
private LocalDateTime finishedAt;
|
||||||
|
}
|
||||||
@@ -0,0 +1,28 @@
|
|||||||
|
package com.nanri.aiimage.modules.task.model.entity;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.annotation.IdType;
|
||||||
|
import com.baomidou.mybatisplus.annotation.TableId;
|
||||||
|
import com.baomidou.mybatisplus.annotation.TableName;
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@TableName("biz_task_progress_snapshot")
|
||||||
|
public class TaskProgressSnapshotEntity {
|
||||||
|
|
||||||
|
@TableId(type = IdType.AUTO)
|
||||||
|
private Long id;
|
||||||
|
private Long taskId;
|
||||||
|
private String moduleType;
|
||||||
|
private String status;
|
||||||
|
private Integer totalCount;
|
||||||
|
private Integer successCount;
|
||||||
|
private Integer failedCount;
|
||||||
|
private Integer pendingCount;
|
||||||
|
private String currentScopeKey;
|
||||||
|
private String message;
|
||||||
|
private String snapshotJson;
|
||||||
|
private LocalDateTime createdAt;
|
||||||
|
private LocalDateTime updatedAt;
|
||||||
|
}
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
package com.nanri.aiimage.modules.task.model.entity;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.annotation.IdType;
|
||||||
|
import com.baomidou.mybatisplus.annotation.TableId;
|
||||||
|
import com.baomidou.mybatisplus.annotation.TableName;
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@TableName("biz_task_result_item")
|
||||||
|
public class TaskResultItemEntity {
|
||||||
|
|
||||||
|
@TableId(type = IdType.AUTO)
|
||||||
|
private Long id;
|
||||||
|
private Long taskId;
|
||||||
|
private String moduleType;
|
||||||
|
private Long resultId;
|
||||||
|
private String scopeKey;
|
||||||
|
private String scopeHash;
|
||||||
|
private String itemKey;
|
||||||
|
private String countryCode;
|
||||||
|
private String asin;
|
||||||
|
private String status;
|
||||||
|
private String payloadJson;
|
||||||
|
private String payloadHash;
|
||||||
|
private LocalDateTime createdAt;
|
||||||
|
private LocalDateTime updatedAt;
|
||||||
|
}
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
package com.nanri.aiimage.modules.task.model.entity;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.annotation.IdType;
|
||||||
|
import com.baomidou.mybatisplus.annotation.TableId;
|
||||||
|
import com.baomidou.mybatisplus.annotation.TableName;
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@TableName("biz_task_result_payload")
|
||||||
|
public class TaskResultPayloadEntity {
|
||||||
|
|
||||||
|
@TableId(type = IdType.AUTO)
|
||||||
|
private Long id;
|
||||||
|
private Long taskId;
|
||||||
|
private String moduleType;
|
||||||
|
private String scopeKey;
|
||||||
|
private String scopeHash;
|
||||||
|
private String submissionId;
|
||||||
|
private String payloadJson;
|
||||||
|
private String payloadHash;
|
||||||
|
private Long version;
|
||||||
|
private LocalDateTime createdAt;
|
||||||
|
private LocalDateTime updatedAt;
|
||||||
|
}
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
package com.nanri.aiimage.modules.task.service;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.TaskFileJobMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.model.dto.TaskFileJobMessage;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.apache.rocketmq.spring.annotation.RocketMQMessageListener;
|
||||||
|
import org.apache.rocketmq.spring.core.RocketMQListener;
|
||||||
|
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
|
@Slf4j
|
||||||
|
@Service
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
@ConditionalOnProperty(prefix = "aiimage.result-file-job", name = "mq-enabled", havingValue = "true")
|
||||||
|
@RocketMQMessageListener(
|
||||||
|
topic = "${aiimage.result-file-job.topic:aiimage-result-file-job}",
|
||||||
|
consumerGroup = "${aiimage.result-file-job.consumer-group:aiimage-result-file-job-consumer}"
|
||||||
|
)
|
||||||
|
public class TaskFileJobConsumer implements RocketMQListener<TaskFileJobMessage> {
|
||||||
|
|
||||||
|
private final TaskFileJobMapper taskFileJobMapper;
|
||||||
|
private final TaskResultFileJobWorker taskResultFileJobWorker;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void onMessage(TaskFileJobMessage message) {
|
||||||
|
if (message == null || message.getJobId() == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
TaskFileJobEntity job = taskFileJobMapper.selectById(message.getJobId());
|
||||||
|
if (job == null) {
|
||||||
|
log.warn("[task-file-job] mq message ignored, job not found jobId={} taskId={} moduleType={}",
|
||||||
|
message.getJobId(), message.getTaskId(), message.getModuleType());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
taskResultFileJobWorker.process(job);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,42 @@
|
|||||||
|
package com.nanri.aiimage.modules.task.service;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.modules.task.model.dto.TaskFileJobDispatchEvent;
|
||||||
|
import com.nanri.aiimage.modules.task.model.dto.TaskFileJobMessage;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
import org.springframework.transaction.event.TransactionPhase;
|
||||||
|
import org.springframework.transaction.event.TransactionalEventListener;
|
||||||
|
|
||||||
|
@Slf4j
|
||||||
|
@Component
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
public class TaskFileJobDispatchCoordinator {
|
||||||
|
|
||||||
|
private final TaskFileJobPublisher taskFileJobPublisher;
|
||||||
|
private final TaskFileJobLocalDispatcher taskFileJobLocalDispatcher;
|
||||||
|
|
||||||
|
@TransactionalEventListener(phase = TransactionPhase.AFTER_COMMIT, fallbackExecution = true)
|
||||||
|
public void onTaskFileJobDispatch(TaskFileJobDispatchEvent event) {
|
||||||
|
if (event == null || event.jobId() == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
TaskFileJobMessage message = new TaskFileJobMessage();
|
||||||
|
message.setJobId(event.jobId());
|
||||||
|
message.setTaskId(event.taskId());
|
||||||
|
message.setModuleType(event.moduleType());
|
||||||
|
message.setResultId(event.resultId());
|
||||||
|
message.setScopeKey(event.scopeKey());
|
||||||
|
message.setJobType(event.jobType());
|
||||||
|
message.setCreatedAt(event.createdAt());
|
||||||
|
boolean published = taskFileJobPublisher.publish(message);
|
||||||
|
if (published) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
boolean dispatched = taskFileJobLocalDispatcher.dispatch(event.jobId(), event.taskId(), event.moduleType());
|
||||||
|
if (dispatched) {
|
||||||
|
log.info("[task-file-job] local async dispatch scheduled jobId={} taskId={} moduleType={}",
|
||||||
|
event.jobId(), event.taskId(), event.moduleType());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,53 @@
|
|||||||
|
package com.nanri.aiimage.modules.task.service;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.TaskFileJobMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.beans.factory.ObjectProvider;
|
||||||
|
import org.springframework.beans.factory.annotation.Qualifier;
|
||||||
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
|
import org.springframework.core.task.TaskExecutor;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
|
@Slf4j
|
||||||
|
@Service
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
public class TaskFileJobLocalDispatcher {
|
||||||
|
|
||||||
|
private final TaskFileJobMapper taskFileJobMapper;
|
||||||
|
private final ObjectProvider<TaskResultFileJobWorker> taskResultFileJobWorkerProvider;
|
||||||
|
|
||||||
|
@Qualifier("taskFileJobDispatchExecutor")
|
||||||
|
private final TaskExecutor taskFileJobDispatchExecutor;
|
||||||
|
|
||||||
|
@Value("${aiimage.result-file-job.local-dispatch-enabled:true}")
|
||||||
|
private boolean localDispatchEnabled;
|
||||||
|
|
||||||
|
public boolean dispatch(Long jobId, Long taskId, String moduleType) {
|
||||||
|
if (!localDispatchEnabled || jobId == null || jobId <= 0) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
taskFileJobDispatchExecutor.execute(() -> {
|
||||||
|
try {
|
||||||
|
TaskFileJobEntity job = taskFileJobMapper.selectById(jobId);
|
||||||
|
if (job == null) {
|
||||||
|
log.warn("[task-file-job] local dispatch ignored, job not found jobId={} taskId={} moduleType={}",
|
||||||
|
jobId, taskId, moduleType);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
TaskResultFileJobWorker taskResultFileJobWorker = taskResultFileJobWorkerProvider.getIfAvailable();
|
||||||
|
if (taskResultFileJobWorker == null) {
|
||||||
|
log.warn("[task-file-job] local dispatch ignored, worker unavailable jobId={} taskId={} moduleType={}",
|
||||||
|
jobId, taskId, moduleType);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
taskResultFileJobWorker.process(job);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[task-file-job] local dispatch failed jobId={} taskId={} moduleType={} msg={}",
|
||||||
|
jobId, taskId, moduleType, ex.getMessage(), ex);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
package com.nanri.aiimage.modules.task.service;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.modules.task.model.dto.TaskFileJobMessage;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.apache.rocketmq.spring.core.RocketMQTemplate;
|
||||||
|
import org.springframework.beans.factory.ObjectProvider;
|
||||||
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
|
@Slf4j
|
||||||
|
@Service
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
public class TaskFileJobPublisher {
|
||||||
|
|
||||||
|
private final ObjectProvider<RocketMQTemplate> rocketMQTemplateProvider;
|
||||||
|
|
||||||
|
@Value("${aiimage.result-file-job.mq-enabled:false}")
|
||||||
|
private boolean mqEnabled;
|
||||||
|
|
||||||
|
@Value("${aiimage.result-file-job.topic:aiimage-result-file-job}")
|
||||||
|
private String topic;
|
||||||
|
|
||||||
|
public boolean publish(TaskFileJobMessage message) {
|
||||||
|
if (message == null || message.getJobId() == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (!mqEnabled) {
|
||||||
|
log.info("[task-file-job] mq disabled, fallback to local dispatcher/scheduled worker jobId={} taskId={} moduleType={}",
|
||||||
|
message.getJobId(), message.getTaskId(), message.getModuleType());
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
RocketMQTemplate template = rocketMQTemplateProvider.getIfAvailable();
|
||||||
|
if (template == null) {
|
||||||
|
log.warn("[task-file-job] RocketMQTemplate unavailable, jobId={} taskId={} moduleType={}",
|
||||||
|
message.getJobId(), message.getTaskId(), message.getModuleType());
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
template.convertAndSend(topic, message);
|
||||||
|
log.info("[task-file-job] mq published jobId={} taskId={} moduleType={} topic={}",
|
||||||
|
message.getJobId(), message.getTaskId(), message.getModuleType(), topic);
|
||||||
|
return true;
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[task-file-job] publish RocketMQ failed, fallback to local dispatcher/scheduled worker jobId={} taskId={} moduleType={} msg={}",
|
||||||
|
message.getJobId(), message.getTaskId(), message.getModuleType(), ex.getMessage());
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,225 @@
|
|||||||
|
package com.nanri.aiimage.modules.task.service;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||||
|
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.TaskFileJobMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.model.dto.TaskFileJobDispatchEvent;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import org.springframework.context.ApplicationEventPublisher;
|
||||||
|
import org.springframework.dao.DuplicateKeyException;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
import org.springframework.transaction.annotation.Transactional;
|
||||||
|
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
@Service
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
public class TaskFileJobService {
|
||||||
|
|
||||||
|
public static final String JOB_TYPE_ASSEMBLE_RESULT = "ASSEMBLE_RESULT";
|
||||||
|
|
||||||
|
private final TaskFileJobMapper taskFileJobMapper;
|
||||||
|
private final ApplicationEventPublisher applicationEventPublisher;
|
||||||
|
|
||||||
|
@Transactional
|
||||||
|
public TaskFileJobEntity enqueueAssembleResult(Long taskId, String moduleType, Long resultId, String scopeKey) {
|
||||||
|
if (taskId == null || taskId <= 0 || moduleType == null || moduleType.isBlank() || resultId == null || resultId <= 0) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
LocalDateTime now = LocalDateTime.now();
|
||||||
|
TaskFileJobEntity existing = findJob(taskId, moduleType, resultId, JOB_TYPE_ASSEMBLE_RESULT);
|
||||||
|
if (existing == null) {
|
||||||
|
TaskFileJobEntity entity = new TaskFileJobEntity();
|
||||||
|
entity.setTaskId(taskId);
|
||||||
|
entity.setModuleType(moduleType);
|
||||||
|
entity.setResultId(resultId);
|
||||||
|
entity.setScopeKey(scopeKey);
|
||||||
|
entity.setJobType(JOB_TYPE_ASSEMBLE_RESULT);
|
||||||
|
entity.setStatus("PENDING");
|
||||||
|
entity.setRetryCount(0);
|
||||||
|
entity.setCreatedAt(now);
|
||||||
|
entity.setUpdatedAt(now);
|
||||||
|
try {
|
||||||
|
taskFileJobMapper.insert(entity);
|
||||||
|
publishDispatchEvent(entity);
|
||||||
|
return entity;
|
||||||
|
} catch (DuplicateKeyException ignored) {
|
||||||
|
existing = findJob(taskId, moduleType, resultId, JOB_TYPE_ASSEMBLE_RESULT);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (existing == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if ("SUCCESS".equals(existing.getStatus()) || "RUNNING".equals(existing.getStatus())) {
|
||||||
|
return existing;
|
||||||
|
}
|
||||||
|
taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||||
|
.eq(TaskFileJobEntity::getId, existing.getId())
|
||||||
|
.set(TaskFileJobEntity::getScopeKey, scopeKey)
|
||||||
|
.set(TaskFileJobEntity::getStatus, "PENDING")
|
||||||
|
.set(TaskFileJobEntity::getErrorMessage, null)
|
||||||
|
.set(TaskFileJobEntity::getUpdatedAt, now)
|
||||||
|
.set(TaskFileJobEntity::getFinishedAt, null));
|
||||||
|
TaskFileJobEntity refreshed = taskFileJobMapper.selectById(existing.getId());
|
||||||
|
publishDispatchEvent(refreshed);
|
||||||
|
return refreshed;
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<TaskFileJobEntity> listRunnableJobs(int limit) {
|
||||||
|
return taskFileJobMapper.selectList(new LambdaQueryWrapper<TaskFileJobEntity>()
|
||||||
|
.in(TaskFileJobEntity::getStatus, List.of("PENDING", "FAILED"))
|
||||||
|
.lt(TaskFileJobEntity::getRetryCount, 5)
|
||||||
|
.orderByAsc(TaskFileJobEntity::getUpdatedAt)
|
||||||
|
.last("limit " + Math.max(1, Math.min(limit, 100))));
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<TaskFileJobEntity> listJobs(String status, String moduleType, Long taskId, int limit) {
|
||||||
|
LambdaQueryWrapper<TaskFileJobEntity> wrapper = new LambdaQueryWrapper<TaskFileJobEntity>()
|
||||||
|
.orderByDesc(TaskFileJobEntity::getUpdatedAt)
|
||||||
|
.last("limit " + Math.max(1, Math.min(limit, 500)));
|
||||||
|
if (status != null && !status.isBlank()) {
|
||||||
|
wrapper.eq(TaskFileJobEntity::getStatus, status.trim());
|
||||||
|
}
|
||||||
|
if (moduleType != null && !moduleType.isBlank()) {
|
||||||
|
wrapper.eq(TaskFileJobEntity::getModuleType, moduleType.trim());
|
||||||
|
}
|
||||||
|
if (taskId != null && taskId > 0) {
|
||||||
|
wrapper.eq(TaskFileJobEntity::getTaskId, taskId);
|
||||||
|
}
|
||||||
|
return taskFileJobMapper.selectList(wrapper);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Transactional
|
||||||
|
public boolean retry(Long jobId) {
|
||||||
|
if (jobId == null || jobId <= 0) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
int updated = taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||||
|
.eq(TaskFileJobEntity::getId, jobId)
|
||||||
|
.ne(TaskFileJobEntity::getStatus, "SUCCESS")
|
||||||
|
.set(TaskFileJobEntity::getStatus, "PENDING")
|
||||||
|
.set(TaskFileJobEntity::getRetryCount, 0)
|
||||||
|
.set(TaskFileJobEntity::getErrorMessage, null)
|
||||||
|
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())
|
||||||
|
.set(TaskFileJobEntity::getFinishedAt, null));
|
||||||
|
if (updated > 0) {
|
||||||
|
TaskFileJobEntity refreshed = taskFileJobMapper.selectById(jobId);
|
||||||
|
publishDispatchEvent(refreshed);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Transactional
|
||||||
|
public int resetStuckRunningJobs(int stuckMinutes, int limit) {
|
||||||
|
LocalDateTime threshold = LocalDateTime.now().minusMinutes(Math.max(5, stuckMinutes));
|
||||||
|
List<TaskFileJobEntity> jobs = taskFileJobMapper.selectList(new LambdaQueryWrapper<TaskFileJobEntity>()
|
||||||
|
.eq(TaskFileJobEntity::getStatus, "RUNNING")
|
||||||
|
.lt(TaskFileJobEntity::getUpdatedAt, threshold)
|
||||||
|
.orderByAsc(TaskFileJobEntity::getUpdatedAt)
|
||||||
|
.last("limit " + Math.max(1, Math.min(limit, 200))));
|
||||||
|
int reset = 0;
|
||||||
|
for (TaskFileJobEntity job : jobs) {
|
||||||
|
int retryCount = job.getRetryCount() == null ? 0 : job.getRetryCount();
|
||||||
|
if (retryCount >= 5) {
|
||||||
|
taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||||
|
.eq(TaskFileJobEntity::getId, job.getId())
|
||||||
|
.eq(TaskFileJobEntity::getStatus, "RUNNING")
|
||||||
|
.set(TaskFileJobEntity::getStatus, "FAILED")
|
||||||
|
.set(TaskFileJobEntity::getErrorMessage, "文件生成任务运行超时")
|
||||||
|
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now()));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
int updated = taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||||
|
.eq(TaskFileJobEntity::getId, job.getId())
|
||||||
|
.eq(TaskFileJobEntity::getStatus, "RUNNING")
|
||||||
|
.set(TaskFileJobEntity::getStatus, "PENDING")
|
||||||
|
.set(TaskFileJobEntity::getErrorMessage, "文件生成任务运行超时,已重新排队")
|
||||||
|
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now()));
|
||||||
|
if (updated > 0) {
|
||||||
|
reset++;
|
||||||
|
publishDispatchEvent(taskFileJobMapper.selectById(job.getId()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return reset;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Transactional
|
||||||
|
public boolean markRunning(Long jobId) {
|
||||||
|
if (jobId == null || jobId <= 0) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||||
|
.eq(TaskFileJobEntity::getId, jobId)
|
||||||
|
.in(TaskFileJobEntity::getStatus, List.of("PENDING", "FAILED"))
|
||||||
|
.set(TaskFileJobEntity::getStatus, "RUNNING")
|
||||||
|
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())) > 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Transactional
|
||||||
|
public void markSuccess(TaskFileJobEntity job, String resultFileUrl) {
|
||||||
|
taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||||
|
.eq(TaskFileJobEntity::getId, job.getId())
|
||||||
|
.set(TaskFileJobEntity::getStatus, "SUCCESS")
|
||||||
|
.set(TaskFileJobEntity::getErrorMessage, null)
|
||||||
|
.set(TaskFileJobEntity::getResultFileUrl, resultFileUrl)
|
||||||
|
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())
|
||||||
|
.set(TaskFileJobEntity::getFinishedAt, LocalDateTime.now()));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Transactional
|
||||||
|
public void markFailed(TaskFileJobEntity job, String message) {
|
||||||
|
int retryCount = job.getRetryCount() == null ? 0 : job.getRetryCount() + 1;
|
||||||
|
taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
|
||||||
|
.eq(TaskFileJobEntity::getId, job.getId())
|
||||||
|
.set(TaskFileJobEntity::getStatus, "FAILED")
|
||||||
|
.set(TaskFileJobEntity::getRetryCount, retryCount)
|
||||||
|
.set(TaskFileJobEntity::getErrorMessage, message == null ? "结果文件生成失败" : message)
|
||||||
|
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now()));
|
||||||
|
}
|
||||||
|
|
||||||
|
public TaskFileJobEntity findAssembleJob(Long taskId, String moduleType, Long resultId) {
|
||||||
|
return findJob(taskId, moduleType, resultId, JOB_TYPE_ASSEMBLE_RESULT);
|
||||||
|
}
|
||||||
|
|
||||||
|
public long countUnfinishedAssembleJobs(Long taskId, String moduleType) {
|
||||||
|
if (taskId == null || taskId <= 0 || moduleType == null || moduleType.isBlank()) {
|
||||||
|
return 0L;
|
||||||
|
}
|
||||||
|
Long count = taskFileJobMapper.selectCount(new LambdaQueryWrapper<TaskFileJobEntity>()
|
||||||
|
.eq(TaskFileJobEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskFileJobEntity::getModuleType, moduleType)
|
||||||
|
.eq(TaskFileJobEntity::getJobType, JOB_TYPE_ASSEMBLE_RESULT)
|
||||||
|
.ne(TaskFileJobEntity::getStatus, "SUCCESS"));
|
||||||
|
return count == null ? 0L : count;
|
||||||
|
}
|
||||||
|
|
||||||
|
private TaskFileJobEntity findJob(Long taskId, String moduleType, Long resultId, String jobType) {
|
||||||
|
if (taskId == null || moduleType == null || moduleType.isBlank() || resultId == null || jobType == null || jobType.isBlank()) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return taskFileJobMapper.selectOne(new LambdaQueryWrapper<TaskFileJobEntity>()
|
||||||
|
.eq(TaskFileJobEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskFileJobEntity::getModuleType, moduleType)
|
||||||
|
.eq(TaskFileJobEntity::getResultId, resultId)
|
||||||
|
.eq(TaskFileJobEntity::getJobType, jobType)
|
||||||
|
.last("limit 1"));
|
||||||
|
}
|
||||||
|
|
||||||
|
private void publishDispatchEvent(TaskFileJobEntity entity) {
|
||||||
|
if (entity == null || entity.getId() == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
applicationEventPublisher.publishEvent(new TaskFileJobDispatchEvent(
|
||||||
|
entity.getId(),
|
||||||
|
entity.getTaskId(),
|
||||||
|
entity.getModuleType(),
|
||||||
|
entity.getResultId(),
|
||||||
|
entity.getScopeKey(),
|
||||||
|
entity.getJobType(),
|
||||||
|
LocalDateTime.now()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,98 @@
|
|||||||
|
package com.nanri.aiimage.modules.task.service;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||||
|
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import com.nanri.aiimage.common.exception.BusinessException;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.TaskProgressSnapshotMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskProgressSnapshotEntity;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import org.springframework.dao.DuplicateKeyException;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
import org.springframework.transaction.annotation.Transactional;
|
||||||
|
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
|
||||||
|
@Service
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
public class TaskProgressSnapshotService {
|
||||||
|
|
||||||
|
private final TaskProgressSnapshotMapper taskProgressSnapshotMapper;
|
||||||
|
private final ObjectMapper objectMapper;
|
||||||
|
|
||||||
|
@Transactional
|
||||||
|
public void save(Long taskId,
|
||||||
|
String moduleType,
|
||||||
|
String status,
|
||||||
|
int totalCount,
|
||||||
|
int successCount,
|
||||||
|
int failedCount,
|
||||||
|
String currentScopeKey,
|
||||||
|
String message,
|
||||||
|
Object snapshot) {
|
||||||
|
if (taskId == null || taskId <= 0 || isBlank(moduleType) || isBlank(status)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
int pendingCount = Math.max(0, totalCount - successCount - failedCount);
|
||||||
|
String snapshotJson = snapshot == null ? null : writeJson(snapshot);
|
||||||
|
LocalDateTime now = LocalDateTime.now();
|
||||||
|
TaskProgressSnapshotEntity existing = find(taskId, moduleType);
|
||||||
|
if (existing == null) {
|
||||||
|
TaskProgressSnapshotEntity entity = new TaskProgressSnapshotEntity();
|
||||||
|
entity.setTaskId(taskId);
|
||||||
|
entity.setModuleType(moduleType);
|
||||||
|
entity.setStatus(status);
|
||||||
|
entity.setTotalCount(totalCount);
|
||||||
|
entity.setSuccessCount(successCount);
|
||||||
|
entity.setFailedCount(failedCount);
|
||||||
|
entity.setPendingCount(pendingCount);
|
||||||
|
entity.setCurrentScopeKey(currentScopeKey);
|
||||||
|
entity.setMessage(message);
|
||||||
|
entity.setSnapshotJson(snapshotJson);
|
||||||
|
entity.setCreatedAt(now);
|
||||||
|
entity.setUpdatedAt(now);
|
||||||
|
try {
|
||||||
|
taskProgressSnapshotMapper.insert(entity);
|
||||||
|
return;
|
||||||
|
} catch (DuplicateKeyException ignored) {
|
||||||
|
existing = find(taskId, moduleType);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (existing == null) {
|
||||||
|
throw new BusinessException("保存任务进度快照失败");
|
||||||
|
}
|
||||||
|
taskProgressSnapshotMapper.update(null, new LambdaUpdateWrapper<TaskProgressSnapshotEntity>()
|
||||||
|
.eq(TaskProgressSnapshotEntity::getId, existing.getId())
|
||||||
|
.set(TaskProgressSnapshotEntity::getStatus, status)
|
||||||
|
.set(TaskProgressSnapshotEntity::getTotalCount, totalCount)
|
||||||
|
.set(TaskProgressSnapshotEntity::getSuccessCount, successCount)
|
||||||
|
.set(TaskProgressSnapshotEntity::getFailedCount, failedCount)
|
||||||
|
.set(TaskProgressSnapshotEntity::getPendingCount, pendingCount)
|
||||||
|
.set(TaskProgressSnapshotEntity::getCurrentScopeKey, currentScopeKey)
|
||||||
|
.set(TaskProgressSnapshotEntity::getMessage, message)
|
||||||
|
.set(TaskProgressSnapshotEntity::getSnapshotJson, snapshotJson)
|
||||||
|
.set(TaskProgressSnapshotEntity::getUpdatedAt, now));
|
||||||
|
}
|
||||||
|
|
||||||
|
public TaskProgressSnapshotEntity find(Long taskId, String moduleType) {
|
||||||
|
if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return taskProgressSnapshotMapper.selectOne(new LambdaQueryWrapper<TaskProgressSnapshotEntity>()
|
||||||
|
.eq(TaskProgressSnapshotEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskProgressSnapshotEntity::getModuleType, moduleType)
|
||||||
|
.last("limit 1"));
|
||||||
|
}
|
||||||
|
|
||||||
|
private String writeJson(Object value) {
|
||||||
|
try {
|
||||||
|
return objectMapper.writeValueAsString(value);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new BusinessException("序列化任务进度快照失败");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean isBlank(String value) {
|
||||||
|
return value == null || value.isBlank();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,166 @@
|
|||||||
|
package com.nanri.aiimage.modules.task.service;
|
||||||
|
|
||||||
|
import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskService;
|
||||||
|
import com.nanri.aiimage.modules.brand.service.BrandTaskService;
|
||||||
|
import com.nanri.aiimage.modules.deletebrand.service.DeleteBrandRunService;
|
||||||
|
import com.nanri.aiimage.modules.patroldelete.service.PatrolDeleteTaskService;
|
||||||
|
import com.nanri.aiimage.modules.pricetrack.service.PriceTrackTaskService;
|
||||||
|
import com.nanri.aiimage.modules.productrisk.service.ProductRiskTaskService;
|
||||||
|
import com.nanri.aiimage.modules.queryasin.service.QueryAsinTaskService;
|
||||||
|
import com.nanri.aiimage.modules.shopmatch.service.ShopMatchTaskService;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
|
import org.springframework.scheduling.annotation.Scheduled;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
@Slf4j
|
||||||
|
@Service
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
public class TaskResultFileJobWorker {
|
||||||
|
|
||||||
|
private final TaskFileJobService taskFileJobService;
|
||||||
|
private final TaskResultPayloadService taskResultPayloadService;
|
||||||
|
private final FileResultMapper fileResultMapper;
|
||||||
|
private final TaskFileJobLocalDispatcher taskFileJobLocalDispatcher;
|
||||||
|
private final ShopMatchTaskService shopMatchTaskService;
|
||||||
|
private final PriceTrackTaskService priceTrackTaskService;
|
||||||
|
private final ProductRiskTaskService productRiskTaskService;
|
||||||
|
private final QueryAsinTaskService queryAsinTaskService;
|
||||||
|
private final PatrolDeleteTaskService patrolDeleteTaskService;
|
||||||
|
private final AppearancePatentTaskService appearancePatentTaskService;
|
||||||
|
private final DeleteBrandRunService deleteBrandRunService;
|
||||||
|
private final BrandTaskService brandTaskService;
|
||||||
|
|
||||||
|
@Value("${aiimage.result-file-job.local-worker-enabled:true}")
|
||||||
|
private boolean localWorkerEnabled;
|
||||||
|
|
||||||
|
@Value("${aiimage.result-file-job.batch-size:20}")
|
||||||
|
private int batchSize;
|
||||||
|
|
||||||
|
@Value("${aiimage.result-file-job.stuck-timeout-minutes:30}")
|
||||||
|
private int stuckTimeoutMinutes;
|
||||||
|
|
||||||
|
@Scheduled(fixedDelayString = "${aiimage.result-file-job.local-worker-delay-ms:15000}")
|
||||||
|
public void runPendingJobs() {
|
||||||
|
if (!localWorkerEnabled) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
List<TaskFileJobEntity> jobs = taskFileJobService.listRunnableJobs(batchSize);
|
||||||
|
if (jobs == null || jobs.isEmpty()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
log.info("[task-file-job] scheduled worker picked jobs count={}", jobs.size());
|
||||||
|
for (TaskFileJobEntity job : jobs) {
|
||||||
|
boolean dispatched = taskFileJobLocalDispatcher.dispatch(
|
||||||
|
job.getId(),
|
||||||
|
job.getTaskId(),
|
||||||
|
job.getModuleType()
|
||||||
|
);
|
||||||
|
if (!dispatched) {
|
||||||
|
process(job);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Scheduled(fixedDelayString = "${aiimage.result-file-job.stuck-scan-delay-ms:60000}")
|
||||||
|
public void resetStuckJobs() {
|
||||||
|
int reset = taskFileJobService.resetStuckRunningJobs(stuckTimeoutMinutes, batchSize);
|
||||||
|
if (reset > 0) {
|
||||||
|
log.warn("[task-file-job] reset stuck running jobs count={} timeoutMinutes={}", reset, stuckTimeoutMinutes);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void process(TaskFileJobEntity job) {
|
||||||
|
if (job == null || job.getId() == null || !taskFileJobService.markRunning(job.getId())) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
long startedAt = System.currentTimeMillis();
|
||||||
|
try {
|
||||||
|
dispatch(job);
|
||||||
|
String resultFileUrl = resolveResultFileUrl(job);
|
||||||
|
taskFileJobService.markSuccess(job, resultFileUrl);
|
||||||
|
cleanupAfterSuccess(job);
|
||||||
|
log.info("[task-file-job] process success jobId={} taskId={} moduleType={} resultId={} elapsedMs={} resultFileUrl={}",
|
||||||
|
job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId(),
|
||||||
|
System.currentTimeMillis() - startedAt, resultFileUrl);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
String message = ex.getMessage() == null ? "结果文件生成失败" : ex.getMessage();
|
||||||
|
log.warn("[task-file-job] process failed jobId={} taskId={} moduleType={} resultId={} msg={}",
|
||||||
|
job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId(), message);
|
||||||
|
taskFileJobService.markFailed(job, message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private String resolveResultFileUrl(TaskFileJobEntity job) {
|
||||||
|
if ("BRAND".equals(job.getModuleType())) {
|
||||||
|
return brandTaskService.resolveResultObjectKey(job.getTaskId());
|
||||||
|
}
|
||||||
|
if (job.getResultId() == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
FileResultEntity result = fileResultMapper.selectById(job.getResultId());
|
||||||
|
return result == null ? null : result.getResultFileUrl();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void dispatch(TaskFileJobEntity job) {
|
||||||
|
String moduleType = job.getModuleType();
|
||||||
|
if ("SHOP_MATCH".equals(moduleType)) {
|
||||||
|
shopMatchTaskService.processResultFileJob(job);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if ("PRICE_TRACK".equals(moduleType)) {
|
||||||
|
priceTrackTaskService.processResultFileJob(job);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if ("PRODUCT_RISK_RESOLVE".equals(moduleType)) {
|
||||||
|
productRiskTaskService.processResultFileJob(job);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if ("QUERY_ASIN".equals(moduleType)) {
|
||||||
|
queryAsinTaskService.processResultFileJob(job);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if ("PATROL_DELETE".equals(moduleType)) {
|
||||||
|
patrolDeleteTaskService.processResultFileJob(job);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if ("APPEARANCE_PATENT".equals(moduleType)) {
|
||||||
|
appearancePatentTaskService.processResultFileJob(job);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if ("DELETE_BRAND".equals(moduleType)) {
|
||||||
|
deleteBrandRunService.processResultFileJob(job);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if ("BRAND".equals(moduleType)) {
|
||||||
|
brandTaskService.processResultFileJob(job);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
throw new IllegalArgumentException("unsupported result file job module: " + moduleType);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void cleanupAfterSuccess(TaskFileJobEntity job) {
|
||||||
|
String moduleType = job.getModuleType();
|
||||||
|
if ("SHOP_MATCH".equals(moduleType)
|
||||||
|
|| "PRICE_TRACK".equals(moduleType)
|
||||||
|
|| "PRODUCT_RISK_RESOLVE".equals(moduleType)
|
||||||
|
|| "QUERY_ASIN".equals(moduleType)
|
||||||
|
|| "PATROL_DELETE".equals(moduleType)) {
|
||||||
|
taskResultPayloadService.deleteLatest(job.getTaskId(), moduleType, job.getScopeKey());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if ("APPEARANCE_PATENT".equals(moduleType)) {
|
||||||
|
appearancePatentTaskService.cleanupResultFileJob(job);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if ("DELETE_BRAND".equals(moduleType)) {
|
||||||
|
deleteBrandRunService.cleanupResultFileJob(job);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,258 @@
|
|||||||
|
package com.nanri.aiimage.modules.task.service;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||||
|
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import com.nanri.aiimage.common.exception.BusinessException;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.TaskResultItemMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskResultItemEntity;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import org.springframework.dao.DuplicateKeyException;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
import org.springframework.transaction.annotation.Transactional;
|
||||||
|
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.security.MessageDigest;
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
@Service
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
public class TaskResultItemService {
|
||||||
|
|
||||||
|
private final TaskResultItemMapper taskResultItemMapper;
|
||||||
|
private final ObjectMapper objectMapper;
|
||||||
|
private final TransientPayloadStorageService transientPayloadStorageService;
|
||||||
|
|
||||||
|
@Transactional
|
||||||
|
public void replaceResultSnapshot(Long taskId, String moduleType, Long resultId, String scopeKey, Object snapshot) {
|
||||||
|
if (taskId == null || taskId <= 0 || isBlank(moduleType) || resultId == null || resultId <= 0 || snapshot == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
String normalizedScopeKey = firstNonBlank(scopeKey, "result:" + resultId).trim();
|
||||||
|
String itemKey = "result:" + resultId;
|
||||||
|
upsert(taskId, moduleType, resultId, normalizedScopeKey, itemKey, null, null, null, snapshot);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Transactional
|
||||||
|
public void replaceTaskSnapshots(Long taskId, String moduleType, List<? extends Object> snapshots, SnapshotKeyResolver resolver) {
|
||||||
|
if (taskId == null || taskId <= 0 || isBlank(moduleType) || snapshots == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
for (Object snapshot : snapshots) {
|
||||||
|
if (snapshot == null) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
Long resultId = resolver == null ? null : resolver.resultId(snapshot);
|
||||||
|
if (resultId == null || resultId <= 0) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
replaceResultSnapshot(taskId, moduleType, resultId, resolver.scopeKey(snapshot), snapshot);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public <T> List<T> listResultSnapshots(Long taskId, String moduleType, Class<T> clazz) {
|
||||||
|
if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
|
||||||
|
return List.of();
|
||||||
|
}
|
||||||
|
List<TaskResultItemEntity> rows = taskResultItemMapper.selectList(new LambdaQueryWrapper<TaskResultItemEntity>()
|
||||||
|
.eq(TaskResultItemEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskResultItemEntity::getModuleType, moduleType)
|
||||||
|
.likeRight(TaskResultItemEntity::getItemKey, "result:")
|
||||||
|
.orderByAsc(TaskResultItemEntity::getResultId)
|
||||||
|
.orderByAsc(TaskResultItemEntity::getId));
|
||||||
|
List<T> out = new ArrayList<>();
|
||||||
|
for (TaskResultItemEntity row : rows) {
|
||||||
|
T value = readPayload(row, clazz);
|
||||||
|
if (value != null) {
|
||||||
|
out.add(value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
public <T> T getResultSnapshot(Long taskId, String moduleType, Long resultId, Class<T> clazz) {
|
||||||
|
if (taskId == null || taskId <= 0 || isBlank(moduleType) || resultId == null || resultId <= 0) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
TaskResultItemEntity row = taskResultItemMapper.selectOne(new LambdaQueryWrapper<TaskResultItemEntity>()
|
||||||
|
.eq(TaskResultItemEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskResultItemEntity::getModuleType, moduleType)
|
||||||
|
.eq(TaskResultItemEntity::getResultId, resultId)
|
||||||
|
.eq(TaskResultItemEntity::getItemKey, "result:" + resultId)
|
||||||
|
.last("limit 1"));
|
||||||
|
return readPayload(row, clazz);
|
||||||
|
}
|
||||||
|
|
||||||
|
public long countResultSnapshots(Long taskId, String moduleType) {
|
||||||
|
if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
|
||||||
|
return 0L;
|
||||||
|
}
|
||||||
|
Long count = taskResultItemMapper.selectCount(new LambdaQueryWrapper<TaskResultItemEntity>()
|
||||||
|
.eq(TaskResultItemEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskResultItemEntity::getModuleType, moduleType)
|
||||||
|
.likeRight(TaskResultItemEntity::getItemKey, "result:"));
|
||||||
|
return count == null ? 0L : count;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Transactional
|
||||||
|
public void deleteTaskItems(Long taskId, String moduleType) {
|
||||||
|
if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
List<TaskResultItemEntity> rows = taskResultItemMapper.selectList(new LambdaQueryWrapper<TaskResultItemEntity>()
|
||||||
|
.select(TaskResultItemEntity::getId, TaskResultItemEntity::getPayloadJson)
|
||||||
|
.eq(TaskResultItemEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskResultItemEntity::getModuleType, moduleType));
|
||||||
|
if (rows != null) {
|
||||||
|
for (TaskResultItemEntity row : rows) {
|
||||||
|
transientPayloadStorageService.deletePayloadIfPresent(row.getPayloadJson());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
taskResultItemMapper.delete(new LambdaQueryWrapper<TaskResultItemEntity>()
|
||||||
|
.eq(TaskResultItemEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskResultItemEntity::getModuleType, moduleType));
|
||||||
|
}
|
||||||
|
|
||||||
|
private void upsert(Long taskId,
|
||||||
|
String moduleType,
|
||||||
|
Long resultId,
|
||||||
|
String scopeKey,
|
||||||
|
String itemKey,
|
||||||
|
String countryCode,
|
||||||
|
String asin,
|
||||||
|
String status,
|
||||||
|
Object payload) {
|
||||||
|
String payloadJson = writeJson(payload);
|
||||||
|
String payloadHash = hash(payloadJson);
|
||||||
|
String normalizedScopeKey = firstNonBlank(scopeKey, "task:" + taskId).trim();
|
||||||
|
String normalizedItemKey = firstNonBlank(itemKey, "result:" + resultId).trim();
|
||||||
|
String scopeHash = hash(normalizedScopeKey);
|
||||||
|
LocalDateTime now = LocalDateTime.now();
|
||||||
|
|
||||||
|
TaskResultItemEntity existing = find(taskId, moduleType, scopeHash, normalizedItemKey);
|
||||||
|
if (isUnchanged(existing, resultId, normalizedScopeKey, countryCode, asin, status, payloadHash)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
String storedPayload = transientPayloadStorageService.storeResultItemPayload(
|
||||||
|
moduleType, taskId, scopeHash, normalizedItemKey, payloadJson);
|
||||||
|
if (existing == null) {
|
||||||
|
TaskResultItemEntity entity = new TaskResultItemEntity();
|
||||||
|
entity.setTaskId(taskId);
|
||||||
|
entity.setModuleType(moduleType);
|
||||||
|
entity.setResultId(resultId);
|
||||||
|
entity.setScopeKey(normalizedScopeKey);
|
||||||
|
entity.setScopeHash(scopeHash);
|
||||||
|
entity.setItemKey(normalizedItemKey);
|
||||||
|
entity.setCountryCode(countryCode);
|
||||||
|
entity.setAsin(asin);
|
||||||
|
entity.setStatus(status);
|
||||||
|
entity.setPayloadJson(storedPayload);
|
||||||
|
entity.setPayloadHash(payloadHash);
|
||||||
|
entity.setCreatedAt(now);
|
||||||
|
entity.setUpdatedAt(now);
|
||||||
|
try {
|
||||||
|
taskResultItemMapper.insert(entity);
|
||||||
|
return;
|
||||||
|
} catch (DuplicateKeyException ignored) {
|
||||||
|
existing = find(taskId, moduleType, scopeHash, normalizedItemKey);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (existing == null) {
|
||||||
|
throw new BusinessException("保存任务结果明细失败");
|
||||||
|
}
|
||||||
|
if (isUnchanged(existing, resultId, normalizedScopeKey, countryCode, asin, status, payloadHash)) {
|
||||||
|
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(storedPayload, existing.getPayloadJson());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(existing.getPayloadJson(), storedPayload);
|
||||||
|
taskResultItemMapper.update(null, new LambdaUpdateWrapper<TaskResultItemEntity>()
|
||||||
|
.eq(TaskResultItemEntity::getId, existing.getId())
|
||||||
|
.set(TaskResultItemEntity::getResultId, resultId)
|
||||||
|
.set(TaskResultItemEntity::getScopeKey, normalizedScopeKey)
|
||||||
|
.set(TaskResultItemEntity::getCountryCode, countryCode)
|
||||||
|
.set(TaskResultItemEntity::getAsin, asin)
|
||||||
|
.set(TaskResultItemEntity::getStatus, status)
|
||||||
|
.set(TaskResultItemEntity::getPayloadJson, storedPayload)
|
||||||
|
.set(TaskResultItemEntity::getPayloadHash, payloadHash)
|
||||||
|
.set(TaskResultItemEntity::getUpdatedAt, now));
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean isUnchanged(TaskResultItemEntity existing,
|
||||||
|
Long resultId,
|
||||||
|
String scopeKey,
|
||||||
|
String countryCode,
|
||||||
|
String asin,
|
||||||
|
String status,
|
||||||
|
String payloadHash) {
|
||||||
|
if (existing == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return java.util.Objects.equals(existing.getResultId(), resultId)
|
||||||
|
&& java.util.Objects.equals(existing.getScopeKey(), scopeKey)
|
||||||
|
&& java.util.Objects.equals(existing.getCountryCode(), countryCode)
|
||||||
|
&& java.util.Objects.equals(existing.getAsin(), asin)
|
||||||
|
&& java.util.Objects.equals(existing.getStatus(), status)
|
||||||
|
&& java.util.Objects.equals(existing.getPayloadHash(), payloadHash);
|
||||||
|
}
|
||||||
|
|
||||||
|
private TaskResultItemEntity find(Long taskId, String moduleType, String scopeHash, String itemKey) {
|
||||||
|
return taskResultItemMapper.selectOne(new LambdaQueryWrapper<TaskResultItemEntity>()
|
||||||
|
.eq(TaskResultItemEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskResultItemEntity::getModuleType, moduleType)
|
||||||
|
.eq(TaskResultItemEntity::getScopeHash, scopeHash)
|
||||||
|
.eq(TaskResultItemEntity::getItemKey, itemKey)
|
||||||
|
.last("limit 1"));
|
||||||
|
}
|
||||||
|
|
||||||
|
private <T> T readPayload(TaskResultItemEntity row, Class<T> clazz) {
|
||||||
|
if (row == null || isBlank(row.getPayloadJson())) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
String payloadJson = transientPayloadStorageService.resolvePayload(row.getPayloadJson(), "read task result item failed");
|
||||||
|
return objectMapper.readValue(payloadJson, clazz);
|
||||||
|
} catch (BusinessException ex) {
|
||||||
|
throw ex;
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new BusinessException("读取任务结果明细失败");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private String writeJson(Object value) {
|
||||||
|
try {
|
||||||
|
return objectMapper.writeValueAsString(value);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new BusinessException("序列化任务结果明细失败");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private String hash(String value) {
|
||||||
|
try {
|
||||||
|
MessageDigest digest = MessageDigest.getInstance("SHA-256");
|
||||||
|
byte[] bytes = digest.digest((value == null ? "" : value).getBytes(StandardCharsets.UTF_8));
|
||||||
|
StringBuilder sb = new StringBuilder(bytes.length * 2);
|
||||||
|
for (byte b : bytes) {
|
||||||
|
sb.append(String.format("%02x", b));
|
||||||
|
}
|
||||||
|
return sb.toString();
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new IllegalStateException("failed to hash task result item", ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private String firstNonBlank(String primary, String fallback) {
|
||||||
|
return primary == null || primary.isBlank() ? fallback : primary;
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean isBlank(String value) {
|
||||||
|
return value == null || value.isBlank();
|
||||||
|
}
|
||||||
|
|
||||||
|
public interface SnapshotKeyResolver {
|
||||||
|
Long resultId(Object snapshot);
|
||||||
|
|
||||||
|
String scopeKey(Object snapshot);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,139 @@
|
|||||||
|
package com.nanri.aiimage.modules.task.service;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||||
|
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import com.nanri.aiimage.common.exception.BusinessException;
|
||||||
|
import com.nanri.aiimage.modules.task.mapper.TaskResultPayloadMapper;
|
||||||
|
import com.nanri.aiimage.modules.task.model.entity.TaskResultPayloadEntity;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import org.springframework.dao.DuplicateKeyException;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
import org.springframework.transaction.annotation.Transactional;
|
||||||
|
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.security.MessageDigest;
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
|
||||||
|
@Service
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
public class TaskResultPayloadService {
|
||||||
|
|
||||||
|
private static final String LATEST_SUBMISSION_ID = "latest";
|
||||||
|
|
||||||
|
private final TaskResultPayloadMapper taskResultPayloadMapper;
|
||||||
|
private final ObjectMapper objectMapper;
|
||||||
|
private final TransientPayloadStorageService transientPayloadStorageService;
|
||||||
|
|
||||||
|
@Transactional
|
||||||
|
public void saveLatest(Long taskId, String moduleType, String scopeKey, Object payload) {
|
||||||
|
if (taskId == null || taskId <= 0 || isBlank(moduleType) || isBlank(scopeKey) || payload == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
String normalizedScopeKey = scopeKey.trim();
|
||||||
|
String scopeHash = hash(normalizedScopeKey);
|
||||||
|
String payloadJson = writeJson(payload);
|
||||||
|
String storedPayload = transientPayloadStorageService.storeResultPayload(
|
||||||
|
moduleType, taskId, scopeHash, LATEST_SUBMISSION_ID, payloadJson);
|
||||||
|
String payloadHash = hash(payloadJson);
|
||||||
|
LocalDateTime now = LocalDateTime.now();
|
||||||
|
|
||||||
|
TaskResultPayloadEntity existing = findLatest(taskId, moduleType, scopeHash);
|
||||||
|
if (existing == null) {
|
||||||
|
TaskResultPayloadEntity entity = new TaskResultPayloadEntity();
|
||||||
|
entity.setTaskId(taskId);
|
||||||
|
entity.setModuleType(moduleType);
|
||||||
|
entity.setScopeKey(normalizedScopeKey);
|
||||||
|
entity.setScopeHash(scopeHash);
|
||||||
|
entity.setSubmissionId(LATEST_SUBMISSION_ID);
|
||||||
|
entity.setPayloadJson(storedPayload);
|
||||||
|
entity.setPayloadHash(payloadHash);
|
||||||
|
entity.setVersion(1L);
|
||||||
|
entity.setCreatedAt(now);
|
||||||
|
entity.setUpdatedAt(now);
|
||||||
|
try {
|
||||||
|
taskResultPayloadMapper.insert(entity);
|
||||||
|
return;
|
||||||
|
} catch (DuplicateKeyException ignored) {
|
||||||
|
existing = findLatest(taskId, moduleType, scopeHash);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (existing == null) {
|
||||||
|
throw new BusinessException("保存任务结果载荷失败");
|
||||||
|
}
|
||||||
|
long nextVersion = existing.getVersion() == null ? 1L : existing.getVersion() + 1L;
|
||||||
|
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(existing.getPayloadJson(), storedPayload);
|
||||||
|
taskResultPayloadMapper.update(null, new LambdaUpdateWrapper<TaskResultPayloadEntity>()
|
||||||
|
.eq(TaskResultPayloadEntity::getId, existing.getId())
|
||||||
|
.set(TaskResultPayloadEntity::getScopeKey, normalizedScopeKey)
|
||||||
|
.set(TaskResultPayloadEntity::getPayloadJson, storedPayload)
|
||||||
|
.set(TaskResultPayloadEntity::getPayloadHash, payloadHash)
|
||||||
|
.set(TaskResultPayloadEntity::getVersion, nextVersion)
|
||||||
|
.set(TaskResultPayloadEntity::getUpdatedAt, now));
|
||||||
|
}
|
||||||
|
|
||||||
|
public <T> T getLatest(Long taskId, String moduleType, String scopeKey, Class<T> clazz) {
|
||||||
|
if (taskId == null || taskId <= 0 || isBlank(moduleType) || isBlank(scopeKey)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
TaskResultPayloadEntity entity = findLatest(taskId, moduleType, hash(scopeKey.trim()));
|
||||||
|
if (entity == null || isBlank(entity.getPayloadJson())) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
String payloadJson = transientPayloadStorageService.resolvePayload(entity.getPayloadJson(), "read task result payload failed");
|
||||||
|
return objectMapper.readValue(payloadJson, clazz);
|
||||||
|
} catch (BusinessException ex) {
|
||||||
|
throw ex;
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new BusinessException("读取任务结果载荷失败");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Transactional
|
||||||
|
public void deleteLatest(Long taskId, String moduleType, String scopeKey) {
|
||||||
|
if (taskId == null || taskId <= 0 || isBlank(moduleType) || isBlank(scopeKey)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
TaskResultPayloadEntity existing = findLatest(taskId, moduleType, hash(scopeKey.trim()));
|
||||||
|
if (existing != null) {
|
||||||
|
transientPayloadStorageService.deletePayloadIfPresent(existing.getPayloadJson());
|
||||||
|
taskResultPayloadMapper.deleteById(existing.getId());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private TaskResultPayloadEntity findLatest(Long taskId, String moduleType, String scopeHash) {
|
||||||
|
return taskResultPayloadMapper.selectOne(new LambdaQueryWrapper<TaskResultPayloadEntity>()
|
||||||
|
.eq(TaskResultPayloadEntity::getTaskId, taskId)
|
||||||
|
.eq(TaskResultPayloadEntity::getModuleType, moduleType)
|
||||||
|
.eq(TaskResultPayloadEntity::getScopeHash, scopeHash)
|
||||||
|
.eq(TaskResultPayloadEntity::getSubmissionId, LATEST_SUBMISSION_ID)
|
||||||
|
.last("limit 1"));
|
||||||
|
}
|
||||||
|
|
||||||
|
private String writeJson(Object value) {
|
||||||
|
try {
|
||||||
|
return objectMapper.writeValueAsString(value);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new BusinessException("序列化任务结果载荷失败");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private String hash(String value) {
|
||||||
|
try {
|
||||||
|
MessageDigest digest = MessageDigest.getInstance("SHA-256");
|
||||||
|
byte[] bytes = digest.digest((value == null ? "" : value).getBytes(StandardCharsets.UTF_8));
|
||||||
|
StringBuilder sb = new StringBuilder(bytes.length * 2);
|
||||||
|
for (byte b : bytes) {
|
||||||
|
sb.append(String.format("%02x", b));
|
||||||
|
}
|
||||||
|
return sb.toString();
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new IllegalStateException("failed to hash task result payload", ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean isBlank(String value) {
|
||||||
|
return value == null || value.isBlank();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -3,9 +3,7 @@ package com.nanri.aiimage.modules.task.service;
|
|||||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||||
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
import com.nanri.aiimage.config.TaskPressureProperties;
|
|
||||||
import com.nanri.aiimage.common.exception.BusinessException;
|
import com.nanri.aiimage.common.exception.BusinessException;
|
||||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
|
||||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
@@ -13,37 +11,23 @@ import org.springframework.dao.DuplicateKeyException;
|
|||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
import org.springframework.transaction.annotation.Transactional;
|
import org.springframework.transaction.annotation.Transactional;
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.nio.file.Files;
|
|
||||||
import java.nio.file.Path;
|
|
||||||
import java.nio.file.StandardOpenOption;
|
|
||||||
import java.nio.charset.StandardCharsets;
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.nio.file.Path;
|
||||||
import java.security.MessageDigest;
|
import java.security.MessageDigest;
|
||||||
import java.time.LocalDateTime;
|
import java.time.LocalDateTime;
|
||||||
import java.util.HashSet;
|
|
||||||
import java.util.LinkedHashMap;
|
import java.util.LinkedHashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Set;
|
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
public class TaskScopePayloadStorageService {
|
public class TaskScopePayloadStorageService {
|
||||||
|
|
||||||
private static final Set<String> OSS_BACKED_STATE_MODULES = new HashSet<>(Set.of(
|
|
||||||
"SHOP_MATCH",
|
|
||||||
"PRODUCT_RISK_RESOLVE",
|
|
||||||
"PRICE_TRACK",
|
|
||||||
"PATROL_DELETE",
|
|
||||||
"QUERY_ASIN"
|
|
||||||
));
|
|
||||||
private static final String OSS_POINTER_PREFIX = "oss:";
|
|
||||||
private static final String LOCAL_BUFFER_DIR = "task-scope-buffer";
|
private static final String LOCAL_BUFFER_DIR = "task-scope-buffer";
|
||||||
|
|
||||||
private final TaskScopeStateMapper taskScopeStateMapper;
|
private final TaskScopeStateMapper taskScopeStateMapper;
|
||||||
private final ObjectMapper objectMapper;
|
private final ObjectMapper objectMapper;
|
||||||
private final OssStorageService ossStorageService;
|
private final TransientPayloadStorageService transientPayloadStorageService;
|
||||||
private final TaskPressureProperties taskPressureProperties;
|
|
||||||
|
|
||||||
@Transactional
|
@Transactional
|
||||||
public <T> void saveScopePayload(Long taskId, String moduleType, String scopeKey, T payload) {
|
public <T> void saveScopePayload(Long taskId, String moduleType, String scopeKey, T payload) {
|
||||||
@@ -53,13 +37,11 @@ public class TaskScopePayloadStorageService {
|
|||||||
String normalizedScopeKey = normalize(scopeKey);
|
String normalizedScopeKey = normalize(scopeKey);
|
||||||
String scopeHash = hash(normalizedScopeKey);
|
String scopeHash = hash(normalizedScopeKey);
|
||||||
String payloadJson = writeJson(payload, "写入任务范围载荷失败");
|
String payloadJson = writeJson(payload, "写入任务范围载荷失败");
|
||||||
|
String stateJson = transientPayloadStorageService.storeScopePayload(
|
||||||
|
moduleType, taskId, scopeHash, payloadJson, true);
|
||||||
LocalDateTime now = LocalDateTime.now();
|
LocalDateTime now = LocalDateTime.now();
|
||||||
|
|
||||||
TaskScopeStateEntity state = getScopeState(taskId, moduleType, scopeHash);
|
TaskScopeStateEntity state = getScopeState(taskId, moduleType, scopeHash);
|
||||||
if (shouldStoreStatePayloadInOss(moduleType)) {
|
|
||||||
saveOssBackedScopePayload(taskId, moduleType, normalizedScopeKey, scopeHash, payloadJson, now, state);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
String stateJson = storeStatePayload(moduleType, taskId, scopeHash, payloadJson);
|
|
||||||
if (state == null) {
|
if (state == null) {
|
||||||
TaskScopeStateEntity entity = new TaskScopeStateEntity();
|
TaskScopeStateEntity entity = new TaskScopeStateEntity();
|
||||||
entity.setTaskId(taskId);
|
entity.setTaskId(taskId);
|
||||||
@@ -84,7 +66,7 @@ public class TaskScopePayloadStorageService {
|
|||||||
if (state == null) {
|
if (state == null) {
|
||||||
throw new BusinessException("写入任务范围载荷失败");
|
throw new BusinessException("写入任务范围载荷失败");
|
||||||
}
|
}
|
||||||
deleteReplacedStatePayloadIfNeeded(state.getStateJson(), stateJson);
|
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(state.getStateJson(), stateJson);
|
||||||
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
|
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
|
||||||
.eq(TaskScopeStateEntity::getId, state.getId())
|
.eq(TaskScopeStateEntity::getId, state.getId())
|
||||||
.set(TaskScopeStateEntity::getScopeKey, normalizedScopeKey)
|
.set(TaskScopeStateEntity::getScopeKey, normalizedScopeKey)
|
||||||
@@ -145,8 +127,7 @@ public class TaskScopePayloadStorageService {
|
|||||||
if (state == null) {
|
if (state == null) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
deleteBufferedPayload(taskId, moduleType, state.getScopeHash());
|
transientPayloadStorageService.deletePayloadIfPresent(state.getStateJson());
|
||||||
deleteStatePayloadIfNeeded(state.getStateJson());
|
|
||||||
boolean hasParsedPayload = !isBlank(state.getParsedPayloadJson());
|
boolean hasParsedPayload = !isBlank(state.getParsedPayloadJson());
|
||||||
if (hasParsedPayload) {
|
if (hasParsedPayload) {
|
||||||
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
|
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
|
||||||
@@ -195,8 +176,7 @@ public class TaskScopePayloadStorageService {
|
|||||||
if (state == null || state.getId() == null) {
|
if (state == null || state.getId() == null) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
deleteBufferedPayload(taskId, moduleType, state.getScopeHash());
|
transientPayloadStorageService.deletePayloadIfPresent(state.getStateJson());
|
||||||
deleteStatePayloadIfNeeded(state.getStateJson());
|
|
||||||
if (!isBlank(state.getParsedPayloadJson())) {
|
if (!isBlank(state.getParsedPayloadJson())) {
|
||||||
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
|
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
|
||||||
.eq(TaskScopeStateEntity::getId, state.getId())
|
.eq(TaskScopeStateEntity::getId, state.getId())
|
||||||
@@ -221,6 +201,9 @@ public class TaskScopePayloadStorageService {
|
|||||||
String scopeKey = normalize(entry.getKey());
|
String scopeKey = normalize(entry.getKey());
|
||||||
String scopeHash = hash(scopeKey);
|
String scopeHash = hash(scopeKey);
|
||||||
String parsedPayloadJson = writeJson(entry.getValue(), "写入任务解析载荷失败");
|
String parsedPayloadJson = writeJson(entry.getValue(), "写入任务解析载荷失败");
|
||||||
|
String storedParsedPayload = transientPayloadStorageService.storeParsedPayload(
|
||||||
|
moduleType, taskId, scopeHash, parsedPayloadJson, true);
|
||||||
|
|
||||||
TaskScopeStateEntity state = getScopeState(taskId, moduleType, scopeHash);
|
TaskScopeStateEntity state = getScopeState(taskId, moduleType, scopeHash);
|
||||||
if (state == null) {
|
if (state == null) {
|
||||||
TaskScopeStateEntity entity = new TaskScopeStateEntity();
|
TaskScopeStateEntity entity = new TaskScopeStateEntity();
|
||||||
@@ -228,7 +211,7 @@ public class TaskScopePayloadStorageService {
|
|||||||
entity.setModuleType(moduleType);
|
entity.setModuleType(moduleType);
|
||||||
entity.setScopeKey(scopeKey);
|
entity.setScopeKey(scopeKey);
|
||||||
entity.setScopeHash(scopeHash);
|
entity.setScopeHash(scopeHash);
|
||||||
entity.setParsedPayloadJson(parsedPayloadJson);
|
entity.setParsedPayloadJson(storedParsedPayload);
|
||||||
entity.setChunkTotal(0);
|
entity.setChunkTotal(0);
|
||||||
entity.setReceivedChunkCount(0);
|
entity.setReceivedChunkCount(0);
|
||||||
entity.setCompleted(0);
|
entity.setCompleted(0);
|
||||||
@@ -244,11 +227,11 @@ public class TaskScopePayloadStorageService {
|
|||||||
if (state == null) {
|
if (state == null) {
|
||||||
throw new BusinessException("写入任务解析载荷失败");
|
throw new BusinessException("写入任务解析载荷失败");
|
||||||
}
|
}
|
||||||
deleteParsedPayloadIfNeeded(state.getParsedPayloadJson(), parsedPayloadJson);
|
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(state.getParsedPayloadJson(), storedParsedPayload);
|
||||||
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
|
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
|
||||||
.eq(TaskScopeStateEntity::getId, state.getId())
|
.eq(TaskScopeStateEntity::getId, state.getId())
|
||||||
.set(TaskScopeStateEntity::getScopeKey, scopeKey)
|
.set(TaskScopeStateEntity::getScopeKey, scopeKey)
|
||||||
.set(TaskScopeStateEntity::getParsedPayloadJson, parsedPayloadJson)
|
.set(TaskScopeStateEntity::getParsedPayloadJson, storedParsedPayload)
|
||||||
.set(TaskScopeStateEntity::getUpdatedAt, now));
|
.set(TaskScopeStateEntity::getUpdatedAt, now));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -271,7 +254,9 @@ public class TaskScopePayloadStorageService {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
result.put(state.getScopeKey(), objectMapper.readValue(resolveParsedPayload(state.getParsedPayloadJson()), clazz));
|
String payloadJson = transientPayloadStorageService.resolvePayload(
|
||||||
|
state.getParsedPayloadJson(), "read parsed payload failed");
|
||||||
|
result.put(state.getScopeKey(), objectMapper.readValue(payloadJson, clazz));
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
throw new BusinessException("读取任务原始载荷失败");
|
throw new BusinessException("读取任务原始载荷失败");
|
||||||
}
|
}
|
||||||
@@ -281,32 +266,7 @@ public class TaskScopePayloadStorageService {
|
|||||||
|
|
||||||
@Transactional
|
@Transactional
|
||||||
public boolean recoverBufferedScopePayload(Long taskId, String moduleType, String scopeHash) {
|
public boolean recoverBufferedScopePayload(Long taskId, String moduleType, String scopeHash) {
|
||||||
if (taskId == null || taskId <= 0 || isBlank(moduleType) || isBlank(scopeHash)) {
|
return false;
|
||||||
return false;
|
|
||||||
}
|
|
||||||
String payloadJson = readBufferedPayload(taskId, moduleType, scopeHash);
|
|
||||||
if (payloadJson == null) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
TaskScopeStateEntity state = getScopeState(taskId, moduleType, scopeHash);
|
|
||||||
if (state == null) {
|
|
||||||
deleteBufferedPayload(taskId, moduleType, scopeHash);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if (!shouldStoreStatePayloadInOss(moduleType)) {
|
|
||||||
deleteBufferedPayload(taskId, moduleType, scopeHash);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
LocalDateTime now = LocalDateTime.now();
|
|
||||||
String stateJson = storeStatePayload(moduleType, taskId, scopeHash, payloadJson);
|
|
||||||
deleteReplacedStatePayloadIfNeeded(state.getStateJson(), stateJson);
|
|
||||||
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
|
|
||||||
.eq(TaskScopeStateEntity::getId, state.getId())
|
|
||||||
.set(TaskScopeStateEntity::getStateJson, stateJson)
|
|
||||||
.set(TaskScopeStateEntity::getLastChunkAt, now)
|
|
||||||
.set(TaskScopeStateEntity::getUpdatedAt, now));
|
|
||||||
deleteBufferedPayload(taskId, moduleType, scopeHash);
|
|
||||||
return true;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public Path getBufferedPayloadRoot() {
|
public Path getBufferedPayloadRoot() {
|
||||||
@@ -321,126 +281,17 @@ public class TaskScopePayloadStorageService {
|
|||||||
.last("limit 1"));
|
.last("limit 1"));
|
||||||
}
|
}
|
||||||
|
|
||||||
private String storeStatePayload(String moduleType, Long taskId, String scopeHash, String payloadJson) {
|
|
||||||
if (!shouldStoreStatePayloadInOss(moduleType)) {
|
|
||||||
return payloadJson;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
String pointer = OSS_POINTER_PREFIX + ossStorageService.uploadTaskScopePayload(moduleType, taskId, scopeHash, payloadJson);
|
|
||||||
return objectMapper.writeValueAsString(pointer);
|
|
||||||
} catch (Exception ex) {
|
|
||||||
throw new BusinessException("写入任务范围载荷失败");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private String resolveStatePayload(TaskScopeStateEntity state) {
|
private String resolveStatePayload(TaskScopeStateEntity state) {
|
||||||
if (state == null || isBlank(state.getStateJson())) {
|
if (state == null || isBlank(state.getStateJson())) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
String bufferedPayload = readBufferedPayload(state.getTaskId(), state.getModuleType(), state.getScopeHash());
|
|
||||||
if (bufferedPayload != null) {
|
|
||||||
return bufferedPayload;
|
|
||||||
}
|
|
||||||
String ossPointer = extractOssPointer(state.getStateJson());
|
|
||||||
if (ossPointer == null) {
|
|
||||||
return state.getStateJson();
|
|
||||||
}
|
|
||||||
try {
|
try {
|
||||||
return ossStorageService.readObjectAsString(stripOssPointerPrefix(ossPointer));
|
return transientPayloadStorageService.resolvePayload(state.getStateJson(), "read task scope payload failed");
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
throw new BusinessException("读取任务范围载荷失败");
|
throw new BusinessException("读取任务范围载荷失败");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void deleteStatePayloadIfNeeded(String value) {
|
|
||||||
String ossPointer = extractOssPointer(value);
|
|
||||||
if (ossPointer == null) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
ossStorageService.deleteObject(stripOssPointerPrefix(ossPointer));
|
|
||||||
} catch (Exception ignored) {
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private String resolveParsedPayload(String value) {
|
|
||||||
if (isBlank(value) || !isOssPointer(value)) {
|
|
||||||
return value;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
return ossStorageService.readObjectAsString(stripOssPointerPrefix(value));
|
|
||||||
} catch (Exception ex) {
|
|
||||||
throw new BusinessException("读取任务解析载荷失败");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void deleteParsedPayloadIfNeeded(String oldValue, String newValue) {
|
|
||||||
if (isBlank(oldValue) || oldValue.equals(newValue) || !isOssPointer(oldValue)) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
ossStorageService.deleteObject(stripOssPointerPrefix(oldValue));
|
|
||||||
} catch (Exception ignored) {
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void deleteReplacedStatePayloadIfNeeded(String oldValue, String newValue) {
|
|
||||||
if (isBlank(oldValue) || oldValue.equals(newValue)) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
deleteStatePayloadIfNeeded(oldValue);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void saveOssBackedScopePayload(Long taskId,
|
|
||||||
String moduleType,
|
|
||||||
String normalizedScopeKey,
|
|
||||||
String scopeHash,
|
|
||||||
String payloadJson,
|
|
||||||
LocalDateTime now,
|
|
||||||
TaskScopeStateEntity state) {
|
|
||||||
if (state == null) {
|
|
||||||
String stateJson = storeStatePayload(moduleType, taskId, scopeHash, payloadJson);
|
|
||||||
TaskScopeStateEntity entity = new TaskScopeStateEntity();
|
|
||||||
entity.setTaskId(taskId);
|
|
||||||
entity.setModuleType(moduleType);
|
|
||||||
entity.setScopeKey(normalizedScopeKey);
|
|
||||||
entity.setScopeHash(scopeHash);
|
|
||||||
entity.setStateJson(stateJson);
|
|
||||||
entity.setChunkTotal(0);
|
|
||||||
entity.setReceivedChunkCount(0);
|
|
||||||
entity.setCompleted(0);
|
|
||||||
entity.setLastChunkAt(now);
|
|
||||||
entity.setLastError(null);
|
|
||||||
entity.setCreatedAt(now);
|
|
||||||
entity.setUpdatedAt(now);
|
|
||||||
try {
|
|
||||||
taskScopeStateMapper.insert(entity);
|
|
||||||
deleteBufferedPayload(taskId, moduleType, scopeHash);
|
|
||||||
return;
|
|
||||||
} catch (DuplicateKeyException ignored) {
|
|
||||||
state = getScopeState(taskId, moduleType, scopeHash);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (state == null) {
|
|
||||||
throw new BusinessException("刷新任务范围缓冲载荷失败");
|
|
||||||
}
|
|
||||||
|
|
||||||
writeBufferedPayload(taskId, moduleType, scopeHash, payloadJson);
|
|
||||||
if (!shouldFlushBufferedPayload(state, now)) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
String stateJson = storeStatePayload(moduleType, taskId, scopeHash, payloadJson);
|
|
||||||
deleteReplacedStatePayloadIfNeeded(state.getStateJson(), stateJson);
|
|
||||||
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
|
|
||||||
.eq(TaskScopeStateEntity::getId, state.getId())
|
|
||||||
.set(TaskScopeStateEntity::getScopeKey, normalizedScopeKey)
|
|
||||||
.set(TaskScopeStateEntity::getStateJson, stateJson)
|
|
||||||
.set(TaskScopeStateEntity::getLastChunkAt, now)
|
|
||||||
.set(TaskScopeStateEntity::getUpdatedAt, now));
|
|
||||||
deleteBufferedPayload(taskId, moduleType, scopeHash);
|
|
||||||
}
|
|
||||||
|
|
||||||
private String writeJson(Object value, String message) {
|
private String writeJson(Object value, String message) {
|
||||||
try {
|
try {
|
||||||
return objectMapper.writeValueAsString(value);
|
return objectMapper.writeValueAsString(value);
|
||||||
@@ -457,76 +308,6 @@ public class TaskScopePayloadStorageService {
|
|||||||
return value == null || value.isBlank();
|
return value == null || value.isBlank();
|
||||||
}
|
}
|
||||||
|
|
||||||
private boolean shouldStoreStatePayloadInOss(String moduleType) {
|
|
||||||
return OSS_BACKED_STATE_MODULES.contains(normalize(moduleType).toUpperCase());
|
|
||||||
}
|
|
||||||
|
|
||||||
private boolean isOssPointer(String value) {
|
|
||||||
return !isBlank(value) && value.startsWith(OSS_POINTER_PREFIX);
|
|
||||||
}
|
|
||||||
|
|
||||||
private String extractOssPointer(String value) {
|
|
||||||
if (isBlank(value)) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
if (isOssPointer(value)) {
|
|
||||||
return value;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
String decoded = objectMapper.readValue(value, String.class);
|
|
||||||
return isOssPointer(decoded) ? decoded : null;
|
|
||||||
} catch (Exception ignored) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private String stripOssPointerPrefix(String value) {
|
|
||||||
return value.substring(OSS_POINTER_PREFIX.length());
|
|
||||||
}
|
|
||||||
|
|
||||||
private boolean shouldFlushBufferedPayload(TaskScopeStateEntity state, LocalDateTime now) {
|
|
||||||
long flushIntervalMillis = Math.max(0L, taskPressureProperties.getScopePayloadFlushIntervalMillis());
|
|
||||||
if (flushIntervalMillis <= 0L || state.getUpdatedAt() == null) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
return java.time.Duration.between(state.getUpdatedAt(), now).toMillis() >= flushIntervalMillis;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void writeBufferedPayload(Long taskId, String moduleType, String scopeHash, String payloadJson) {
|
|
||||||
try {
|
|
||||||
Path path = buildBufferedPayloadPath(taskId, moduleType, scopeHash);
|
|
||||||
Files.createDirectories(path.getParent());
|
|
||||||
Files.writeString(path, payloadJson, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE);
|
|
||||||
} catch (IOException ex) {
|
|
||||||
throw new BusinessException("写入任务范围缓冲载荷失败");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private String readBufferedPayload(Long taskId, String moduleType, String scopeHash) {
|
|
||||||
try {
|
|
||||||
Path path = buildBufferedPayloadPath(taskId, moduleType, scopeHash);
|
|
||||||
if (!Files.isRegularFile(path)) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
return Files.readString(path);
|
|
||||||
} catch (IOException ex) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void deleteBufferedPayload(Long taskId, String moduleType, String scopeHash) {
|
|
||||||
try {
|
|
||||||
Path path = buildBufferedPayloadPath(taskId, moduleType, scopeHash);
|
|
||||||
Files.deleteIfExists(path);
|
|
||||||
} catch (IOException ignored) {
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private Path buildBufferedPayloadPath(Long taskId, String moduleType, String scopeHash) {
|
|
||||||
String normalizedModuleType = normalize(moduleType).toLowerCase();
|
|
||||||
return Path.of(System.getProperty("java.io.tmpdir"), LOCAL_BUFFER_DIR, normalizedModuleType, String.valueOf(taskId), scopeHash + ".json");
|
|
||||||
}
|
|
||||||
|
|
||||||
private String hash(String value) {
|
private String hash(String value) {
|
||||||
try {
|
try {
|
||||||
MessageDigest digest = MessageDigest.getInstance("SHA-256");
|
MessageDigest digest = MessageDigest.getInstance("SHA-256");
|
||||||
|
|||||||
@@ -0,0 +1,149 @@
|
|||||||
|
package com.nanri.aiimage.modules.task.service;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import com.nanri.aiimage.config.TransientStorageProperties;
|
||||||
|
import com.nanri.aiimage.modules.file.service.object.RustfsObjectStorageService;
|
||||||
|
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
|
import java.util.Locale;
|
||||||
|
import java.util.UUID;
|
||||||
|
|
||||||
|
@Service
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
public class TransientPayloadStorageService {
|
||||||
|
|
||||||
|
private static final String RUSTFS_POINTER_PREFIX = "rustfs:";
|
||||||
|
private static final String OSS_POINTER_PREFIX = "oss:";
|
||||||
|
|
||||||
|
private final TransientStorageProperties properties;
|
||||||
|
private final RustfsObjectStorageService rustfsObjectStorageService;
|
||||||
|
private final OssStorageService ossStorageService;
|
||||||
|
private final ObjectMapper objectMapper;
|
||||||
|
|
||||||
|
public boolean isWriteEnabled() {
|
||||||
|
return properties.isEnabled() && rustfsObjectStorageService.isConfigured();
|
||||||
|
}
|
||||||
|
|
||||||
|
public String storeScopePayload(String moduleType, Long taskId, String scopeHash, String content, boolean encodeAsJsonString) {
|
||||||
|
return store("task-scope", moduleType, taskId, scopeHash, "latest", content, encodeAsJsonString);
|
||||||
|
}
|
||||||
|
|
||||||
|
public String storeParsedPayload(String moduleType, Long taskId, String scopeHash, String content, boolean encodeAsJsonString) {
|
||||||
|
return store("task-parsed", moduleType, taskId, scopeHash, "latest", content, encodeAsJsonString);
|
||||||
|
}
|
||||||
|
|
||||||
|
public String storeResultPayload(String moduleType, Long taskId, String scopeHash, String submissionId, String content) {
|
||||||
|
return store("task-result-payload", moduleType, taskId, scopeHash, submissionId, content, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
public String storeResultItemPayload(String moduleType, Long taskId, String scopeHash, String itemKey, String content) {
|
||||||
|
return store("task-result-item", moduleType, taskId, scopeHash, itemKey, content, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
public String storeChunkPayload(String moduleType, Long taskId, String scopeHash, Integer chunkIndex, String content) {
|
||||||
|
String entryKey = chunkIndex == null ? UUID.randomUUID().toString() : "chunk-" + chunkIndex;
|
||||||
|
return store("task-chunk", moduleType, taskId, scopeHash, entryKey, content, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
public String resolvePayload(String value, String errorMessage) {
|
||||||
|
String pointer = extractPointer(value);
|
||||||
|
if (pointer == null) {
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
if (pointer.startsWith(RUSTFS_POINTER_PREFIX)) {
|
||||||
|
return rustfsObjectStorageService.readObjectAsString(pointer.substring(RUSTFS_POINTER_PREFIX.length()));
|
||||||
|
}
|
||||||
|
if (pointer.startsWith(OSS_POINTER_PREFIX)) {
|
||||||
|
return ossStorageService.readObjectAsString(pointer.substring(OSS_POINTER_PREFIX.length()));
|
||||||
|
}
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new IllegalStateException(errorMessage, ex);
|
||||||
|
}
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void deletePayloadIfPresent(String value) {
|
||||||
|
String pointer = extractPointer(value);
|
||||||
|
if (pointer == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (pointer.startsWith(RUSTFS_POINTER_PREFIX)) {
|
||||||
|
rustfsObjectStorageService.deleteObject(pointer.substring(RUSTFS_POINTER_PREFIX.length()));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (pointer.startsWith(OSS_POINTER_PREFIX)) {
|
||||||
|
ossStorageService.deleteObject(pointer.substring(OSS_POINTER_PREFIX.length()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void deleteReplacedPayloadIfNeeded(String oldValue, String newValue) {
|
||||||
|
String oldPointer = extractPointer(oldValue);
|
||||||
|
String newPointer = extractPointer(newValue);
|
||||||
|
if (oldPointer == null || oldPointer.equals(newPointer)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
deletePayloadIfPresent(oldPointer);
|
||||||
|
}
|
||||||
|
|
||||||
|
public String extractPointer(String value) {
|
||||||
|
if (value == null || value.isBlank()) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (isPointer(value)) {
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
String decoded = objectMapper.readValue(value, String.class);
|
||||||
|
return isPointer(decoded) ? decoded : null;
|
||||||
|
} catch (Exception ignored) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean isPointer(String value) {
|
||||||
|
return value != null && (value.startsWith(RUSTFS_POINTER_PREFIX) || value.startsWith(OSS_POINTER_PREFIX));
|
||||||
|
}
|
||||||
|
|
||||||
|
private String store(String category,
|
||||||
|
String moduleType,
|
||||||
|
Long taskId,
|
||||||
|
String scopeHash,
|
||||||
|
String entryKey,
|
||||||
|
String content,
|
||||||
|
boolean encodeAsJsonString) {
|
||||||
|
if (!isWriteEnabled()) {
|
||||||
|
return content;
|
||||||
|
}
|
||||||
|
String objectKey = buildObjectKey(category, moduleType, taskId, scopeHash, entryKey);
|
||||||
|
String pointer = RUSTFS_POINTER_PREFIX + rustfsObjectStorageService.uploadText(objectKey, content);
|
||||||
|
if (!encodeAsJsonString) {
|
||||||
|
return pointer;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return objectMapper.writeValueAsString(pointer);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new IllegalStateException("failed to encode transient storage pointer", ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private String buildObjectKey(String category, String moduleType, Long taskId, String scopeHash, String entryKey) {
|
||||||
|
String normalizedCategory = normalize(category, "unknown");
|
||||||
|
String normalizedModuleType = normalize(moduleType, "unknown");
|
||||||
|
String normalizedScopeHash = normalize(scopeHash, UUID.randomUUID().toString());
|
||||||
|
String normalizedEntryKey = normalize(entryKey, "latest");
|
||||||
|
return String.format("%s/%s/%s/%s/%s.json",
|
||||||
|
normalizedCategory,
|
||||||
|
normalizedModuleType,
|
||||||
|
taskId == null ? 0L : taskId,
|
||||||
|
normalizedScopeHash,
|
||||||
|
normalizedEntryKey);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String normalize(String value, String fallback) {
|
||||||
|
String normalized = value == null ? "" : value.trim().toLowerCase(Locale.ROOT);
|
||||||
|
return normalized.isBlank() ? fallback : normalized.replaceAll("[^a-z0-9._-]+", "_");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -2,11 +2,11 @@ package com.nanri.aiimage.modules.ziniao.service;
|
|||||||
|
|
||||||
import com.fasterxml.jackson.core.type.TypeReference;
|
import com.fasterxml.jackson.core.type.TypeReference;
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
import com.nanri.aiimage.common.exception.BusinessException;
|
|
||||||
import com.nanri.aiimage.config.ZiniaoProperties;
|
import com.nanri.aiimage.config.ZiniaoProperties;
|
||||||
import com.nanri.aiimage.modules.ziniao.model.cache.ZiniaoSessionCacheDto;
|
import com.nanri.aiimage.modules.ziniao.model.cache.ZiniaoSessionCacheDto;
|
||||||
import com.nanri.aiimage.modules.ziniao.model.cache.ZiniaoShopCacheDto;
|
import com.nanri.aiimage.modules.ziniao.model.cache.ZiniaoShopCacheDto;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
@@ -15,6 +15,7 @@ import java.util.List;
|
|||||||
|
|
||||||
@Service
|
@Service
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
|
@Slf4j
|
||||||
public class ZiniaoSessionCacheService {
|
public class ZiniaoSessionCacheService {
|
||||||
|
|
||||||
private final StringRedisTemplate stringRedisTemplate;
|
private final StringRedisTemplate stringRedisTemplate;
|
||||||
@@ -22,62 +23,96 @@ public class ZiniaoSessionCacheService {
|
|||||||
private final ZiniaoProperties ziniaoProperties;
|
private final ZiniaoProperties ziniaoProperties;
|
||||||
|
|
||||||
public void saveSession(ZiniaoSessionCacheDto session) {
|
public void saveSession(ZiniaoSessionCacheDto session) {
|
||||||
|
if (session == null || session.getSessionId() == null || session.getSessionId().isBlank()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
try {
|
try {
|
||||||
stringRedisTemplate.opsForValue().set(buildSessionKey(session.getSessionId()), objectMapper.writeValueAsString(session), Duration.ofHours(ziniaoProperties.getSessionTtlHours()));
|
stringRedisTemplate.opsForValue().set(
|
||||||
|
buildSessionKey(session.getSessionId()),
|
||||||
|
objectMapper.writeValueAsString(session),
|
||||||
|
Duration.ofHours(ziniaoProperties.getSessionTtlHours()));
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
throw new BusinessException("保存紫鸟会话失败");
|
log.warn("[ziniao-session-cache] save session degraded sessionId={} msg={}",
|
||||||
|
session.getSessionId(), ex.getMessage());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public ZiniaoSessionCacheDto getSession(String sessionId) {
|
public ZiniaoSessionCacheDto getSession(String sessionId) {
|
||||||
String raw = stringRedisTemplate.opsForValue().get(buildSessionKey(sessionId));
|
String raw;
|
||||||
|
try {
|
||||||
|
raw = stringRedisTemplate.opsForValue().get(buildSessionKey(sessionId));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[ziniao-session-cache] get session degraded sessionId={} msg={}", sessionId, ex.getMessage());
|
||||||
|
return null;
|
||||||
|
}
|
||||||
if (raw == null || raw.isBlank()) {
|
if (raw == null || raw.isBlank()) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
return objectMapper.readValue(raw, ZiniaoSessionCacheDto.class);
|
return objectMapper.readValue(raw, ZiniaoSessionCacheDto.class);
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
throw new BusinessException("读取紫鸟会话失败");
|
log.warn("[ziniao-session-cache] parse session degraded sessionId={} msg={}", sessionId, ex.getMessage());
|
||||||
|
return null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public void saveShops(String sessionId, List<ZiniaoShopCacheDto> shops) {
|
public void saveShops(String sessionId, List<ZiniaoShopCacheDto> shops) {
|
||||||
try {
|
try {
|
||||||
stringRedisTemplate.opsForValue().set(buildShopsKey(sessionId), objectMapper.writeValueAsString(shops), Duration.ofMinutes(ziniaoProperties.getShopsCacheMinutes()));
|
stringRedisTemplate.opsForValue().set(
|
||||||
|
buildShopsKey(sessionId),
|
||||||
|
objectMapper.writeValueAsString(shops),
|
||||||
|
Duration.ofMinutes(ziniaoProperties.getShopsCacheMinutes()));
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
throw new BusinessException("保存紫鸟店铺列表失败");
|
log.warn("[ziniao-session-cache] save shops degraded sessionId={} msg={}", sessionId, ex.getMessage());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public List<ZiniaoShopCacheDto> getShops(String sessionId) {
|
public List<ZiniaoShopCacheDto> getShops(String sessionId) {
|
||||||
String raw = stringRedisTemplate.opsForValue().get(buildShopsKey(sessionId));
|
String raw;
|
||||||
|
try {
|
||||||
|
raw = stringRedisTemplate.opsForValue().get(buildShopsKey(sessionId));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[ziniao-session-cache] get shops degraded sessionId={} msg={}", sessionId, ex.getMessage());
|
||||||
|
return List.of();
|
||||||
|
}
|
||||||
if (raw == null || raw.isBlank()) {
|
if (raw == null || raw.isBlank()) {
|
||||||
return List.of();
|
return List.of();
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
return objectMapper.readValue(raw, new TypeReference<List<ZiniaoShopCacheDto>>() {});
|
return objectMapper.readValue(raw, new TypeReference<List<ZiniaoShopCacheDto>>() {});
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
throw new BusinessException("读取紫鸟店铺列表失败");
|
log.warn("[ziniao-session-cache] parse shops degraded sessionId={} msg={}", sessionId, ex.getMessage());
|
||||||
|
return List.of();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public void saveCurrentShop(String sessionId, ZiniaoShopCacheDto shop) {
|
public void saveCurrentShop(String sessionId, ZiniaoShopCacheDto shop) {
|
||||||
try {
|
try {
|
||||||
stringRedisTemplate.opsForValue().set(buildCurrentShopKey(sessionId), objectMapper.writeValueAsString(shop), Duration.ofHours(ziniaoProperties.getSessionTtlHours()));
|
stringRedisTemplate.opsForValue().set(
|
||||||
|
buildCurrentShopKey(sessionId),
|
||||||
|
objectMapper.writeValueAsString(shop),
|
||||||
|
Duration.ofHours(ziniaoProperties.getSessionTtlHours()));
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
throw new BusinessException("保存当前紫鸟店铺失败");
|
log.warn("[ziniao-session-cache] save current shop degraded sessionId={} msg={}", sessionId, ex.getMessage());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public ZiniaoShopCacheDto getCurrentShop(String sessionId) {
|
public ZiniaoShopCacheDto getCurrentShop(String sessionId) {
|
||||||
String raw = stringRedisTemplate.opsForValue().get(buildCurrentShopKey(sessionId));
|
String raw;
|
||||||
|
try {
|
||||||
|
raw = stringRedisTemplate.opsForValue().get(buildCurrentShopKey(sessionId));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.warn("[ziniao-session-cache] get current shop degraded sessionId={} msg={}", sessionId, ex.getMessage());
|
||||||
|
return null;
|
||||||
|
}
|
||||||
if (raw == null || raw.isBlank()) {
|
if (raw == null || raw.isBlank()) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
return objectMapper.readValue(raw, ZiniaoShopCacheDto.class);
|
return objectMapper.readValue(raw, ZiniaoShopCacheDto.class);
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
throw new BusinessException("读取当前紫鸟店铺失败");
|
log.warn("[ziniao-session-cache] parse current shop degraded sessionId={} msg={}", sessionId, ex.getMessage());
|
||||||
|
return null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -101,6 +101,8 @@ public class ZiniaoShopIndexService {
|
|||||||
return pendingResult("店铺索引信息不完整,请等待后台刷新");
|
return pendingResult("店铺索引信息不完整,请等待后台刷新");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
boolean treatAsFresh = isFresh(entry) || shouldBypassFreshnessBecauseWhitelistFailure(entry);
|
||||||
|
|
||||||
if (!buildOpenStoreUrl) {
|
if (!buildOpenStoreUrl) {
|
||||||
ZiniaoShopMatchResultVo vo = new ZiniaoShopMatchResultVo();
|
ZiniaoShopMatchResultVo vo = new ZiniaoShopMatchResultVo();
|
||||||
vo.setMatched(true);
|
vo.setMatched(true);
|
||||||
@@ -110,8 +112,8 @@ public class ZiniaoShopIndexService {
|
|||||||
vo.setPlatform(entry.getPlatform());
|
vo.setPlatform(entry.getPlatform());
|
||||||
vo.setMatchedUserId(entry.getMatchedUserId());
|
vo.setMatchedUserId(entry.getMatchedUserId());
|
||||||
vo.setOpenStoreUrl(null);
|
vo.setOpenStoreUrl(null);
|
||||||
vo.setMatchStatus(isFresh(entry) ? MATCH_STATUS_MATCHED : MATCH_STATUS_STALE);
|
vo.setMatchStatus(treatAsFresh ? MATCH_STATUS_MATCHED : MATCH_STATUS_STALE);
|
||||||
vo.setMatchMessage(isFresh(entry) ? null : "店铺索引已过保鲜期,请等待后台刷新");
|
vo.setMatchMessage(treatAsFresh ? null : "店铺索引已过保鲜期,请等待后台刷新");
|
||||||
return vo;
|
return vo;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -145,8 +147,8 @@ public class ZiniaoShopIndexService {
|
|||||||
vo.setMatchStatus(MATCH_STATUS_STALE);
|
vo.setMatchStatus(MATCH_STATUS_STALE);
|
||||||
vo.setMatchMessage("打开店铺链接暂时无法生成,请稍后重试");
|
vo.setMatchMessage("打开店铺链接暂时无法生成,请稍后重试");
|
||||||
} else {
|
} else {
|
||||||
vo.setMatchStatus(isFresh(entry) ? MATCH_STATUS_MATCHED : MATCH_STATUS_STALE);
|
vo.setMatchStatus(treatAsFresh ? MATCH_STATUS_MATCHED : MATCH_STATUS_STALE);
|
||||||
vo.setMatchMessage(isFresh(entry) ? null : "店铺索引已过保鲜期,请等待后台刷新");
|
vo.setMatchMessage(treatAsFresh ? null : "店铺索引已过保鲜期,请等待后台刷新");
|
||||||
}
|
}
|
||||||
return vo;
|
return vo;
|
||||||
}
|
}
|
||||||
@@ -406,7 +408,19 @@ public class ZiniaoShopIndexService {
|
|||||||
private ZiniaoShopMatchResultVo staleOrPendingResult(String defaultMessage, ZiniaoShopIndexEntryDto entry) {
|
private ZiniaoShopMatchResultVo staleOrPendingResult(String defaultMessage, ZiniaoShopIndexEntryDto entry) {
|
||||||
if (entry != null && STATUS_STALE.equals(entry.getStatus())) {
|
if (entry != null && STATUS_STALE.equals(entry.getStatus())) {
|
||||||
ZiniaoShopMatchResultVo vo = new ZiniaoShopMatchResultVo();
|
ZiniaoShopMatchResultVo vo = new ZiniaoShopMatchResultVo();
|
||||||
vo.setMatched(false);
|
boolean hasReusableMatch = entry.getShopId() != null
|
||||||
|
&& !entry.getShopId().isBlank()
|
||||||
|
&& entry.getMatchedUserId() != null
|
||||||
|
&& entry.getMatchedUserId() > 0;
|
||||||
|
vo.setMatched(hasReusableMatch);
|
||||||
|
if (hasReusableMatch) {
|
||||||
|
vo.setShopId(entry.getShopId());
|
||||||
|
vo.setShopName(entry.getShopName());
|
||||||
|
vo.setCompanyName(entry.getCompanyName());
|
||||||
|
vo.setPlatform(entry.getPlatform());
|
||||||
|
vo.setMatchedUserId(entry.getMatchedUserId());
|
||||||
|
vo.setOpenStoreUrl(null);
|
||||||
|
}
|
||||||
vo.setMatchStatus(MATCH_STATUS_STALE);
|
vo.setMatchStatus(MATCH_STATUS_STALE);
|
||||||
vo.setMatchMessage(defaultMessage);
|
vo.setMatchMessage(defaultMessage);
|
||||||
return vo;
|
return vo;
|
||||||
@@ -538,6 +552,25 @@ public class ZiniaoShopIndexService {
|
|||||||
return Instant.now().toEpochMilli() - entry.getLastRefreshedAt() <= freshWindowMillis;
|
return Instant.now().toEpochMilli() - entry.getLastRefreshedAt() <= freshWindowMillis;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private boolean shouldBypassFreshnessBecauseWhitelistFailure(ZiniaoShopIndexEntryDto entry) {
|
||||||
|
if (entry == null || !STATUS_ACTIVE.equals(entry.getStatus())) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (entry.getLastRefreshedAt() == null || entry.getLastRefreshedAt() <= 0) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
ZiniaoShopIndexRefreshCursorDto cursor = getRefreshCursor();
|
||||||
|
if (cursor == null || !"FAILED".equals(cursor.getStatus())) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
String message = cursor.getMessage();
|
||||||
|
if (message == null || !message.contains("白名单")) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
Long lastFinishedAt = cursor.getLastFinishedAt();
|
||||||
|
return lastFinishedAt != null && lastFinishedAt >= entry.getLastRefreshedAt();
|
||||||
|
}
|
||||||
|
|
||||||
private Duration resolveEntryTtl() {
|
private Duration resolveEntryTtl() {
|
||||||
Integer ttlHours = ziniaoProperties.getShopIndexEntryTtlHours();
|
Integer ttlHours = ziniaoProperties.getShopIndexEntryTtlHours();
|
||||||
if (ttlHours == null || ttlHours <= 0) {
|
if (ttlHours == null || ttlHours <= 0) {
|
||||||
|
|||||||
@@ -3,15 +3,22 @@
|
|||||||
|
|
||||||
AIIMAGE_SERVER_PORT=18080
|
AIIMAGE_SERVER_PORT=18080
|
||||||
|
|
||||||
AIIMAGE_DB_URL=jdbc:mysql://159.75.121.33:3306/aiimage?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai&useSSL=false
|
AIIMAGE_DB_URL=jdbc:mysql://127.0.0.1:3306/aiimage?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai&useSSL=false
|
||||||
AIIMAGE_DB_USERNAME=AIimage
|
AIIMAGE_DB_USERNAME=change-me
|
||||||
AIIMAGE_DB_PASSWORD=WTFrb5y6hNLz6hNy
|
AIIMAGE_DB_PASSWORD=change-me
|
||||||
|
|
||||||
AIIMAGE_OSS_REGION=cn-hangzhou
|
AIIMAGE_OSS_REGION=cn-hangzhou
|
||||||
AIIMAGE_OSS_ENDPOINT=oss-cn-hangzhou.aliyuncs.com
|
AIIMAGE_OSS_ENDPOINT=oss-cn-hangzhou.aliyuncs.com
|
||||||
AIIMAGE_OSS_BUCKET=nanri-ai-images
|
AIIMAGE_OSS_BUCKET=change-me
|
||||||
AIIMAGE_OSS_ACCESS_KEY_ID=LTAI5tNpyvzMNz9f2dHarsm8
|
AIIMAGE_OSS_ACCESS_KEY_ID=change-me
|
||||||
AIIMAGE_OSS_ACCESS_KEY_SECRET=bQSZnFH455i8tzyOgeahJmUzwmhynz
|
AIIMAGE_OSS_ACCESS_KEY_SECRET=change-me
|
||||||
|
|
||||||
|
AIIMAGE_TRANSIENT_STORAGE_ENABLED=true
|
||||||
|
AIIMAGE_TRANSIENT_STORAGE_ENDPOINT=http://change-me
|
||||||
|
AIIMAGE_TRANSIENT_STORAGE_BUCKET=change-me
|
||||||
|
AIIMAGE_TRANSIENT_STORAGE_ACCESS_KEY_ID=change-me
|
||||||
|
AIIMAGE_TRANSIENT_STORAGE_ACCESS_KEY_SECRET=change-me
|
||||||
|
AIIMAGE_TRANSIENT_STORAGE_REGION=us-east-1
|
||||||
|
|
||||||
AIIMAGE_STORAGE_LOCAL_TEMP_DIR=F:/project/crawler-plugin/backend-java/data/tmp
|
AIIMAGE_STORAGE_LOCAL_TEMP_DIR=F:/project/crawler-plugin/backend-java/data/tmp
|
||||||
|
|
||||||
@@ -19,6 +26,23 @@ AIIMAGE_MODULE_CLEANUP_ENABLED=true
|
|||||||
AIIMAGE_MODULE_CLEANUP_CRON=0 0 0 * * *
|
AIIMAGE_MODULE_CLEANUP_CRON=0 0 0 * * *
|
||||||
AIIMAGE_MODULE_CLEANUP_MODULE_TYPES=DEDUPE,SPLIT,CONVERT,DELETE_BRAND
|
AIIMAGE_MODULE_CLEANUP_MODULE_TYPES=DEDUPE,SPLIT,CONVERT,DELETE_BRAND
|
||||||
|
|
||||||
|
AIIMAGE_APPEARANCE_PATENT_COZE_BASE_URL=https://api.coze.cn
|
||||||
|
AIIMAGE_APPEARANCE_PATENT_COZE_WORKFLOW_PATH=/v1/workflow/run
|
||||||
|
AIIMAGE_APPEARANCE_PATENT_COZE_WORKFLOW_ID=7632683471312355338
|
||||||
|
AIIMAGE_APPEARANCE_PATENT_COZE_TOKEN=
|
||||||
|
AIIMAGE_APPEARANCE_PATENT_COZE_BATCH_SIZE=10
|
||||||
|
AIIMAGE_APPEARANCE_PATENT_COZE_READ_TIMEOUT_MILLIS=60000
|
||||||
|
AIIMAGE_APPEARANCE_PATENT_STALE_TIMEOUT_MINUTES=20
|
||||||
|
|
||||||
|
AIIMAGE_ROCKETMQ_NAME_SERVER=121.196.149.225:9876
|
||||||
|
AIIMAGE_RESULT_FILE_JOB_MQ_ENABLED=true
|
||||||
|
AIIMAGE_RESULT_FILE_JOB_LOCAL_DISPATCH_ENABLED=true
|
||||||
|
AIIMAGE_RESULT_FILE_JOB_LOCAL_DISPATCH_POOL_SIZE=2
|
||||||
|
AIIMAGE_RESULT_FILE_JOB_LOCAL_DISPATCH_QUEUE_CAPACITY=200
|
||||||
|
AIIMAGE_RESULT_FILE_JOB_LOCAL_WORKER_ENABLED=false
|
||||||
|
AIIMAGE_RESULT_FILE_JOB_LOCAL_WORKER_DELAY_MS=10000
|
||||||
|
AIIMAGE_RESULT_FILE_JOB_STUCK_TIMEOUT_MINUTES=30
|
||||||
|
|
||||||
AIIMAGE_ZINIAO_ENABLED=false
|
AIIMAGE_ZINIAO_ENABLED=false
|
||||||
AIIMAGE_ZINIAO_BASE_URL=https://sbappstoreapi.ziniao.com/openapi-router
|
AIIMAGE_ZINIAO_BASE_URL=https://sbappstoreapi.ziniao.com/openapi-router
|
||||||
AIIMAGE_ZINIAO_API_KEY=change-me
|
AIIMAGE_ZINIAO_API_KEY=change-me
|
||||||
|
|||||||
@@ -28,6 +28,12 @@ spring:
|
|||||||
database: ${AIIMAGE_REDIS_DATABASE:0}
|
database: ${AIIMAGE_REDIS_DATABASE:0}
|
||||||
timeout: ${AIIMAGE_REDIS_TIMEOUT:5s}
|
timeout: ${AIIMAGE_REDIS_TIMEOUT:5s}
|
||||||
|
|
||||||
|
rocketmq:
|
||||||
|
name-server: ${AIIMAGE_ROCKETMQ_NAME_SERVER:121.196.149.225:9876}
|
||||||
|
producer:
|
||||||
|
group: ${AIIMAGE_ROCKETMQ_PRODUCER_GROUP:aiimage-backend-producer}
|
||||||
|
send-message-timeout: ${AIIMAGE_ROCKETMQ_SEND_TIMEOUT_MS:3000}
|
||||||
|
|
||||||
management:
|
management:
|
||||||
health:
|
health:
|
||||||
db:
|
db:
|
||||||
@@ -67,6 +73,13 @@ aiimage:
|
|||||||
bucket: ${AIIMAGE_OSS_BUCKET:change-me}
|
bucket: ${AIIMAGE_OSS_BUCKET:change-me}
|
||||||
access-key-id: ${AIIMAGE_OSS_ACCESS_KEY_ID:change-me}
|
access-key-id: ${AIIMAGE_OSS_ACCESS_KEY_ID:change-me}
|
||||||
access-key-secret: ${AIIMAGE_OSS_ACCESS_KEY_SECRET:change-me}
|
access-key-secret: ${AIIMAGE_OSS_ACCESS_KEY_SECRET:change-me}
|
||||||
|
transient-storage:
|
||||||
|
enabled: ${AIIMAGE_TRANSIENT_STORAGE_ENABLED:false}
|
||||||
|
endpoint: ${AIIMAGE_TRANSIENT_STORAGE_ENDPOINT:}
|
||||||
|
bucket: ${AIIMAGE_TRANSIENT_STORAGE_BUCKET:}
|
||||||
|
access-key-id: ${AIIMAGE_TRANSIENT_STORAGE_ACCESS_KEY_ID:}
|
||||||
|
access-key-secret: ${AIIMAGE_TRANSIENT_STORAGE_ACCESS_KEY_SECRET:}
|
||||||
|
region: ${AIIMAGE_TRANSIENT_STORAGE_REGION:us-east-1}
|
||||||
storage:
|
storage:
|
||||||
local-temp-dir: ${AIIMAGE_STORAGE_LOCAL_TEMP_DIR:./data/tmp}
|
local-temp-dir: ${AIIMAGE_STORAGE_LOCAL_TEMP_DIR:./data/tmp}
|
||||||
cleanup-enabled: ${AIIMAGE_STORAGE_CLEANUP_ENABLED:true}
|
cleanup-enabled: ${AIIMAGE_STORAGE_CLEANUP_ENABLED:true}
|
||||||
@@ -96,6 +109,8 @@ aiimage:
|
|||||||
enabled: ${AIIMAGE_MODULE_CLEANUP_ENABLED:true}
|
enabled: ${AIIMAGE_MODULE_CLEANUP_ENABLED:true}
|
||||||
cron: ${AIIMAGE_MODULE_CLEANUP_CRON:0 0 0 * * *}
|
cron: ${AIIMAGE_MODULE_CLEANUP_CRON:0 0 0 * * *}
|
||||||
module-types: ${AIIMAGE_MODULE_CLEANUP_MODULE_TYPES:DEDUPE,SPLIT,CONVERT,DELETE_BRAND,PRODUCT_RISK_RESOLVE,PRICE_TRACK,SHOP_MATCH,PATROL_DELETE}
|
module-types: ${AIIMAGE_MODULE_CLEANUP_MODULE_TYPES:DEDUPE,SPLIT,CONVERT,DELETE_BRAND,PRODUCT_RISK_RESOLVE,PRICE_TRACK,SHOP_MATCH,PATROL_DELETE}
|
||||||
|
permission-schema-init:
|
||||||
|
enabled: ${AIIMAGE_PERMISSION_SCHEMA_INIT_ENABLED:false}
|
||||||
task-pressure:
|
task-pressure:
|
||||||
local-task-entity-cache-millis: ${AIIMAGE_TASK_LOCAL_ENTITY_CACHE_MILLIS:3000}
|
local-task-entity-cache-millis: ${AIIMAGE_TASK_LOCAL_ENTITY_CACHE_MILLIS:3000}
|
||||||
db-select-batch-size: ${AIIMAGE_TASK_DB_SELECT_BATCH_SIZE:200}
|
db-select-batch-size: ${AIIMAGE_TASK_DB_SELECT_BATCH_SIZE:200}
|
||||||
@@ -103,6 +118,28 @@ aiimage:
|
|||||||
scope-payload-buffer-retention-hours: ${AIIMAGE_TASK_SCOPE_PAYLOAD_BUFFER_RETENTION_HOURS:24}
|
scope-payload-buffer-retention-hours: ${AIIMAGE_TASK_SCOPE_PAYLOAD_BUFFER_RETENTION_HOURS:24}
|
||||||
scope-payload-recovery-max-files: ${AIIMAGE_TASK_SCOPE_PAYLOAD_RECOVERY_MAX_FILES:200}
|
scope-payload-recovery-max-files: ${AIIMAGE_TASK_SCOPE_PAYLOAD_RECOVERY_MAX_FILES:200}
|
||||||
scope-payload-cleanup-cron: ${AIIMAGE_TASK_SCOPE_PAYLOAD_CLEANUP_CRON:15 */30 * * * *}
|
scope-payload-cleanup-cron: ${AIIMAGE_TASK_SCOPE_PAYLOAD_CLEANUP_CRON:15 */30 * * * *}
|
||||||
|
result-file-job:
|
||||||
|
mq-enabled: ${AIIMAGE_RESULT_FILE_JOB_MQ_ENABLED:false}
|
||||||
|
topic: ${AIIMAGE_RESULT_FILE_JOB_TOPIC:aiimage-result-file-job}
|
||||||
|
consumer-group: ${AIIMAGE_RESULT_FILE_JOB_CONSUMER_GROUP:aiimage-result-file-job-consumer}
|
||||||
|
local-dispatch-enabled: ${AIIMAGE_RESULT_FILE_JOB_LOCAL_DISPATCH_ENABLED:true}
|
||||||
|
local-dispatch-pool-size: ${AIIMAGE_RESULT_FILE_JOB_LOCAL_DISPATCH_POOL_SIZE:2}
|
||||||
|
local-dispatch-queue-capacity: ${AIIMAGE_RESULT_FILE_JOB_LOCAL_DISPATCH_QUEUE_CAPACITY:200}
|
||||||
|
local-worker-enabled: ${AIIMAGE_RESULT_FILE_JOB_LOCAL_WORKER_ENABLED:true}
|
||||||
|
local-worker-delay-ms: ${AIIMAGE_RESULT_FILE_JOB_LOCAL_WORKER_DELAY_MS:10000}
|
||||||
|
stuck-scan-delay-ms: ${AIIMAGE_RESULT_FILE_JOB_STUCK_SCAN_DELAY_MS:60000}
|
||||||
|
stuck-timeout-minutes: ${AIIMAGE_RESULT_FILE_JOB_STUCK_TIMEOUT_MINUTES:30}
|
||||||
|
batch-size: ${AIIMAGE_RESULT_FILE_JOB_BATCH_SIZE:20}
|
||||||
|
appearance-patent:
|
||||||
|
coze-base-url: ${AIIMAGE_APPEARANCE_PATENT_COZE_BASE_URL:https://api.coze.cn}
|
||||||
|
coze-workflow-path: ${AIIMAGE_APPEARANCE_PATENT_COZE_WORKFLOW_PATH:/v1/workflow/run}
|
||||||
|
coze-workflow-id: ${AIIMAGE_APPEARANCE_PATENT_COZE_WORKFLOW_ID:7632683471312355338}
|
||||||
|
coze-token: ${AIIMAGE_APPEARANCE_PATENT_COZE_TOKEN:}
|
||||||
|
coze-batch-size: ${AIIMAGE_APPEARANCE_PATENT_COZE_BATCH_SIZE:10}
|
||||||
|
coze-connect-timeout-millis: ${AIIMAGE_APPEARANCE_PATENT_COZE_CONNECT_TIMEOUT_MILLIS:10000}
|
||||||
|
coze-read-timeout-millis: ${AIIMAGE_APPEARANCE_PATENT_COZE_READ_TIMEOUT_MILLIS:60000}
|
||||||
|
stale-timeout-minutes: ${AIIMAGE_APPEARANCE_PATENT_STALE_TIMEOUT_MINUTES:20}
|
||||||
|
stale-finalize-cron: ${AIIMAGE_APPEARANCE_PATENT_STALE_FINALIZE_CRON:0 */2 * * * *}
|
||||||
security:
|
security:
|
||||||
shop-credential-key: ${AIIMAGE_SHOP_CREDENTIAL_KEY:change-me-shop-credential-key}
|
shop-credential-key: ${AIIMAGE_SHOP_CREDENTIAL_KEY:change-me-shop-credential-key}
|
||||||
internal-token: ${AIIMAGE_INTERNAL_TOKEN:}
|
internal-token: ${AIIMAGE_INTERNAL_TOKEN:}
|
||||||
|
|||||||
@@ -0,0 +1,85 @@
|
|||||||
|
-- 外观专利检测模块复用通用任务表:
|
||||||
|
-- biz_file_task 保存任务主记录,module_type = APPEARANCE_PATENT
|
||||||
|
-- biz_file_result 保存最终 xlsx 结果文件和历史记录
|
||||||
|
-- biz_task_scope_state 保存解析载荷 OSS 指针、scope 状态和心跳辅助状态
|
||||||
|
-- biz_task_chunk 保存 Python 回传分片和 Java/Coze 处理后的分片结果
|
||||||
|
|
||||||
|
-- 兼容老库:如果通用任务表缺少 user_id,则补齐用户归属字段。
|
||||||
|
SET @biz_file_task_user_id_exists := (
|
||||||
|
SELECT COUNT(1)
|
||||||
|
FROM information_schema.COLUMNS
|
||||||
|
WHERE TABLE_SCHEMA = DATABASE()
|
||||||
|
AND TABLE_NAME = 'biz_file_task'
|
||||||
|
AND COLUMN_NAME = 'user_id'
|
||||||
|
);
|
||||||
|
SET @sql_add_biz_file_task_user_id := IF(
|
||||||
|
@biz_file_task_user_id_exists = 0,
|
||||||
|
'ALTER TABLE biz_file_task ADD COLUMN user_id BIGINT NULL COMMENT ''用户ID'' AFTER created_by',
|
||||||
|
'SELECT 1'
|
||||||
|
);
|
||||||
|
PREPARE stmt_add_biz_file_task_user_id FROM @sql_add_biz_file_task_user_id;
|
||||||
|
EXECUTE stmt_add_biz_file_task_user_id;
|
||||||
|
DEALLOCATE PREPARE stmt_add_biz_file_task_user_id;
|
||||||
|
|
||||||
|
SET @biz_file_result_user_id_exists := (
|
||||||
|
SELECT COUNT(1)
|
||||||
|
FROM information_schema.COLUMNS
|
||||||
|
WHERE TABLE_SCHEMA = DATABASE()
|
||||||
|
AND TABLE_NAME = 'biz_file_result'
|
||||||
|
AND COLUMN_NAME = 'user_id'
|
||||||
|
);
|
||||||
|
SET @sql_add_biz_file_result_user_id := IF(
|
||||||
|
@biz_file_result_user_id_exists = 0,
|
||||||
|
'ALTER TABLE biz_file_result ADD COLUMN user_id BIGINT NULL COMMENT ''用户ID'' AFTER error_message',
|
||||||
|
'SELECT 1'
|
||||||
|
);
|
||||||
|
PREPARE stmt_add_biz_file_result_user_id FROM @sql_add_biz_file_result_user_id;
|
||||||
|
EXECUTE stmt_add_biz_file_result_user_id;
|
||||||
|
DEALLOCATE PREPARE stmt_add_biz_file_result_user_id;
|
||||||
|
|
||||||
|
-- 外观专利检测常用查询索引:按用户查任务、按用户查历史。
|
||||||
|
SET @idx_file_task_module_user_created_exists := (
|
||||||
|
SELECT COUNT(1)
|
||||||
|
FROM information_schema.STATISTICS
|
||||||
|
WHERE TABLE_SCHEMA = DATABASE()
|
||||||
|
AND TABLE_NAME = 'biz_file_task'
|
||||||
|
AND INDEX_NAME = 'idx_biz_file_task_module_user_created'
|
||||||
|
);
|
||||||
|
SET @sql_add_idx_file_task_module_user_created := IF(
|
||||||
|
@idx_file_task_module_user_created_exists = 0,
|
||||||
|
'ALTER TABLE biz_file_task ADD INDEX idx_biz_file_task_module_user_created (module_type, user_id, created_at)',
|
||||||
|
'SELECT 1'
|
||||||
|
);
|
||||||
|
PREPARE stmt_add_idx_file_task_module_user_created FROM @sql_add_idx_file_task_module_user_created;
|
||||||
|
EXECUTE stmt_add_idx_file_task_module_user_created;
|
||||||
|
DEALLOCATE PREPARE stmt_add_idx_file_task_module_user_created;
|
||||||
|
|
||||||
|
SET @idx_file_result_module_user_created_exists := (
|
||||||
|
SELECT COUNT(1)
|
||||||
|
FROM information_schema.STATISTICS
|
||||||
|
WHERE TABLE_SCHEMA = DATABASE()
|
||||||
|
AND TABLE_NAME = 'biz_file_result'
|
||||||
|
AND INDEX_NAME = 'idx_biz_file_result_module_user_created'
|
||||||
|
);
|
||||||
|
SET @sql_add_idx_file_result_module_user_created := IF(
|
||||||
|
@idx_file_result_module_user_created_exists = 0,
|
||||||
|
'ALTER TABLE biz_file_result ADD INDEX idx_biz_file_result_module_user_created (module_type, user_id, created_at)',
|
||||||
|
'SELECT 1'
|
||||||
|
);
|
||||||
|
PREPARE stmt_add_idx_file_result_module_user_created FROM @sql_add_idx_file_result_module_user_created;
|
||||||
|
EXECUTE stmt_add_idx_file_result_module_user_created;
|
||||||
|
DEALLOCATE PREPARE stmt_add_idx_file_result_module_user_created;
|
||||||
|
|
||||||
|
-- 前端栏目权限兜底:外观专利检测属于前端工具下的页面。
|
||||||
|
INSERT INTO `columns` (`name`, `column_key`, `menu_type`, `route_path`, `sort_order`)
|
||||||
|
SELECT '外观专利检测', 'appearance_patent', 'app', 'appearance-patent', 111
|
||||||
|
WHERE NOT EXISTS (
|
||||||
|
SELECT 1 FROM `columns` WHERE `column_key` = 'appearance_patent'
|
||||||
|
);
|
||||||
|
|
||||||
|
UPDATE `columns`
|
||||||
|
SET `name` = '外观专利检测',
|
||||||
|
`menu_type` = 'app',
|
||||||
|
`route_path` = 'appearance-patent',
|
||||||
|
`sort_order` = 111
|
||||||
|
WHERE `column_key` = 'appearance_patent';
|
||||||
@@ -0,0 +1,74 @@
|
|||||||
|
CREATE TABLE IF NOT EXISTS biz_task_result_payload (
|
||||||
|
id BIGINT PRIMARY KEY AUTO_INCREMENT COMMENT 'primary key',
|
||||||
|
task_id BIGINT NOT NULL COMMENT 'task id',
|
||||||
|
module_type VARCHAR(32) NOT NULL COMMENT 'module type',
|
||||||
|
scope_key VARCHAR(1000) NOT NULL COMMENT 'scope identifier, such as shop name or file key',
|
||||||
|
scope_hash CHAR(64) NOT NULL COMMENT 'hash of scope key',
|
||||||
|
submission_id VARCHAR(128) NULL COMMENT 'idempotency key from submitter when available',
|
||||||
|
payload_json JSON NOT NULL COMMENT 'raw or normalized callback payload',
|
||||||
|
payload_hash CHAR(64) NOT NULL COMMENT 'sha256 of payload json',
|
||||||
|
version BIGINT NOT NULL DEFAULT 1 COMMENT 'scope payload version',
|
||||||
|
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'created time',
|
||||||
|
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'updated time',
|
||||||
|
UNIQUE KEY uk_task_scope_submission (task_id, module_type, scope_hash, submission_id),
|
||||||
|
KEY idx_task_module_scope (task_id, module_type, scope_hash),
|
||||||
|
KEY idx_task_module_updated (task_id, module_type, updated_at)
|
||||||
|
) COMMENT='task result callback payload detail';
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS biz_task_file_job (
|
||||||
|
id BIGINT PRIMARY KEY AUTO_INCREMENT COMMENT 'primary key',
|
||||||
|
task_id BIGINT NOT NULL COMMENT 'task id',
|
||||||
|
module_type VARCHAR(32) NOT NULL COMMENT 'module type',
|
||||||
|
result_id BIGINT NULL COMMENT 'biz_file_result id when job is scoped to one result row',
|
||||||
|
scope_key VARCHAR(1000) NULL COMMENT 'scope identifier',
|
||||||
|
job_type VARCHAR(32) NOT NULL DEFAULT 'ASSEMBLE_RESULT' COMMENT 'job type',
|
||||||
|
status VARCHAR(32) NOT NULL DEFAULT 'PENDING' COMMENT 'PENDING/RUNNING/SUCCESS/FAILED',
|
||||||
|
retry_count INT NOT NULL DEFAULT 0 COMMENT 'retry count',
|
||||||
|
error_message VARCHAR(1000) NULL COMMENT 'last error',
|
||||||
|
result_file_url VARCHAR(1000) NULL COMMENT 'generated file object key or url',
|
||||||
|
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'created time',
|
||||||
|
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'updated time',
|
||||||
|
finished_at DATETIME NULL COMMENT 'finished time',
|
||||||
|
UNIQUE KEY uk_file_job_scope (task_id, module_type, result_id, job_type),
|
||||||
|
KEY idx_file_job_status (status, updated_at),
|
||||||
|
KEY idx_file_job_task (task_id, module_type)
|
||||||
|
) COMMENT='async result file assembly job';
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS biz_task_result_item (
|
||||||
|
id BIGINT PRIMARY KEY AUTO_INCREMENT COMMENT 'primary key',
|
||||||
|
task_id BIGINT NOT NULL COMMENT 'task id',
|
||||||
|
module_type VARCHAR(32) NOT NULL COMMENT 'module type',
|
||||||
|
result_id BIGINT NULL COMMENT 'biz_file_result id when available',
|
||||||
|
scope_key VARCHAR(1000) NOT NULL COMMENT 'scope identifier, such as shop name or file key',
|
||||||
|
scope_hash CHAR(64) NOT NULL COMMENT 'hash of scope key',
|
||||||
|
item_key VARCHAR(512) NOT NULL COMMENT 'dedupe key, such as country + asin',
|
||||||
|
country_code VARCHAR(32) NULL COMMENT 'country code',
|
||||||
|
asin VARCHAR(128) NULL COMMENT 'asin or item id',
|
||||||
|
status VARCHAR(64) NULL COMMENT 'business status',
|
||||||
|
payload_json JSON NOT NULL COMMENT 'normalized row payload',
|
||||||
|
payload_hash CHAR(64) NOT NULL COMMENT 'sha256 of payload json',
|
||||||
|
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'created time',
|
||||||
|
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'updated time',
|
||||||
|
UNIQUE KEY uk_task_scope_item (task_id, module_type, scope_hash, item_key),
|
||||||
|
KEY idx_result_item_result (result_id),
|
||||||
|
KEY idx_result_item_task (task_id, module_type),
|
||||||
|
KEY idx_result_item_country_asin (task_id, module_type, country_code, asin)
|
||||||
|
) COMMENT='normalized task result item detail';
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS biz_task_progress_snapshot (
|
||||||
|
id BIGINT PRIMARY KEY AUTO_INCREMENT COMMENT 'primary key',
|
||||||
|
task_id BIGINT NOT NULL COMMENT 'task id',
|
||||||
|
module_type VARCHAR(32) NOT NULL COMMENT 'module type',
|
||||||
|
status VARCHAR(32) NOT NULL COMMENT 'task status',
|
||||||
|
total_count INT NOT NULL DEFAULT 0 COMMENT 'total item count',
|
||||||
|
success_count INT NOT NULL DEFAULT 0 COMMENT 'success count',
|
||||||
|
failed_count INT NOT NULL DEFAULT 0 COMMENT 'failed count',
|
||||||
|
pending_count INT NOT NULL DEFAULT 0 COMMENT 'pending count',
|
||||||
|
current_scope_key VARCHAR(1000) NULL COMMENT 'current scope identifier',
|
||||||
|
message VARCHAR(1000) NULL COMMENT 'progress message',
|
||||||
|
snapshot_json JSON NULL COMMENT 'small progress snapshot',
|
||||||
|
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'created time',
|
||||||
|
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'updated time',
|
||||||
|
UNIQUE KEY uk_task_progress_snapshot (task_id, module_type),
|
||||||
|
KEY idx_progress_module_status (module_type, status, updated_at)
|
||||||
|
) COMMENT='lightweight task progress snapshot';
|
||||||
Binary file not shown.
65
backend/codex-flask-local.err.log
Normal file
65
backend/codex-flask-local.err.log
Normal file
@@ -0,0 +1,65 @@
|
|||||||
|
WARNING: This is a development server. Do not use it in a production deployment. Use a production WSGI server instead.
|
||||||
|
* Running on all addresses (0.0.0.0)
|
||||||
|
* Running on http://127.0.0.1:15124
|
||||||
|
* Running on http://192.168.9.14:15124
|
||||||
|
Press CTRL+C to quit
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:32:53] "POST /login HTTP/1.1" 302 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:32:54] "GET /api/admin/shop-manage-groups HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:32:55] "POST /api/admin/query-asins/import HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:32:55] "GET /api/admin/query-asins/import/3bda54f728ff447f9edc6d80aa47986c HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:32:56] "GET /api/admin/query-asins?page=1&page_size=20&group_id=13&shop_name=codex_qasin_wide_20260427181749 HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:32:57] "POST /api/admin/query-asins/import HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:32:57] "GET /api/admin/query-asins/import/7779a5fac0c840138579f549d297e016 HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:32:58] "GET /api/admin/query-asins?page=1&page_size=20&group_id=13&shop_name=codex_qasin_narrow_a_20260427181749 HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:32:59] "GET /api/admin/query-asins?page=1&page_size=20&group_id=13&shop_name=codex_qasin_narrow_b_20260427181749 HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:32:59] "POST /api/admin/query-asins/delete-import HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:33:00] "GET /api/admin/query-asins/delete-import/474b72f74c414d94a534069668cdbe69 HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:33:01] "GET /api/admin/query-asins?page=1&page_size=20&group_id=13&shop_name=codex_qasin_wide_20260427181749 HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:33:01] "POST /api/admin/query-asins/delete-import HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:33:02] "GET /api/admin/query-asins/delete-import/4cdd99881e174c638a531f0805612722 HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:33:03] "GET /api/admin/query-asins?page=1&page_size=20&group_id=13&shop_name=codex_qasin_narrow_a_20260427181749 HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:33:03] "GET /api/admin/query-asins?page=1&page_size=20&group_id=13&shop_name=codex_qasin_narrow_b_20260427181749 HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:35:51] "GET /admin HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:35:51] "GET /api/admin/current-user HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:35:51] "GET /api/admin/current-user/menus HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:35:52] "GET /api/admin/shop-manage-groups HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:35:52] "GET /api/admin/users?page=1&page_size=999 HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:35:53] "GET /api/admin/users?page=1&page_size=15 HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:35:54] "GET /api/admin/columns HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:35:54] "GET /api/admin/skip-price-asins?page=1&page_size=15 HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:35:56] "GET /api/admin/columns HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:38:01] "GET /api/admin/dedupe-total-data?page=1&page_size=15 HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:38:04] "GET /api/admin/skip-price-asins?page=1&page_size=15 HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:38:06] "GET /api/admin/query-asins?page=1&page_size=15 HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:39:08] "GET /admin HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:39:08] "GET /api/admin/current-user HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:39:09] "GET /api/admin/shop-manage-groups HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:39:09] "GET /api/admin/current-user/menus HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:39:09] "GET /api/admin/users?page=1&page_size=999 HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:39:10] "GET /api/admin/users?page=1&page_size=15 HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:39:11] "GET /api/admin/columns HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:39:11] "GET /api/admin/skip-price-asins?page=1&page_size=15 HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:39:14] "GET /api/admin/columns HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:39:19] "GET /api/admin/query-asins?page=1&page_size=15 HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:39:22] "GET /api/admin/skip-price-asins?page=1&page_size=15 HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:40:07] "GET /api/admin/query-asins?page=1&page_size=15 HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:43:55] "POST /api/admin/query-asins/import HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:43:55] "GET /api/admin/query-asins/import/57ff18ffe1de4c6f93a627cf163f3b99 HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:43:56] "GET /api/admin/query-asins?page=1&page_size=15 HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:44:09] "POST /api/admin/query-asins/import HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:44:10] "GET /api/admin/query-asins/import/094bd7fc53944e2582aa6946b64f973f HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:44:10] "GET /api/admin/query-asins?page=1&page_size=15 HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:44:18] "POST /api/admin/query-asins/delete-import HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:44:19] "GET /api/admin/query-asins/delete-import/0f2e8441746f40309e77042f576842a8 HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:44:19] "GET /api/admin/query-asins?page=1&page_size=15 HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:44:25] "POST /api/admin/query-asins/delete-import HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:44:25] "GET /api/admin/query-asins/delete-import/b6369ca86d904a26ae1badbe89d512d4 HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:44:26] "GET /api/admin/query-asins?page=1&page_size=15 HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:44:31] "GET /admin HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:44:32] "GET /api/admin/current-user HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:44:32] "GET /api/admin/current-user/menus HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:44:32] "GET /api/admin/shop-manage-groups HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:44:33] "GET /api/admin/users?page=1&page_size=999 HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:44:34] "GET /api/admin/users?page=1&page_size=15 HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:44:34] "GET /api/admin/columns HTTP/1.1" 200 -
|
||||||
|
127.0.0.1 - - [27/Apr/2026 18:44:38] "GET /api/admin/columns HTTP/1.1" 200 -
|
||||||
2
backend/codex-flask-local.out.log
Normal file
2
backend/codex-flask-local.out.log
Normal file
@@ -0,0 +1,2 @@
|
|||||||
|
* Serving Flask app 'app'
|
||||||
|
* Debug mode: off
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user