Merge master into dev/koko after patrol delete fixes

The merge keeps dev/koko patrol-delete asset fallback behavior while accepting the latest master feature work. Conflicts were limited to ignore rules, local environment/config artifacts, tracked pycache binaries, and the Flask main blueprint.

Constraint: master and dev/koko both edited app/blueprints/main.py around brand and asset serving

Rejected: Drop dev/koko asset fallback logic | patrol delete and rebuilt Vite assets still need hashed asset resolution across app/new_web_source locations

Confidence: medium

Scope-risk: broad

Directive: app/.env remains a tracked local-config file in this repository; do not print or normalize secrets during conflict handling

Tested: uv run python -m py_compile blueprints\\main.py amazon\\base.py amazon\\main.py

Tested: uv run --group dev pytest tests\\test_amazon_base.py tests\\test_patrol_delete.py

Not-tested: Full backend-java/frontend-vue test suites
This commit is contained in:
koko
2026-04-28 20:55:27 +08:00
170 changed files with 7985 additions and 7475 deletions
+2
View File
@@ -105,3 +105,5 @@ xlsx/
.codex
.rtk
OPS_REDIS_MYSQL_OPTIMIZATION_NOTES.md
架构.md
*ts.%
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
+7 -3
View File
@@ -475,7 +475,8 @@ class AmzoneApprove(AmamzonBase):
retry_num = 0
already_asin = set()
now_page = None
while retry_num < 3: # 最多重试3次
max_retry_num = 5
while retry_num < max_retry_num: # 最多重试3次
# if num > 3: #测试
# return
# 等待加载完成
@@ -505,8 +506,11 @@ class AmzoneApprove(AmamzonBase):
sku_ls = self.tab.eles("xpath://div[@data-sku]",timeout=10)
print(f"获取到 {len(sku_ls)}")
if len(sku_ls) == 0:
print(f"搜索不出SKU停止")
break
retry_num += 1
print(f"没有获取到 ASIN 商品,重试{retry_num}/{max_retry_num}")
self.tab.refresh()
self.tab.wait.doc_loaded(raise_err=False, timeout=120)
continue
# for sku_ele in sku_ls[0:2]:
for sku_ele in sku_ls[1:]:
# solve_problem = sku_ele.eles('xpath:.//kat-link[@label="解决商品信息问题"]')
+107 -6484
View File
File diff suppressed because it is too large Load Diff
+69 -50
View File
@@ -28,7 +28,7 @@ class RepricingLogic:
if is_first_time:
return rank1_price - 0.3
diff = abs(my_price - rank1_price)
diff = my_price - rank1_price
# 精简了原图中长串的阶梯逻辑
if diff <= 0.3:
@@ -47,7 +47,7 @@ class RepricingLogic:
情况2:已经是自己购物车
逻辑:判断与第二名的差价,如果拉开足够差距,则稍微降一点点(减0.5)保持优势,防止卖太便宜。
"""
diff = abs(rank2_price - my_price)
diff = rank2_price - my_price
# 1. 绝对差价2欧以上
if diff >= 2.0:
@@ -55,18 +55,18 @@ class RepricingLogic:
# 2. 产品售价区间判定 (是否要提高价格)
if (20 <= my_price < 30 and diff >= 4) or \
(30 <= my_price < 60 and diff >= 8) or \
(60 <= my_price <= 150 and diff >= 15):
(30 <= my_price < 60 and diff >= 8) or \
(60 <= my_price <= 150 and diff >= 15):
return rank2_price - 0.3 # 满足区间大差价,提价至第二名之下
return my_price # 不满足条件则保持原价
return my_price # 不满足条件则保持原价
@staticmethod
def situation_3_not_own_buybox(my_price, rank1_price):
"""
情况3:不是自己购物车 (常规情况)
"""
diff = abs(my_price - rank1_price)
diff = my_price - rank1_price
if diff <= 0.3:
return rank1_price - 0.5
@@ -74,36 +74,36 @@ class RepricingLogic:
return rank1_price - 0.7
elif 0.8 < diff <= 2.5:
return rank1_price - 1.0
elif diff > 2.5: # 2.5-3.5以上跳过
elif diff > 2.5: # 2.5-3.5以上跳过
return None
return rank1_price - 0.3 # 默认按第一名减0.3
return rank1_price - 0.3 # 默认按第一名减0.3
@staticmethod
def situation_4_encounter_amz_us_1st(my_price, amz_us_price):
"""
情况4:第一名是 Amazon US 卖家
"""
diff = abs(my_price - amz_us_price)
diff = my_price - amz_us_price
if diff <= 3:
return amz_us_price - 2
elif 4 <= diff <= 6:
return amz_us_price - 3
elif 8 <= diff <= 12:
return None # 跳过不跟价
return None # 跳过不跟价
return amz_us_price - 3 # 默认直接按照 Amazon US 减去3
return amz_us_price - 3 # 默认直接按照 Amazon US 减去3
@staticmethod
def situation_5_im_1st_amz_us_2nd(my_price, amz_us_price):
"""
情况5:自己是第一名,第二名是 Amazon US 卖家
"""
diff = abs(amz_us_price - my_price)
diff = amz_us_price - my_price
if diff <= 7:
return None # 相差5以内跳过 (保持原价)
return None # 相差5以内跳过 (保持原价)
else:
return amz_us_price - 5
@@ -127,7 +127,12 @@ def calculate_target_price(
# --- Step 2: 紫鸟后台特殊判定 (最高优先级) ---
if recommended_shipping > 0:
price_1 = float(front_end_data.get("top_sellers")[0].get("price")) # 实际第一名价格
return RepricingLogic.situation_1_general_decrease(my_price,price_1,is_my_buybox)
if is_my_buybox:
price_col = price_1
else:
price_col = recommended_price + recommended_shipping
print(f"【紫鸟后台价格】{is_my_buybox} 推荐价格: {recommended_price},推荐运费: {recommended_shipping},总和: {price_col}")
return RepricingLogic.situation_1_general_decrease(my_price,price_col,is_my_buybox)
# backend_base_price = recommended_price
# backend_shipping = recommended_shipping
# total_price = backend_base_price + backend_shipping
@@ -151,7 +156,7 @@ def calculate_target_price(
# 分支 A:第一名是 Amazon US 卖家 (特殊强敌优先)
price_1 = float(front_end_data.get("top_sellers")[0].get("price")) # 实际第一名价格
if "Amazon" in shop_name_1:
return RepricingLogic.isituation_4_encounter_amz_us_1st(my_price,price_1)
return RepricingLogic.situation_4_encounter_amz_us_1st(my_price,price_1)
# 分支 B:不是 Amazon US,且目前是自己的购物车
@@ -619,6 +624,11 @@ class AmzonePriceMatch(AmamzonBase):
page_pamel = self.tab.eles('xpath://kat-pagination', timeout=5)
if len(page_pamel) > 0:
current_page = page_pamel[0].sr('xpath:.//ul[@class="pages"]//li[@aria-current="true"]').text
#测试=====
# if int(current_page) > 3:
# break
# 测试===========
# yield (None,{"page":int(current_page.strip())})
# 总页数
total_page = page_pamel[0].sr.eles(
@@ -653,9 +663,9 @@ class AmzonePriceMatch(AmamzonBase):
'xpath:.//div[contains(@class,"JanusSplitBox-module__container")]//div[contains(@class,"JanusSplitBox-module__panel--") and contains(string(.),"ASIN")]/..//div[last()]',
timeout=3).text
print(f"{self.mark_name}】ASIN {asin} 找到....")
if asin in already_asin:
print(f"{self.mark_name}{asin} 已经处理过了,跳过")
continue
# if asin in already_asin:
# print(f"【{self.mark_name}】{asin} 已经处理过了,跳过")
# continue
if asin in skip_asin:
yield (asin, {
"statu": "跳过,需要跳过的ASIN"
@@ -779,7 +789,8 @@ class AmzonePriceMatch(AmamzonBase):
"secondPlace": price_2,
"cartShopName" : cartShopName,
"shippingFee" : f"{shipping_fee}",
"priceChangeStatus" : "改价成功",
"priceChangeStatus" : f"{adjust_prices}",
"modifyCount" : "1",
})
else:
recommendedPrice = recommend_price + shipping_fee
@@ -797,7 +808,8 @@ class AmzonePriceMatch(AmamzonBase):
"secondPlace": price_2,
"cartShopName": cartShopName,
"shippingFee": f"{shipping_fee}",
"priceChangeStatus": "跳过,无需改价",
"priceChangeStatus": "",
"modifyCount": "0",
})
already_asin.add(asin)
@@ -1322,30 +1334,35 @@ class PriceTask:
url = f"{DELETE_BRAND_API_BASE}/api/price-track/tasks/{task_id}/result"
country_aliases = {name: code for code, name in self.country_info.items()}
country_key = country_aliases.get(str(country_code).strip(), str(country_code).strip())
countries = {}
if asin or status:
countries[country_key] = [
{
"shopMallName": shopMallName,
"asin": asin,
"price": status.get("currentPrice") if status.get("currentPrice") else "",
"recommendedPrice": status.get("recommendedPrice") if status.get("recommendedPrice") else "",
"minimumPrice": status.get("minimumPrice") if status.get("minimumPrice") else "",
"firstPlace": status.get("firstPlace") if status.get("firstPlace") else "",
"secondPlace": status.get("secondPlace") if status.get("secondPlace") else "",
"cartShopName": status.get("cartShopName") if status.get("cartShopName") else "",
"priceChangeStatus": status.get("priceChangeStatus") if status.get("priceChangeStatus") else "",
"deleteSkipAsin": status.get("deleteSkipAsin", False),
"removeAsin": status.get("removeAsin") if status.get("removeAsin") else "",
"modifyCount": status.get("modifyCount") if status.get("modifyCount") else "",
"shippingFee": status.get("shippingFee") if status.get("shippingFee") else "",
"status": status.get("statu")
}
]
payload = {
"shops": [
{
"shopName": shop_name,
"countries": {
"additionalProperties1": [
{
"shopMallName": shopMallName,
"asin": asin,
"price": status.get("currentPrice") if status.get("currentPrice") else "",
"recommendedPrice": status.get("recommendedPrice") if status.get("recommendedPrice") else "",
"minimumPrice": status.get("minimumPrice") if status.get("minimumPrice") else "",
"firstPlace": status.get("firstPlace") if status.get("firstPlace") else "",
"secondPlace": status.get("secondPlace") if status.get("secondPlace") else "",
"cartShopName": status.get("cartShopName") if status.get("cartShopName") else "",
"priceChangeStatus": "UPDATED",
"deleteSkipAsin": status.get("deleteSkipAsin",False),
"removeAsin": status.get("removeAsin") if status.get("removeAsin") else "",
# "modifyCount": "2",
"shippingFee" : status.get("shippingFee") if status.get("shippingFee") else "",
"status": status.get("statu")
}
]
},
"countries": countries,
"error": ""
}
]
@@ -1391,11 +1408,11 @@ class PriceTask:
if __name__ == '__main__':
# 使用示例
# user_info = {
# "company": "尾号5578的公司115",
# "company": "rongchuang123",
# "username": "自动化_Robot",
# "password": "#20zsg25"
# }
# shop_name = "刘建煌"
# shop_name = "郭亚芳"
# country = "德国"
# kill_process('v6')
# driver = AmzonePriceMatch(user_info)
@@ -1403,8 +1420,8 @@ if __name__ == '__main__':
# sw_suc = driver.SwitchingCountries(country)
# driver.SwitchPage()
# risk_listing_filter = "Active"
# _shopMallName = "Jianhuang 888"
# ap_asin ="B0BGHRP6BS"
# _shopMallName = "yafang123"
# ap_asin ="B0BTHBJDKG"
# skip_asin = []
# for _ in range(3):
# try:
@@ -1422,12 +1439,14 @@ if __name__ == '__main__':
# print(f"ASIN {asin} 的处理结果: {status}")
# print("已完成操作")
front_end_data = {"top_sellers": [{"rank": 1, "price": "36.81", "stock": "5", "shop_name": "Windera"}, {"rank": 2, "price": "36.50", "stock": "100", "shop_name": "Jianhuang 888"}], "cart_seller": "Windera", "timestamp": "2026-04-25 09:32:58"}
current_Price = 36.50
current_shop_name = "Jianhuang 888"
recommended_price = 36.81
recommended_shipping = 0
calculate_target_price(
front_end_data = {'top_sellers': [{'rank': 1, 'price': '50.24', 'stock': '26', 'shop_name': 'yafang123'}, {'rank': 2, 'price': '44.35', 'stock': '100', 'shop_name': 'QinPimy'}], 'cart_seller': 'Amazon US', 'timestamp': '2026-04-27 16:17:42'}
current_Price = 44.35
current_shop_name = "yafang123"
recommended_price = 44.35
recommended_shipping =0.0
res = calculate_target_price(
front_end_data, current_Price, current_shop_name,
recommended_price, recommended_shipping
)
print(res)
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
+7 -14
View File
@@ -169,20 +169,13 @@ def serve_assets(filename):
@main_bp.route('/new_web_source/<path:filename>')
def serve_new_web_source(filename):
"""提供 new_web_source 目录下的静态页面文件访问。"""
candidate_dirs = [
os.path.abspath(os.path.join(BASE_DIR, 'new_web_source')),
os.path.abspath(os.path.join(BASE_DIR, '..', 'new_web_source')),
]
for static_abs in candidate_dirs:
filepath = os.path.normpath(os.path.join(static_abs, filename))
file_abs = os.path.abspath(filepath)
if not file_abs.startswith(static_abs):
continue
if os.path.isfile(file_abs):
return send_file(file_abs, as_attachment=False)
return '', 404
"""提供 static 目录及子目录下的静态文件访问。"""
filepath = os.path.normpath(os.path.join("new_web_source", filename))
static_abs = os.path.abspath("new_web_source")
file_abs = os.path.abspath(filepath)
if not file_abs.startswith(static_abs) or not os.path.isfile(file_abs):
return '', 404
return send_file(file_abs, as_attachment=False)
@main_bp.route('/logo.jpg', methods=['GET'])
def get_logo_image():
Binary file not shown.
+1 -1
View File
@@ -51,7 +51,7 @@ os.environ['OSS_ACCESS_KEY_SECRET'] = accessKeySecret
os.environ['SECRET_KEY'] = "ddffc7c1d02121d9554d7b080b2511b6"
debug = False
debug = os.getenv("debug", "false").strip().lower() in ("1", "true", "yes", "on")
version = "1.0.56"
APP_UPDATE_URL = f"{_base_url}/api/version/latest" # 检测更新接口地址,返回 { "file_url": "...", "version": "x.x.x" }
os.environ['APP_VERSION'] = version
+2 -2
View File
@@ -50,8 +50,8 @@ class WindowAPI:
def __init__(self, window):
self._window = window
self._is_maximized = False
window.events.maximized += lambda _: setattr(self, '_is_maximized', True)
window.events.restored += lambda _: setattr(self, '_is_maximized', False)
window.events.maximized += lambda *args: setattr(self, '_is_maximized', True)
window.events.restored += lambda *args: setattr(self, '_is_maximized', False)
def close(self):
"""关闭窗口,异步执行清理逻辑"""
Binary file not shown.
+12
View File
@@ -23,6 +23,8 @@
<easyexcel.version>4.0.3</easyexcel.version>
<aliyun.oss.version>3.17.4</aliyun.oss.version>
<hutool.version>5.8.36</hutool.version>
<rocketmq-spring.version>2.3.5</rocketmq-spring.version>
<minio.version>8.5.17</minio.version>
</properties>
<dependencies>
@@ -81,6 +83,16 @@
<artifactId>hutool-all</artifactId>
<version>${hutool.version}</version>
</dependency>
<dependency>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-spring-boot-starter</artifactId>
<version>${rocketmq-spring.version}</version>
</dependency>
<dependency>
<groupId>io.minio</groupId>
<artifactId>minio</artifactId>
<version>${minio.version}</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
@@ -13,6 +13,9 @@ public class GlobalExceptionHandler {
@ExceptionHandler(BusinessException.class)
public ApiResponse<Void> handleBusinessException(BusinessException ex) {
if (Integer.valueOf(40901).equals(ex.getCode())) {
return ApiResponse.success("任务已结束,忽略重复提交", null);
}
return ex.getCode() == null
? ApiResponse.fail(ex.getMessage())
: ApiResponse.fail(ex.getCode(), ex.getMessage());
@@ -36,7 +36,13 @@ public class DistributedJobLockService {
Duration actualTtl = (ttl == null || ttl.isNegative() || ttl.isZero()) ? Duration.ofMinutes(10) : ttl;
String key = buildLockKey(jobName);
String token = ownerPrefix + ":" + UUID.randomUUID();
Boolean locked = stringRedisTemplate.opsForValue().setIfAbsent(key, token, actualTtl);
Boolean locked;
try {
locked = stringRedisTemplate.opsForValue().setIfAbsent(key, token, actualTtl);
} catch (Exception ex) {
log.warn("[job-lock] acquire skipped jobName={} key={} msg={}", jobName, key, ex.getMessage());
return null;
}
if (!Boolean.TRUE.equals(locked)) {
return null;
}
@@ -0,0 +1,21 @@
package com.nanri.aiimage.config;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
@Data
@ConfigurationProperties(prefix = "aiimage.appearance-patent")
public class AppearancePatentProperties {
private String cozeBaseUrl = "https://api.coze.cn";
private String cozeWorkflowPath = "/v1/workflow/run";
private String cozeWorkflowHistoryPath = "/v1/workflows/{workflow_id}/run_histories/{execute_id}";
private String cozeWorkflowId = "7632683471312355338";
private String cozeToken = "";
private int cozeBatchSize = 10;
private int cozeConnectTimeoutMillis = 10000;
private int cozeReadTimeoutMillis = 60000;
private int cozePollIntervalMillis = 2000;
private int cozePollTimeoutMillis = 120000;
private int staleTimeoutMinutes = 20;
private String staleFinalizeCron = "0 */2 * * * *";
}
@@ -7,6 +7,7 @@ import org.springframework.boot.context.properties.ConfigurationProperties;
@ConfigurationProperties(prefix = "aiimage.delete-brand-progress")
public class DeleteBrandProgressProperties {
private long heartbeatTimeoutMinutes = 15;
private long deleteBrandInitialTimeoutMinutes = 20;
private String staleCheckCron = "*/30 * * * * *";
/**
@@ -4,6 +4,6 @@ import org.springframework.boot.context.properties.EnableConfigurationProperties
import org.springframework.context.annotation.Configuration;
@Configuration
@EnableConfigurationProperties({OssProperties.class, StorageProperties.class, BrandProgressProperties.class, DeleteBrandProgressProperties.class, ZiniaoProperties.class, ModuleCleanupProperties.class, TaskPressureProperties.class})
@EnableConfigurationProperties({OssProperties.class, TransientStorageProperties.class, StorageProperties.class, BrandProgressProperties.class, DeleteBrandProgressProperties.class, ZiniaoProperties.class, ModuleCleanupProperties.class, TaskPressureProperties.class, AppearancePatentProperties.class})
public class PropertiesConfig {
}
@@ -0,0 +1,27 @@
package com.nanri.aiimage.config;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.task.TaskExecutor;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
@Configuration
public class TaskFileJobConfig {
@Bean("taskFileJobDispatchExecutor")
public TaskExecutor taskFileJobDispatchExecutor(
@Value("${aiimage.result-file-job.local-dispatch-pool-size:2}") int poolSize,
@Value("${aiimage.result-file-job.local-dispatch-queue-capacity:200}") int queueCapacity) {
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
int normalizedPoolSize = Math.max(1, poolSize);
executor.setCorePoolSize(normalizedPoolSize);
executor.setMaxPoolSize(normalizedPoolSize);
executor.setQueueCapacity(Math.max(10, queueCapacity));
executor.setThreadNamePrefix("task-file-job-dispatch-");
executor.setWaitForTasksToCompleteOnShutdown(true);
executor.setAwaitTerminationSeconds(30);
executor.initialize();
return executor;
}
}
@@ -0,0 +1,15 @@
package com.nanri.aiimage.config;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
@Data
@ConfigurationProperties(prefix = "aiimage.transient-storage")
public class TransientStorageProperties {
private boolean enabled = false;
private String endpoint;
private String bucket;
private String accessKeyId;
private String accessKeySecret;
private String region = "us-east-1";
}
@@ -0,0 +1,746 @@
package com.nanri.aiimage.modules.appearancepatent.client;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.AppearancePatentProperties;
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultRowDto;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.http.MediaType;
import org.springframework.http.client.SimpleClientHttpRequestFactory;
import org.springframework.stereotype.Component;
import org.springframework.util.StreamUtils;
import org.springframework.web.client.RestClient;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@Component
@RequiredArgsConstructor
@Slf4j
public class AppearancePatentCozeClient {
private final AppearancePatentProperties properties;
private final ObjectMapper objectMapper;
public List<AppearancePatentResultRowDto> inspect(List<AppearancePatentResultRowDto> rows, String prompt) {
if (rows == null || rows.isEmpty()) {
return List.of();
}
if (properties.getCozeToken() == null || properties.getCozeToken().isBlank()) {
log.warn("[appearance-patent] coze token not configured, keep raw rows size={}", rows.size());
return rows.stream().map(this::copy).toList();
}
try {
return inspectWithFallback(rows, prompt);
} catch (Exception ex) {
String failureMessage = failureMessage(ex);
log.warn("[appearance-patent] coze batch failed size={} err={}", rows.size(), failureMessage);
return rows.stream().map(this::copy).map(row -> markFailed(row, failureMessage)).toList();
}
}
private List<AppearancePatentResultRowDto> inspectWithFallback(List<AppearancePatentResultRowDto> rows, String prompt) {
try {
if (rows.size() == 1) {
return inspectSingleRowWithRetry(rows, prompt);
}
InspectAttempt attempt = inspectOnce(rows, prompt);
if (attempt.resolvedCount() < rows.size()) {
throw new PartialCozeResultException(attempt.resolvedCount(), rows.size(), attempt.rawResultCount());
}
return attempt.mergedRows();
} catch (Exception ex) {
if (shouldSplitBatch(rows, ex)) {
int middle = rows.size() / 2;
log.warn("[appearance-patent] coze batch fallback split size={} left={} right={} err={}",
rows.size(), middle, rows.size() - middle, failureMessage(ex));
List<AppearancePatentResultRowDto> merged = new ArrayList<>(rows.size());
merged.addAll(inspectWithFallback(rows.subList(0, middle), prompt));
merged.addAll(inspectWithFallback(rows.subList(middle, rows.size()), prompt));
return merged;
}
throw propagate(ex);
}
}
private List<AppearancePatentResultRowDto> inspectSingleRowWithRetry(List<AppearancePatentResultRowDto> rows, String prompt) throws Exception {
AppearancePatentResultRowDto row = rows.getFirst();
PartialCozeResultException lastFailure = null;
for (int attemptIndex = 1; attemptIndex <= 3; attemptIndex++) {
try {
InspectAttempt attempt = inspectOnce(rows, prompt);
if (attempt.resolvedCount() == rows.size()) {
return attempt.mergedRows();
}
log.warn("[appearance-patent] coze single unresolved attempt={} rowId={} asin={} country={} title={} url={} raw={}",
attemptIndex,
row.getId(),
row.getAsin(),
row.getCountry(),
abbreviate(row.getTitle(), 120),
abbreviate(row.getUrl(), 120),
abbreviate(attempt.raw(), 500));
lastFailure = new PartialCozeResultException(attempt.resolvedCount(), rows.size(), attempt.rawResultCount());
} catch (Exception ex) {
if (attemptIndex >= 3 || !isRetryableBatchFailure(ex)) {
throw ex;
}
log.warn("[appearance-patent] coze single retryable failure attempt={} rowId={} asin={} country={} err={}",
attemptIndex,
row.getId(),
row.getAsin(),
row.getCountry(),
failureMessage(ex));
}
if (attemptIndex < 3) {
sleepBeforeRetry(attemptIndex);
}
}
throw lastFailure == null ? new PartialCozeResultException(0, rows.size(), 0) : lastFailure;
}
private InspectAttempt inspectOnce(List<AppearancePatentResultRowDto> rows, String prompt) throws Exception {
String raw = runWorkflowAsyncAndWait(rows, prompt);
List<CozeResult> results = parseResults(raw);
if (rows.size() > 1 && !results.isEmpty() && results.stream().noneMatch(this::hasIdentity)) {
throw new PartialCozeResultException(0, rows.size(), results.size());
}
List<AppearancePatentResultRowDto> merged = mergeRows(rows, results);
return new InspectAttempt(raw, merged, resolvedCount(merged), results.size());
}
private String runWorkflowAsyncAndWait(List<AppearancePatentResultRowDto> rows, String prompt) throws Exception {
JsonNode submitRoot = objectMapper.readTree(postWorkflow(rows, prompt));
ensureSuccess(submitRoot);
String immediateData = extractResultDataText(submitRoot);
if (!immediateData.isBlank()) {
return wrapDataPayload(immediateData);
}
String executeId = extractExecuteId(submitRoot);
if (executeId == null || executeId.isBlank()) {
throw new IllegalStateException("Coze async execute_id missing");
}
long deadline = System.currentTimeMillis() + Math.max(10000, properties.getCozePollTimeoutMillis());
while (System.currentTimeMillis() < deadline) {
JsonNode pollRoot = objectMapper.readTree(getWorkflowHistory(executeId));
ensureSuccess(pollRoot);
String dataText = extractResultDataText(pollRoot);
if (!dataText.isBlank()) {
return wrapDataPayload(dataText);
}
String status = normalize(resolveWorkflowStatus(pollRoot)).toUpperCase(Locale.ROOT);
if (status.contains("FAIL") || status.contains("ERROR") || status.contains("CANCEL")) {
throw new IllegalStateException(firstNonBlank(resolveFailureMessage(pollRoot), "Coze async workflow failed"));
}
sleepQuietly(Math.max(200, properties.getCozePollIntervalMillis()));
}
throw new IllegalStateException("Coze async workflow poll timeout");
}
private String postWorkflow(List<AppearancePatentResultRowDto> rows, String prompt) {
Map<String, Object> parameters = buildParameters(rows, prompt);
Map<String, Object> body = new LinkedHashMap<>();
body.put("workflow_id", properties.getCozeWorkflowId());
body.put("parameters", parameters);
body.put("is_async", Boolean.TRUE);
RestClient.RequestBodySpec request = restClient().post()
.uri(joinUrl(properties.getCozeBaseUrl(), properties.getCozeWorkflowPath()))
.headers(headers -> {
headers.setBearerAuth(stripBearer(properties.getCozeToken()));
headers.setContentType(MediaType.APPLICATION_JSON);
});
request.body(body);
return request.exchange((clientRequest, clientResponse) -> {
byte[] responseBytes = StreamUtils.copyToByteArray(clientResponse.getBody());
return responseBytes.length == 0 ? "" : new String(responseBytes, StandardCharsets.UTF_8);
});
}
private String getWorkflowHistory(String executeId) {
String path = properties.getCozeWorkflowHistoryPath()
.replace("{workflow_id}", properties.getCozeWorkflowId())
.replace("{execute_id}", executeId);
return restClient().get()
.uri(joinUrl(properties.getCozeBaseUrl(), path))
.headers(headers -> {
headers.setBearerAuth(stripBearer(properties.getCozeToken()));
headers.setContentType(MediaType.APPLICATION_JSON);
})
.exchange((clientRequest, clientResponse) -> {
byte[] responseBytes = StreamUtils.copyToByteArray(clientResponse.getBody());
return responseBytes.length == 0 ? "" : new String(responseBytes, StandardCharsets.UTF_8);
});
}
private Map<String, Object> buildParameters(List<AppearancePatentResultRowDto> rows, String prompt) {
List<String> groupKeys = rows.stream().map(row -> nonBlank(row.getGroupKey(), rowKey(row))).toList();
List<String> rowIds = rows.stream().map(row -> nonBlank(row.getId(), "")).toList();
List<String> asins = rows.stream().map(row -> nonBlank(row.getAsin(), "")).toList();
List<String> countries = rows.stream().map(row -> nonBlank(row.getCountry(), "")).toList();
List<String> titles = rows.stream().map(row -> nonBlank(row.getTitle(), row.getAsin())).toList();
List<String> urls = rows.stream().map(row -> nonBlank(row.getUrl(), "")).toList();
Map<String, Object> parameters = new LinkedHashMap<>();
parameters.put("title_list", titles);
parameters.put("url_list", urls);
parameters.put("items", buildItemObjects(rows, groupKeys, rowIds, asins, countries, titles, urls));
parameters.put("prompt", prompt == null ? "" : prompt);
return parameters;
}
private List<Map<String, Object>> buildItemObjects(List<AppearancePatentResultRowDto> rows,
List<String> groupKeys,
List<String> rowIds,
List<String> asins,
List<String> countries,
List<String> titles,
List<String> urls) {
List<Map<String, Object>> items = new ArrayList<>(rows.size());
for (int i = 0; i < rows.size(); i++) {
Map<String, Object> item = new LinkedHashMap<>();
item.put("group_key", groupKeys.get(i));
item.put("row_id", rowIds.get(i));
item.put("asin", asins.get(i));
item.put("country", countries.get(i));
item.put("title", titles.get(i));
item.put("url", urls.get(i));
items.add(item);
}
return items;
}
private List<CozeResult> parseResults(String raw) throws Exception {
JsonNode root = objectMapper.readTree(raw);
ensureSuccess(root);
String dataText = extractResultDataText(root);
if (dataText.isBlank()) {
return List.of();
}
JsonNode dataRoot = objectMapper.readTree(dataText);
JsonNode array = dataRoot.path("data");
List<CozeResult> results = new ArrayList<>();
if (array.isArray()) {
for (JsonNode node : array) {
JsonNode itemNode = resultItemNode(node);
results.add(new CozeResult(
text(firstNonNull(
firstNonNull(node.get("group_key"), node.get("groupKey")),
firstNonNull(itemNode.get("group_key"), itemNode.get("groupKey")))),
text(firstNonNull(
firstNonNull(node.get("row_id"), firstNonNull(node.get("rowId"), node.get("id"))),
firstNonNull(itemNode.get("row_id"), firstNonNull(itemNode.get("rowId"), itemNode.get("id"))))),
text(firstNonNull(node.get("asin"), itemNode.get("asin"))),
text(firstNonNull(
firstNonNull(node.get("country"), node.get("site")),
firstNonNull(itemNode.get("country"), itemNode.get("site")))),
text(node.get("title")),
text(node.get("appearance")),
text(firstNonNull(node.get("patent"), node.get("patent "))),
text(node.get("result"))
));
}
}
return results;
}
private JsonNode resultItemNode(JsonNode node) {
if (node == null || node.isMissingNode() || node.isNull()) {
return objectMapper.missingNode();
}
JsonNode item = firstNonNull(node.get("item"), node.get("items"));
if (item == null || item.isMissingNode() || item.isNull()) {
return objectMapper.missingNode();
}
if (item.isArray()) {
return item.isEmpty() ? objectMapper.missingNode() : item.get(0);
}
return item;
}
private List<AppearancePatentResultRowDto> mergeRows(List<AppearancePatentResultRowDto> rows, List<CozeResult> results) {
Map<String, CozeResult> resultByGroupKey = new LinkedHashMap<>();
Map<String, CozeResult> resultByCompositeKey = new LinkedHashMap<>();
Map<String, CozeResult> resultByAsinCountry = new LinkedHashMap<>();
Map<String, CozeResult> resultByAsin = new LinkedHashMap<>();
Map<String, CozeResult> resultByRowId = new LinkedHashMap<>();
for (CozeResult result : results) {
String groupKey = normalize(result.groupKey());
if (!groupKey.isBlank()) {
resultByGroupKey.putIfAbsent(groupKey, result);
}
String compositeKey = rowKey(result.rowId(), result.asin(), result.country());
if (!compositeKey.isBlank()) {
resultByCompositeKey.putIfAbsent(compositeKey, result);
}
String asinCountryKey = asinCountryKey(result.asin(), result.country());
if (!asinCountryKey.isBlank()) {
resultByAsinCountry.putIfAbsent(asinCountryKey, result);
}
String asinKey = normalize(result.asin()).toUpperCase(Locale.ROOT);
if (!asinKey.isBlank()) {
resultByAsin.putIfAbsent(asinKey, result);
}
String rowIdKey = normalize(result.rowId());
if (!rowIdKey.isBlank()) {
resultByRowId.putIfAbsent(rowIdKey, result);
}
}
List<AppearancePatentResultRowDto> merged = new ArrayList<>(rows.size());
boolean allowIndexFallback = results.size() == rows.size() && results.stream().noneMatch(this::hasIdentity);
for (int i = 0; i < rows.size(); i++) {
AppearancePatentResultRowDto row = copy(rows.get(i));
CozeResult result = resultByGroupKey.get(normalize(row.getGroupKey()));
if (result == null) {
result = resultByCompositeKey.get(rowKey(row.getId(), row.getAsin(), row.getCountry()));
}
if (result == null) {
result = resultByAsinCountry.get(asinCountryKey(row.getAsin(), row.getCountry()));
}
if (result == null) {
result = resultByAsin.get(normalize(row.getAsin()).toUpperCase(Locale.ROOT));
}
if (result == null) {
result = resultByRowId.get(normalize(row.getId()));
}
if (result == null && allowIndexFallback && i < results.size()) {
result = results.get(i);
}
applyResult(row, result);
merged.add(row);
}
return merged;
}
private String asinCountryKey(String asin, String country) {
String normalizedAsin = normalize(asin).toUpperCase(Locale.ROOT);
if (normalizedAsin.isBlank()) {
return "";
}
return normalizedAsin + "::" + normalize(country);
}
private boolean hasIdentity(CozeResult result) {
if (result == null) {
return false;
}
return !normalize(result.groupKey()).isBlank()
|| !normalize(result.rowId()).isBlank()
|| !normalize(result.asin()).isBlank();
}
private void applyResult(AppearancePatentResultRowDto row, CozeResult result) {
if (row == null || result == null) {
return;
}
row.setTitleRisk(result.title());
row.setAppearanceRisk(result.appearance());
row.setPatentRisk(result.patent());
row.setConclusion(result.result());
}
private RestClient restClient() {
SimpleClientHttpRequestFactory requestFactory = new SimpleClientHttpRequestFactory();
requestFactory.setConnectTimeout(properties.getCozeConnectTimeoutMillis());
requestFactory.setReadTimeout(properties.getCozeReadTimeoutMillis());
return RestClient.builder().requestFactory(requestFactory).build();
}
private AppearancePatentResultRowDto copy(AppearancePatentResultRowDto source) {
AppearancePatentResultRowDto row = new AppearancePatentResultRowDto();
row.setSourceFileKey(source.getSourceFileKey());
row.setSourceFilename(source.getSourceFilename());
row.setRowToken(source.getRowToken());
row.setGroupKey(source.getGroupKey());
row.setId(source.getId());
row.setAsin(source.getAsin());
row.setCountry(source.getCountry());
row.setUrl(source.getUrl());
row.setTitle(source.getTitle());
row.setError(source.getError());
row.setDone(source.getDone());
row.setTitleRisk(source.getTitleRisk());
row.setAppearanceRisk(source.getAppearanceRisk());
row.setPatentRisk(source.getPatentRisk());
row.setConclusion(source.getConclusion());
return row;
}
private AppearancePatentResultRowDto markFailed(AppearancePatentResultRowDto row, String failureMessage) {
if (row.getError() == null || row.getError().isBlank()) {
row.setError(failureMessage);
}
if (row.getConclusion() == null || row.getConclusion().isBlank()) {
row.setConclusion(failureMessage);
}
return row;
}
private boolean shouldSplitBatch(List<AppearancePatentResultRowDto> rows, Exception ex) {
return rows != null && rows.size() > 1 && isRetryableBatchFailure(ex);
}
private boolean isRetryableBatchFailure(Exception ex) {
if (ex instanceof PartialCozeResultException) {
return true;
}
String message = ex == null ? "" : nonBlank(ex.getMessage(), "");
return message.contains("Workflow node execution limit exceeded")
|| message.contains("Read timed out")
|| message.contains("Connection reset")
|| message.contains("I/O error on POST request")
|| message.toLowerCase(Locale.ROOT).contains("timeout");
}
private int resolvedCount(List<AppearancePatentResultRowDto> rows) {
int resolved = 0;
for (AppearancePatentResultRowDto row : rows) {
if (hasResolvedCozeFields(row)) {
resolved++;
}
}
return resolved;
}
private boolean hasResolvedCozeFields(AppearancePatentResultRowDto row) {
if (row == null) {
return false;
}
return !normalize(row.getTitleRisk()).isBlank()
|| !normalize(row.getAppearanceRisk()).isBlank()
|| !normalize(row.getPatentRisk()).isBlank()
|| !normalize(row.getConclusion()).isBlank();
}
private void ensureSuccess(JsonNode root) {
if (root.path("code").asInt(-1) != 0) {
throw new IllegalStateException(root.path("msg").asText("Coze response code is not 0"));
}
}
private String extractResultDataText(JsonNode root) {
if (root == null || root.isMissingNode() || root.isNull()) {
return "";
}
JsonNode dataNode = root.path("data");
if (dataNode.isTextual()) {
String value = dataNode.asText("");
if (looksLikeResultDataPayload(value)) {
return value;
}
return discoverEmbeddedData(parseJsonOrMissing(value));
}
if (dataNode.isObject()) {
String nested = text(firstNonNull(dataNode.get("data"), firstNonNull(dataNode.get("output"), dataNode.get("result"))));
if (nested != null && !nested.isBlank() && looksLikeResultDataPayload(nested)) {
return nested;
}
JsonNode outputs = firstNonNull(dataNode.get("outputs"), dataNode.get("details"));
String discovered = discoverEmbeddedData(outputs);
if (!discovered.isBlank()) {
return discovered;
}
}
return discoverEmbeddedData(root);
}
private String discoverEmbeddedData(JsonNode node) {
if (node == null || node.isNull() || node.isMissingNode()) {
return "";
}
if (node.isTextual()) {
String value = node.asText("");
if (looksLikeResultDataPayload(value)) {
return value;
}
return discoverEmbeddedData(parseJsonOrMissing(value));
}
if (node.isArray()) {
for (JsonNode child : node) {
String discovered = discoverEmbeddedData(child);
if (!discovered.isBlank()) {
return discovered;
}
}
return "";
}
if (node.isObject()) {
if (isResultDataPayload(node)) {
return node.toString();
}
for (java.util.Iterator<Map.Entry<String, JsonNode>> it = node.fields(); it.hasNext(); ) {
Map.Entry<String, JsonNode> entry = it.next();
String discovered = discoverEmbeddedData(entry.getValue());
if (!discovered.isBlank()) {
return discovered;
}
}
}
return "";
}
private boolean looksLikeResultDataPayload(String value) {
JsonNode parsed = parseJsonOrMissing(value);
return isResultDataPayload(parsed);
}
private boolean isResultDataPayload(JsonNode node) {
if (node == null || node.isMissingNode() || node.isNull()) {
return false;
}
JsonNode array = node.path("data");
if (!array.isArray()) {
return false;
}
for (JsonNode item : array) {
if (item.has("appearance") || item.has("patent") || item.has("patent ") || item.has("result")) {
return true;
}
}
return false;
}
private JsonNode parseJsonOrMissing(String value) {
String normalized = normalize(value);
if (!(normalized.startsWith("{") || normalized.startsWith("["))) {
return objectMapper.missingNode();
}
try {
return objectMapper.readTree(normalized);
} catch (Exception ignored) {
return objectMapper.missingNode();
}
}
private String resolveWorkflowStatus(JsonNode root) {
JsonNode dataNode = root.path("data");
JsonNode statusNode = firstNonNull(
firstNonNull(dataNode.get("status"), dataNode.get("execute_status")),
firstNonNull(root.get("status"), root.get("execute_status")));
String status = text(statusNode);
if (status != null && !status.isBlank()) {
return status;
}
return findTextByFieldName(root, "execute_status", "status");
}
private String resolveFailureMessage(JsonNode root) {
JsonNode dataNode = root.path("data");
String message = text(firstNonNull(dataNode.get("error_message"), firstNonNull(dataNode.get("msg"), root.get("msg"))));
return message == null ? "" : message;
}
private String extractExecuteId(JsonNode root) {
return findTextByFieldName(root, "execute_id", "executeId");
}
private String findTextByFieldName(JsonNode node, String... names) {
if (node == null || node.isMissingNode() || node.isNull()) {
return "";
}
if (node.isTextual()) {
return findTextByFieldName(parseJsonOrMissing(node.asText("")), names);
}
if (node.isArray()) {
for (JsonNode child : node) {
String found = findTextByFieldName(child, names);
if (!found.isBlank()) {
return found;
}
}
return "";
}
if (node.isObject()) {
for (String name : names) {
String value = text(node.get(name));
if (value != null && !value.isBlank()) {
return value;
}
}
JsonNode dataNode = node.get("data");
if (dataNode != null && dataNode.isTextual()) {
String found = findTextByFieldName(parseJsonOrMissing(dataNode.asText("")), names);
if (!found.isBlank()) {
return found;
}
}
for (java.util.Iterator<Map.Entry<String, JsonNode>> it = node.fields(); it.hasNext(); ) {
Map.Entry<String, JsonNode> entry = it.next();
String found = findTextByFieldName(entry.getValue(), names);
if (!found.isBlank()) {
return found;
}
}
}
return "";
}
private String wrapDataPayload(String dataText) {
Map<String, Object> payload = new LinkedHashMap<>();
payload.put("code", 0);
payload.put("data", dataText);
return writeJson(payload);
}
private String writeJson(Object value) {
try {
return objectMapper.writeValueAsString(value);
} catch (Exception ex) {
throw new IllegalStateException("Failed to serialize Coze payload", ex);
}
}
private String abbreviate(String value, int maxLength) {
String normalized = value == null ? "" : value.trim();
if (normalized.length() <= maxLength) {
return normalized;
}
return normalized.substring(0, Math.max(0, maxLength - 3)) + "...";
}
private void sleepBeforeRetry(int attemptIndex) {
long delayMillis = Math.max(1, attemptIndex) * 1500L;
sleepQuietly(delayMillis);
}
private void sleepQuietly(long delayMillis) {
try {
Thread.sleep(delayMillis);
} catch (InterruptedException interruptedException) {
Thread.currentThread().interrupt();
}
}
private RuntimeException propagate(Exception ex) {
if (ex instanceof RuntimeException runtimeException) {
return runtimeException;
}
return new IllegalStateException(nonBlank(ex.getMessage(), "Coze call failed"), ex);
}
private JsonNode firstNonNull(JsonNode left, JsonNode right) {
return left == null || left.isNull() ? right : left;
}
private String text(JsonNode node) {
return node == null || node.isNull() ? null : node.asText();
}
private String nonBlank(String value, String fallback) {
return value == null || value.isBlank() ? fallback : value;
}
private String firstNonBlank(String preferred, String fallback) {
return preferred == null || preferred.isBlank() ? fallback : preferred.trim();
}
private String normalize(String value) {
return value == null ? "" : value.replace("\ufeff", "").replace("\u3000", " ").trim();
}
private String rowKey(AppearancePatentResultRowDto row) {
if (row == null) {
return "";
}
return rowKey(row.getId(), row.getAsin(), row.getCountry());
}
private String rowKey(String rowId, String asin, String country) {
return normalize(rowId) + "::" + normalize(asin).toUpperCase(Locale.ROOT) + "::" + normalize(country);
}
private String failureMessage(Exception ex) {
if (ex instanceof PartialCozeResultException partial) {
return "Coze result incomplete(" + partial.resolvedCount() + "/" + partial.expectedCount() + ")";
}
String message = ex == null ? null : ex.getMessage();
if (message == null || message.isBlank()) {
return "Coze call failed";
}
if (message.contains("Workflow node execution limit exceeded")) {
return "Coze workflow node execution limit exceeded";
}
if (message.contains("Read timed out")) {
return "Coze call timed out";
}
return "Coze call failed: " + message;
}
private String stripBearer(String token) {
String normalized = token == null ? "" : token.trim();
return normalized.regionMatches(true, 0, "Bearer ", 0, 7) ? normalized.substring(7).trim() : normalized;
}
private String joinUrl(String baseUrl, String path) {
String base = baseUrl == null ? "" : baseUrl.trim();
String suffix = path == null ? "" : path.trim();
if (base.endsWith("/") && suffix.startsWith("/")) {
return base + suffix.substring(1);
}
if (!base.endsWith("/") && !suffix.startsWith("/")) {
return base + "/" + suffix;
}
return base + suffix;
}
private record CozeResult(
String groupKey,
String rowId,
String asin,
String country,
String title,
String appearance,
String patent,
String result
) {
}
private record InspectAttempt(
String raw,
List<AppearancePatentResultRowDto> mergedRows,
int resolvedCount,
int rawResultCount
) {
}
private static final class PartialCozeResultException extends RuntimeException {
private final int resolvedCount;
private final int expectedCount;
private final int rawResultCount;
private PartialCozeResultException(int resolvedCount, int expectedCount, int rawResultCount) {
super("partial-result resolved=" + resolvedCount + "/" + expectedCount + " raw=" + rawResultCount);
this.resolvedCount = resolvedCount;
this.expectedCount = expectedCount;
this.rawResultCount = rawResultCount;
}
private int resolvedCount() {
return resolvedCount;
}
private int expectedCount() {
return expectedCount;
}
@SuppressWarnings("unused")
private int rawResultCount() {
return rawResultCount;
}
}
}
@@ -0,0 +1,143 @@
package com.nanri.aiimage.modules.appearancepatent.controller;
import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentParseRequest;
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentSubmitResultRequest;
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentTaskBatchRequest;
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentDashboardVo;
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentHistoryVo;
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParseVo;
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentTaskBatchVo;
import com.nanri.aiimage.modules.appearancepatent.service.AppearancePatentTaskService;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.server.ResponseStatusException;
import java.io.InputStream;
import java.net.URI;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
@RestController
@RequiredArgsConstructor
@RequestMapping("/api/appearance-patent")
@Tag(name = "外观专利检测", description = "外观专利检测任务接口。前端上传 Excel 后由 Java 解析并创建任务;Python 回传商品数据;Java 负责攒批调用 Coze、补齐子行、生成最终 xlsx 并上传 OSS。")
public class AppearancePatentController {
private final AppearancePatentTaskService service;
@PostMapping("/parse")
@Operation(summary = "解析 Excel 并创建任务", description = "解析上传后的 Excel 文件,提取 id、ASIN、国家、URL、标题等字段。返回给前端的数据只包含整数 id 和 n_1 行;n_2、n_3 等子行会保存在 OSS 解析载荷中,用于最终结果补齐。创建后的任务状态为 PENDING,不会自动推送 Python。")
public ApiResponse<AppearancePatentParseVo> parse(@Valid @RequestBody AppearancePatentParseRequest request) {
return ApiResponse.success(service.parseAndCreateTask(request));
}
@GetMapping("/dashboard")
@Operation(summary = "查询外观专利检测总览", description = "查询当前用户的运行中、成功、失败和已结束任务数量。页面进入时请求一次即可,不需要持续轮询。")
public ApiResponse<AppearancePatentDashboardVo> dashboard(
@Parameter(description = "当前用户 ID,用于隔离不同用户的任务和历史记录。", required = true, example = "1")
@RequestParam("user_id") Long userId) {
return ApiResponse.success(service.dashboard(userId));
}
@GetMapping("/history")
@Operation(summary = "查询外观专利检测历史", description = "查询当前用户最近的外观专利检测历史记录,包含源文件名、任务状态、行数、错误信息和最终 xlsx 下载地址。")
public ApiResponse<AppearancePatentHistoryVo> history(
@Parameter(description = "当前用户 ID,用于查询该用户自己的历史记录。", required = true, example = "1")
@RequestParam("user_id") Long userId) {
return ApiResponse.success(service.history(userId));
}
@PostMapping("/tasks/progress/batch")
@Operation(summary = "批量查询任务进度", description = "前端只对活跃任务调用该接口,建议 6 秒一次。接口只返回轻量任务状态,不返回明细结果。")
public ApiResponse<AppearancePatentTaskBatchVo> progress(@Valid @RequestBody AppearancePatentTaskBatchRequest request) {
return ApiResponse.success(service.progressBatch(request.getTaskIds()));
}
@PostMapping("/tasks/{taskId}/activate")
@Operation(summary = "激活任务", description = "前端手动推送 Python 队列成功后调用,将任务从 PENDING 改为 RUNNING,并记录后端内部活跃时间。后续活跃时间由 Python 回传结果接口自动刷新,不需要单独心跳接口。")
public ApiResponse<Void> activate(
@Parameter(description = "外观专利检测任务 ID,即解析接口返回的 taskId。", required = true, example = "3938")
@PathVariable Long taskId,
@Parameter(description = "当前用户 ID,必须与创建任务的用户一致。", required = true, example = "1")
@RequestParam("user_id") Long userId) {
service.activateTask(taskId, userId);
return ApiResponse.success(null);
}
@PostMapping("/tasks/{taskId}/result")
@Operation(summary = "提交 Python 回传结果", description = "Python 回传商品数据接口。items 可以是一条或多条;Java 先原样保存回传数据,再内部攒够 10 条调用 Coze。done=true 表示 Python 已完成全部回传,Java 会强制处理剩余不足 10 条的数据并生成最终 xlsx。")
public ApiResponse<Void> result(
@Parameter(description = "外观专利检测任务 ID。任务必须处于 RUNNING 状态。", required = true, example = "3938")
@PathVariable Long taskId,
@Valid @RequestBody AppearancePatentSubmitResultRequest request) {
service.submitResult(taskId, request);
return ApiResponse.success(null);
}
@DeleteMapping("/tasks/{taskId}")
@Operation(summary = "删除任务", description = "删除当前用户的一条外观专利检测任务,同时清理任务结果、scope 状态和分片记录。")
public ApiResponse<Void> deleteTask(
@Parameter(description = "外观专利检测任务 ID。", required = true, example = "3938")
@PathVariable Long taskId,
@Parameter(description = "当前用户 ID,必须与创建任务的用户一致。", required = true, example = "1")
@RequestParam("user_id") Long userId) {
service.deleteTask(taskId, userId);
return ApiResponse.success(null);
}
@DeleteMapping("/history/{resultId}")
@Operation(summary = "删除历史记录", description = "删除当前用户的一条外观专利检测历史记录。只删除 biz_file_result 记录,不主动删除任务主记录。")
public ApiResponse<Void> deleteHistory(
@Parameter(description = "历史结果记录 ID,即 history 接口返回的 resultId。", required = true, example = "1001")
@PathVariable Long resultId,
@Parameter(description = "当前用户 ID。", required = true, example = "1")
@RequestParam("user_id") Long userId) {
service.deleteHistory(resultId, userId);
return ApiResponse.success(null);
}
@GetMapping("/results/{resultId}/download")
@Operation(summary = "下载外观专利检测结果文件")
public void downloadResult(
@Parameter(description = "结果记录 ID", required = true, example = "1001")
@PathVariable Long resultId,
@Parameter(description = "当前用户 ID", required = true, example = "1")
@RequestParam("user_id") Long userId,
jakarta.servlet.http.HttpServletResponse response) {
String url = service.resolveResultDownloadUrl(resultId, userId);
String filename = service.resolveResultDownloadFilename(resultId, userId);
if (url == null || url.isBlank()) {
throw new ResponseStatusException(HttpStatus.NOT_FOUND, "暂无可下载结果");
}
try {
String encodedFilename = URLEncoder.encode(filename, StandardCharsets.UTF_8).replace("+", "%20");
response.setContentType("application/octet-stream");
response.setHeader(HttpHeaders.CONTENT_DISPOSITION,
"attachment; filename=\"" + filename + "\"; filename*=UTF-8''" + encodedFilename);
try (InputStream in = URI.create(url).toURL().openStream()) {
byte[] buffer = new byte[65536];
int read;
while ((read = in.read(buffer)) != -1) {
response.getOutputStream().write(buffer, 0, read);
}
response.getOutputStream().flush();
}
} catch (Exception ex) {
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "下载失败");
}
}
}
@@ -0,0 +1,28 @@
package com.nanri.aiimage.modules.appearancepatent.model.dto;
import com.fasterxml.jackson.annotation.JsonAlias;
import com.fasterxml.jackson.annotation.JsonProperty;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotEmpty;
import jakarta.validation.constraints.NotNull;
import lombok.Data;
import java.util.List;
@Data
@Schema(description = "外观专利检测解析请求")
public class AppearancePatentParseRequest {
@JsonProperty("user_id")
@NotNull
@Schema(description = "当前用户 ID。后端会把创建的任务、历史记录和结果文件归属到该用户。", example = "1", requiredMode = Schema.RequiredMode.REQUIRED)
private Long userId;
@NotEmpty
@Schema(description = "已上传的 Excel 文件列表。支持多文件聚合解析;文件对象来自统一上传接口返回值。", requiredMode = Schema.RequiredMode.REQUIRED)
private List<AppearancePatentSourceFileDto> files;
@JsonProperty("ai_prompt")
@JsonAlias({"aiPrompt", "prompt"})
@Schema(description = "AI 提示词。非必填;为空时前端会使用默认提示词。后端会保存该提示词,并在调用 Coze workflow 时作为 prompt 参数传入。", example = "请排查这些亚马逊商品在英国及欧洲地区是否存在知识产权侵权风险。")
private String aiPrompt;
}
@@ -0,0 +1,31 @@
package com.nanri.aiimage.modules.appearancepatent.model.dto;
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedGroupVo;
import com.nanri.aiimage.modules.appearancepatent.model.vo.AppearancePatentParsedRowVo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "外观专利解析载荷")
public class AppearancePatentParsedPayloadDto {
@Schema(description = "AI 提示词")
private String aiPrompt;
@Schema(description = "本次解析的源文件列表")
private List<AppearancePatentSourceFileDto> sourceFiles = new ArrayList<>();
@Schema(description = "Excel 原始表头列表")
private List<String> headers = new ArrayList<>();
@Schema(description = "兼容旧链路的平铺有效行,现为全部有效行")
private List<AppearancePatentParsedRowVo> items = new ArrayList<>();
@Schema(description = "按相邻主 ID 块分组后的完整数据")
private List<AppearancePatentParsedGroupVo> groups = new ArrayList<>();
@Schema(description = "完整有效行")
private List<AppearancePatentParsedRowVo> allItems = new ArrayList<>();
}
@@ -0,0 +1,29 @@
package com.nanri.aiimage.modules.appearancepatent.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "Python 回传的外观专利分组结果")
public class AppearancePatentResultGroupDto {
@Schema(description = "来源文件 key")
private String sourceFileKey;
@Schema(description = "来源文件名")
private String sourceFilename;
@Schema(description = "相邻主 ID 块的分组 key")
private String groupKey;
@Schema(description = "主 ID,例如 1、2、10")
private String baseId;
@Schema(description = "分组首条展示 ID")
private String displayId;
@Schema(description = "分组内全部抓取结果")
private List<AppearancePatentResultRowDto> items = new ArrayList<>();
}
@@ -0,0 +1,55 @@
package com.nanri.aiimage.modules.appearancepatent.model.dto;
import com.fasterxml.jackson.annotation.JsonAlias;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@Schema(description = "外观专利检测单行商品数据")
public class AppearancePatentResultRowDto {
@Schema(description = "来源文件 key。多文件回传时用于避免行结果串到别的文件。", example = "uploads/20260426/appearance_patent_17.xlsx")
private String sourceFileKey;
@Schema(description = "来源文件名。仅用于调试与追踪。", example = "17.xlsx")
private String sourceFilename;
@Schema(description = "行级唯一 token。Python 应从解析结果原样透传。", example = "uploads/20260426/appearance_patent_17.xlsx::row::2")
private String rowToken;
@Schema(description = "主数据分组 key。Python 应从解析结果原样透传,用于把同组子行补回。", example = "uploads/20260426/appearance_patent_17.xlsx::2@2")
private String groupKey;
@Schema(description = "Excel 中的 id。代表行通常是整数 id 或 n_1,例如 2_1;最终生成 xlsx 时,2_2、2_3 会复用同组 2_1 的 Coze 检测结果。", example = "2_1")
private String id;
@Schema(description = "亚马逊 ASIN。后端会统一按大写处理和匹配。", example = "B0CJ8SNXXV")
private String asin;
@Schema(description = "站点或国家。来自 Excel 的国家列,例如英国、德国、法国。", example = "英国")
private String country;
@Schema(description = "商品主图或待检测图片 URL。Java 调用 Coze 时会放入 url_list。", example = "https://webstatic.aiproxy.vip/output/20260425/103322/demo.jpg")
private String url;
@Schema(description = "商品标题。Java 调用 Coze 时会放入 title_list;为空时会回退使用 ASIN。", example = "Women Floral Dress Summer Casual")
private String title;
@Schema(description = "单行错误信息。通常用于记录 Python 单行处理异常;Coze 失败时后端会尽量保留原始行,不强行写入风险结果。", example = "图片地址为空")
private String error;
@Schema(description = "单行完成标记。当前主要使用请求体顶层 done 控制任务收尾,该字段仅作兼容。", example = "true")
private Boolean done;
@Schema(description = "Java 调用 Coze 后生成的标题维度检测结果,对应最终 xlsx 的“标题维度(商标)”列。Python 回传请求中不要传该字段;即使传入,后端也会以 Java/Coze 处理结果为准。", example = "标题未发现明显商标侵权风险。", accessMode = Schema.AccessMode.READ_ONLY)
private String titleRisk;
@Schema(description = "Java 调用 Coze 后生成的外观维度检测结果,对应最终 xlsx 的“外观维度(外观设计专利)”列。Python 回传请求中不要传该字段。", example = "未发现明显外观设计专利侵权风险。", accessMode = Schema.AccessMode.READ_ONLY)
private String appearanceRisk;
@JsonAlias({"patent ", "patent"})
@Schema(description = "Java 调用 Coze 后生成的专利维度检测结果,对应最终 xlsx 的“专利维度(发明/实用新型专利)”列。兼容 Coze 返回字段 patent 和 patent 后带空格的情况;Python 回传请求中不要传该字段。", example = "未发现明显发明或实用新型专利侵权风险。", accessMode = Schema.AccessMode.READ_ONLY)
private String patentRisk;
@Schema(description = "Java 调用 Coze 后生成的最终结论,对应最终 xlsx 的“结论”列。Python 回传请求中不要传该字段。", example = "未发现明显侵权风险", accessMode = Schema.AccessMode.READ_ONLY)
private String conclusion;
}
@@ -0,0 +1,15 @@
package com.nanri.aiimage.modules.appearancepatent.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@Schema(description = "外观专利检测源文件信息")
public class AppearancePatentSourceFileDto {
@Schema(description = "上传接口返回的临时文件 key。后端根据该 key 查找本地临时 Excel 文件并解析。", example = "uploads/20260426/appearance_patent_17.xlsx", requiredMode = Schema.RequiredMode.REQUIRED)
private String fileKey;
@Schema(description = "原始文件名。用于历史记录展示和最终结果文件命名。", example = "17.xlsx")
private String originalFilename;
@Schema(description = "相对目录路径。当前仅记录来源,外观专利检测不依赖该字段处理。", example = "xlsx/17.xlsx")
private String relativePath;
}
@@ -0,0 +1,32 @@
package com.nanri.aiimage.modules.appearancepatent.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "Python 回传外观专利检测结果请求")
public class AppearancePatentSubmitResultRequest {
@Schema(description = "本次 Python 回传的提交批次标识", example = "appearance-patent-3938")
private String submissionId;
@Schema(description = "当前回传分片序号", example = "1")
private Integer chunkIndex;
@Schema(description = "本任务预计总分片数", example = "36")
private Integer chunkTotal;
@Schema(description = "是否为最后一次回传", example = "false")
private Boolean done;
@Schema(description = "Python 侧任务级错误信息", example = "浏览器执行异常,任务提前结束")
private String error;
@Schema(description = "本次回传的分组结果列表,推荐优先使用")
private List<AppearancePatentResultGroupDto> groups = new ArrayList<>();
@Schema(description = "兼容旧链路的平铺结果列表")
private List<AppearancePatentResultRowDto> items = new ArrayList<>();
}
@@ -0,0 +1,15 @@
package com.nanri.aiimage.modules.appearancepatent.model.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotEmpty;
import lombok.Data;
import java.util.List;
@Data
@Schema(description = "外观专利检测批量进度查询请求")
public class AppearancePatentTaskBatchRequest {
@NotEmpty
@Schema(description = "需要查询进度的任务 ID 列表。前端只传正在轮询的活跃任务;后端会批量查询,避免每个任务单独请求。", example = "[3938,3939]", requiredMode = Schema.RequiredMode.REQUIRED)
private List<Long> taskIds;
}
@@ -0,0 +1,17 @@
package com.nanri.aiimage.modules.appearancepatent.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@Schema(description = "外观专利检测总览统计")
public class AppearancePatentDashboardVo {
@Schema(description = "运行中任务数量。这里统计 RUNNING 状态任务。", example = "1")
private Long pendingTaskCount;
@Schema(description = "已结束任务数量,等于成功任务数加失败任务数。", example = "12")
private Long processedTaskCount;
@Schema(description = "成功任务数量。", example = "10")
private Long successTaskCount;
@Schema(description = "失败任务数量。", example = "2")
private Long failedTaskCount;
}
@@ -0,0 +1,33 @@
package com.nanri.aiimage.modules.appearancepatent.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@Schema(description = "外观专利检测历史记录项")
public class AppearancePatentHistoryItemVo {
@Schema(description = "结果记录 ID。删除历史、下载结果时使用。", example = "1001")
private Long resultId;
@Schema(description = "任务 ID。", example = "3938")
private Long taskId;
@Schema(description = "源 Excel 文件名。", example = "17.xlsx")
private String sourceFilename;
@Schema(description = "最终结果文件名。任务完成并生成 xlsx 后返回。", example = "17-result.xlsx")
private String resultFilename;
@Schema(description = "最终结果文件下载地址。后端基于 OSS objectKey 生成的新鲜预签名 URL。", example = "https://bucket.oss-cn-hangzhou.aliyuncs.com/result/appearance_patent/xxx/17-result.xlsx?Expires=...")
private String downloadUrl;
private Long fileJobId;
private String fileStatus;
private String fileError;
private Boolean fileReady;
@Schema(description = "任务状态:PENDING=已解析待推送,RUNNING=执行中,SUCCESS=成功,FAILED=失败。", example = "SUCCESS")
private String taskStatus;
@Schema(description = "结果是否成功。true 表示任务完成并生成结果文件;false 表示失败或未完成。", example = "true")
private Boolean success;
@Schema(description = "错误信息。任务失败时返回,例如 Python 超时、结果文件生成失败等。", example = "Python interrupted before uploading final appearance patent result")
private String error;
@Schema(description = "最终结果行数。包含解析阶段被过滤但最终需要补回的 2_2、2_3 等子行。", example = "716")
private Integer rowCount;
@Schema(description = "历史记录创建时间,ISO 本地时间字符串。", example = "2026-04-26T10:30:00")
private String createdAt;
}
@@ -0,0 +1,14 @@
package com.nanri.aiimage.modules.appearancepatent.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "外观专利检测历史记录列表")
public class AppearancePatentHistoryVo {
@Schema(description = "历史记录项列表,默认返回最近 100 条。")
private List<AppearancePatentHistoryItemVo> items = new ArrayList<>();
}
@@ -0,0 +1,41 @@
package com.nanri.aiimage.modules.appearancepatent.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "外观专利检测解析结果")
public class AppearancePatentParseVo {
@Schema(description = "新创建的任务 ID", example = "3938")
private Long taskId;
@Schema(description = "来源 Excel 文件名,多个文件时为聚合展示文案", example = "17.xlsx 等 2 个文件")
private String sourceFilename;
@Schema(description = "参与本次解析的源文件数量", example = "2")
private Integer sourceFileCount;
@Schema(description = "Excel 中检测到的非空数据总行数", example = "716")
private Integer totalRows;
@Schema(description = "解析出的有效行数", example = "458")
private Integer acceptedRows;
@Schema(description = "因缺少必要字段而被丢弃的行数", example = "12")
private Integer droppedRows;
@Schema(description = "按相邻主 ID 块生成的分组数", example = "36")
private Integer groupCount;
@Schema(description = "本任务最终使用的 AI 提示词")
private String aiPrompt;
@Schema(description = "兼容旧前端的平铺有效行列表,现为全部有效行")
private List<AppearancePatentParsedRowVo> items = new ArrayList<>();
@Schema(description = "返回前端和推送 Python 的分组列表")
private List<AppearancePatentParsedGroupVo> groups = new ArrayList<>();
}
@@ -0,0 +1,32 @@
package com.nanri.aiimage.modules.appearancepatent.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "外观专利解析后的分组数据")
public class AppearancePatentParsedGroupVo {
@Schema(description = "来源文件 key")
private String sourceFileKey;
@Schema(description = "来源文件名")
private String sourceFilename;
@Schema(description = "相邻主 ID 块的分组 key")
private String groupKey;
@Schema(description = "主 ID,例如 1、2、10")
private String baseId;
@Schema(description = "分组首条展示 ID,例如 1 或 2_1")
private String displayId;
@Schema(description = "分组内行数")
private Integer itemCount;
@Schema(description = "分组内全部行,顺序与原 Excel 保持一致")
private List<AppearancePatentParsedRowVo> items = new ArrayList<>();
}
@@ -0,0 +1,47 @@
package com.nanri.aiimage.modules.appearancepatent.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.LinkedHashMap;
import java.util.Map;
@Data
@Schema(description = "外观专利检测解析出的单行数据")
public class AppearancePatentParsedRowVo {
@Schema(description = "来源文件 key。用于多文件场景下区分同名/同 ID 行。", example = "uploads/20260426/appearance_patent_17.xlsx")
private String sourceFileKey;
@Schema(description = "来源文件名。用于调试和结果追踪。", example = "17.xlsx")
private String sourceFilename;
@Schema(description = "Excel 原始行号,从 1 开始。", example = "2")
private Integer rowIndex;
@Schema(description = "Excel 中原始 id 值。", example = "2_1")
private String sourceId;
@Schema(description = "前端展示和 Python 回传使用的 id。整数 id 原样保留,子数据第一条如 2_1 原样保留。", example = "2_1")
private String displayId;
@Schema(description = "行级唯一 token。多文件、重复主数据时用于精确匹配回传结果。", example = "uploads/20260426/appearance_patent_17.xlsx::row::2")
private String rowToken;
@Schema(description = "同一主数据块的分组 key。用于把 2_1、2_2、2_3 等子行重新补回同一组结果。", example = "uploads/20260426/appearance_patent_17.xlsx::2@2")
private String groupKey;
@Schema(description = "亚马逊 ASIN。", example = "B0CJ8SNXXV")
private String asin;
@Schema(description = "国家或站点。", example = "英国")
private String country;
@Schema(description = "商品图片 URL 或商品 URL,供 Coze 检测使用。", example = "https://webstatic.aiproxy.vip/output/demo.jpg")
private String url;
@Schema(description = "商品标题。", example = "Women Floral Dress Summer Casual")
private String title;
@Schema(description = "该 Excel 行的原始列值映射。最终生成 xlsx 时可以从这里读取价格等字段。", example = "{\"id\":\"2_1\",\"asin\":\"B0CJ8SNXXV\",\"国家\":\"英国\",\"价格\":\"12.99\"}")
private Map<String, String> values = new LinkedHashMap<>();
}
@@ -0,0 +1,16 @@
package com.nanri.aiimage.modules.appearancepatent.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "外观专利检测批量进度响应")
public class AppearancePatentTaskBatchVo {
@Schema(description = "查询到的任务详情列表。顺序按请求 taskIds 处理。")
private List<AppearancePatentTaskDetailVo> items = new ArrayList<>();
@Schema(description = "未找到或不属于外观专利检测模块的任务 ID 列表。", example = "[99999]")
private List<Long> missingTaskIds = new ArrayList<>();
}
@@ -0,0 +1,16 @@
package com.nanri.aiimage.modules.appearancepatent.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
@Data
@Schema(description = "外观专利检测任务进度详情")
public class AppearancePatentTaskDetailVo {
@Schema(description = "任务主记录轻量信息。")
private AppearancePatentTaskItemVo task;
@Schema(description = "预留的任务明细列表。当前进度接口主要返回任务轻量状态,不返回完整结果明细。")
private List<AppearancePatentHistoryItemVo> items = new ArrayList<>();
}
@@ -0,0 +1,23 @@
package com.nanri.aiimage.modules.appearancepatent.model.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Data
@Schema(description = "外观专利检测任务轻量信息")
public class AppearancePatentTaskItemVo {
@Schema(description = "任务 ID。", example = "3938")
private Long id;
@Schema(description = "任务编号,后端自动生成。", example = "APPEARANCE_PATENT-1780000000000000000")
private String taskNo;
@Schema(description = "任务状态:PENDING=已解析待推送,RUNNING=执行中,SUCCESS=成功,FAILED=失败。", example = "RUNNING")
private String status;
@Schema(description = "任务级错误信息。失败时返回。", example = "生成外观专利检测结果失败")
private String errorMessage;
@Schema(description = "创建时间,ISO 本地时间字符串。", example = "2026-04-26T10:00:00")
private String createdAt;
@Schema(description = "最后更新时间,通常由 Python 回传结果或任务收尾更新。", example = "2026-04-26T10:05:00")
private String updatedAt;
@Schema(description = "完成时间。任务未结束时为空。", example = "2026-04-26T10:10:00")
private String finishedAt;
}
@@ -0,0 +1,144 @@
package com.nanri.aiimage.modules.appearancepatent.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.modules.appearancepatent.model.dto.AppearancePatentResultRowDto;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
@Service
@RequiredArgsConstructor
@Slf4j
public class AppearancePatentTaskCacheService {
private static final long TTL_HOURS = 24;
private final StringRedisTemplate stringRedisTemplate;
private final ObjectMapper objectMapper;
public void appendPendingRow(Long taskId, String scopeHash, Integer chunkIndex, AppearancePatentResultRowDto row) {
if (taskId == null || taskId <= 0 || scopeHash == null || scopeHash.isBlank() || chunkIndex == null || row == null) {
return;
}
try {
stringRedisTemplate.opsForList().rightPush(
pendingRowsKey(taskId),
objectMapper.writeValueAsString(new PendingRow(scopeHash, chunkIndex, row))
);
stringRedisTemplate.expire(pendingRowsKey(taskId), Duration.ofHours(TTL_HOURS));
touchTaskHeartbeat(taskId);
} catch (Exception ignored) {
}
}
public long pendingRowCount(Long taskId) {
Long size;
try {
size = stringRedisTemplate.opsForList().size(pendingRowsKey(taskId));
} catch (Exception ex) {
log.warn("[appearance-patent-cache] pending row count degraded taskId={} msg={}", taskId, ex.getMessage());
return 0L;
}
return size == null ? 0L : size;
}
public List<PendingRow> drainPendingRows(Long taskId, int limit) {
if (taskId == null || taskId <= 0 || limit <= 0) {
return List.of();
}
String key = pendingRowsKey(taskId);
List<String> values;
try {
values = stringRedisTemplate.opsForList().range(key, 0, limit - 1L);
} catch (Exception ex) {
log.warn("[appearance-patent-cache] drain range degraded taskId={} msg={}", taskId, ex.getMessage());
return List.of();
}
if (values == null || values.isEmpty()) {
return List.of();
}
try {
stringRedisTemplate.opsForList().trim(key, values.size(), -1);
} catch (Exception ex) {
log.warn("[appearance-patent-cache] drain trim degraded taskId={} msg={}", taskId, ex.getMessage());
return List.of();
}
List<PendingRow> rows = new ArrayList<>();
for (String value : values) {
if (value == null || value.isBlank()) {
continue;
}
try {
rows.add(objectMapper.readValue(value, PendingRow.class));
} catch (Exception ignored) {
}
}
touchTaskHeartbeat(taskId);
return rows;
}
public void touchTaskHeartbeat(Long taskId) {
if (taskId == null || taskId <= 0) {
return;
}
try {
stringRedisTemplate.opsForValue().set(
heartbeatKey(taskId),
String.valueOf(Instant.now().toEpochMilli()),
Duration.ofHours(TTL_HOURS)
);
} catch (Exception ex) {
log.warn("[appearance-patent-cache] touch heartbeat degraded taskId={} msg={}", taskId, ex.getMessage());
}
}
public long getTaskHeartbeatMillis(Long taskId) {
if (taskId == null || taskId <= 0) {
return 0L;
}
String raw;
try {
raw = stringRedisTemplate.opsForValue().get(heartbeatKey(taskId));
} catch (Exception ex) {
log.warn("[appearance-patent-cache] get heartbeat degraded taskId={} msg={}", taskId, ex.getMessage());
return 0L;
}
if (raw == null || raw.isBlank()) {
return 0L;
}
try {
return Long.parseLong(raw);
} catch (NumberFormatException ignored) {
return 0L;
}
}
public void deleteTaskCache(Long taskId) {
if (taskId == null || taskId <= 0) {
return;
}
try {
stringRedisTemplate.delete(pendingRowsKey(taskId));
stringRedisTemplate.delete(heartbeatKey(taskId));
} catch (Exception ex) {
log.warn("[appearance-patent-cache] delete degraded taskId={} msg={}", taskId, ex.getMessage());
}
}
private String pendingRowsKey(Long taskId) {
return "appearance-patent:task:pending-rows:" + taskId;
}
private String heartbeatKey(Long taskId) {
return "appearance-patent:task:heartbeat:" + taskId;
}
public record PendingRow(String scopeHash, Integer chunkIndex, AppearancePatentResultRowDto row) {
}
}
@@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.brand.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.BrandProgressProperties;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
@@ -11,6 +12,7 @@ import java.util.LinkedHashMap;
import java.util.Map;
@Service
@Slf4j
public class BrandTaskProgressCacheService {
public static final String PHASE_CRAWLING = "crawling";
@@ -53,8 +55,12 @@ public class BrandTaskProgressCacheService {
values.put("finished_files", String.valueOf(Math.max(finishedFiles, 0)));
values.put("updated_at", now);
values.put("last_heartbeat_at", now);
stringRedisTemplate.opsForHash().putAll(key, values);
stringRedisTemplate.expire(key, ttl());
try {
stringRedisTemplate.opsForHash().putAll(key, values);
stringRedisTemplate.expire(key, ttl());
} catch (Exception ex) {
log.warn("[brand-progress-cache] save progress degraded taskId={} msg={}", taskId, ex.getMessage());
}
}
public void updatePhase(Long taskId, String phase, int finishedFiles, int fileTotal) {
@@ -66,8 +72,12 @@ public class BrandTaskProgressCacheService {
values.put("file_total", String.valueOf(Math.max(fileTotal, 0)));
values.put("updated_at", now);
values.put("last_heartbeat_at", now);
stringRedisTemplate.opsForHash().putAll(key, values);
stringRedisTemplate.expire(key, ttl());
try {
stringRedisTemplate.opsForHash().putAll(key, values);
stringRedisTemplate.expire(key, ttl());
} catch (Exception ex) {
log.warn("[brand-progress-cache] update phase degraded taskId={} msg={}", taskId, ex.getMessage());
}
}
public void markFailed(Long taskId, String message) {
@@ -77,27 +87,49 @@ public class BrandTaskProgressCacheService {
values.put("phase", PHASE_FAILED);
values.put("updated_at", now);
values.put("error_message", blankToEmpty(message));
stringRedisTemplate.opsForHash().putAll(key, values);
stringRedisTemplate.expire(key, Duration.ofHours(brandProgressProperties.getFailedTtlHours()));
try {
stringRedisTemplate.opsForHash().putAll(key, values);
stringRedisTemplate.expire(key, Duration.ofHours(brandProgressProperties.getFailedTtlHours()));
} catch (Exception ex) {
log.warn("[brand-progress-cache] mark failed degraded taskId={} msg={}", taskId, ex.getMessage());
}
}
public Map<Object, Object> getProgress(Long taskId) {
return stringRedisTemplate.opsForHash().entries(buildKey(taskId));
try {
return stringRedisTemplate.opsForHash().entries(buildKey(taskId));
} catch (Exception ex) {
log.warn("[brand-progress-cache] get progress degraded taskId={} msg={}", taskId, ex.getMessage());
return Map.of();
}
}
public boolean acquireFinalizeLock(Long taskId) {
Boolean ok = stringRedisTemplate.opsForValue()
.setIfAbsent(buildFinalizeLockKey(taskId), String.valueOf(Instant.now().toEpochMilli()), FINALIZE_LOCK_TTL);
return Boolean.TRUE.equals(ok);
try {
Boolean ok = stringRedisTemplate.opsForValue()
.setIfAbsent(buildFinalizeLockKey(taskId), String.valueOf(Instant.now().toEpochMilli()), FINALIZE_LOCK_TTL);
return Boolean.TRUE.equals(ok);
} catch (Exception ex) {
log.warn("[brand-progress-cache] acquire finalize lock degraded taskId={} msg={}", taskId, ex.getMessage());
return false;
}
}
public void releaseFinalizeLock(Long taskId) {
stringRedisTemplate.delete(buildFinalizeLockKey(taskId));
try {
stringRedisTemplate.delete(buildFinalizeLockKey(taskId));
} catch (Exception ex) {
log.warn("[brand-progress-cache] release finalize lock degraded taskId={} msg={}", taskId, ex.getMessage());
}
}
public void delete(Long taskId) {
stringRedisTemplate.delete(buildKey(taskId));
stringRedisTemplate.delete(buildFinalizeLockKey(taskId));
try {
stringRedisTemplate.delete(buildKey(taskId));
stringRedisTemplate.delete(buildFinalizeLockKey(taskId));
} catch (Exception ex) {
log.warn("[brand-progress-cache] delete degraded taskId={} msg={}", taskId, ex.getMessage());
}
}
public String buildKey(Long taskId) {
@@ -31,6 +31,9 @@ import com.nanri.aiimage.modules.brand.model.vo.LegacyBrandTaskDetailVo;
import com.nanri.aiimage.modules.brand.model.vo.LegacyBrandTaskItemVo;
import com.nanri.aiimage.modules.brand.model.vo.LegacyBrandTaskListVo;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.poi.ss.usermodel.Cell;
@@ -77,6 +80,7 @@ public class BrandTaskService {
private static final String STATUS_SUCCESS = "success";
private static final String STATUS_FAILED = "failed";
private static final String STATUS_CANCELLED = "cancelled";
private static final String MODULE_TYPE = "BRAND";
private final BrandCrawlTaskMapper brandCrawlTaskMapper;
private final OssStorageService ossStorageService;
@@ -86,6 +90,8 @@ public class BrandTaskService {
private final BrandTaskStorageService brandTaskStorageService;
private final DistributedJobLockService distributedJobLockService;
private final ObjectMapper objectMapper;
private final TaskFileJobService taskFileJobService;
private final TaskProgressSnapshotService taskProgressSnapshotService;
public BrandTaskCreateVo createTask(Long userId, BrandTaskCreateRequest request) {
if (userId == null || userId <= 0) {
@@ -244,6 +250,7 @@ public class BrandTaskService {
int totalCount = sourceFiles.size();
markTaskRunning(taskId, totalCount);
afterTaskStarted(taskId, totalCount);
log.info("[brand-submit] taskId={} chunks={} files={} thread={}",
taskId,
@@ -295,6 +302,7 @@ public class BrandTaskService {
currentTotalLines,
finishedCount);
updateTaskProgress(taskId, finishedCount, totalCount);
saveBrandProgressSnapshot(taskId, STATUS_RUNNING, totalCount, finishedCount, 0, null);
}
tryFinalizeTask(taskId, request.getStrategy(), sourceFiles, cachedByUrl, finishedCount, totalCount);
@@ -357,6 +365,23 @@ public class BrandTaskService {
return ossStorageService.generateFreshDownloadUrl(stored);
}
public String resolveResultObjectKey(Long taskId) {
BrandCrawlTaskEntity task = requireTask(taskId);
Object raw = parseJsonValue(task.getResultPaths());
if (!(raw instanceof Map<?, ?> map)) {
return null;
}
Object zipObjectKey = map.get("zip_object_key");
if (zipObjectKey instanceof String key && !key.isBlank()) {
return key;
}
Object zipUrl = map.get("zip_url");
if (zipUrl instanceof String url && !url.isBlank()) {
return url;
}
return null;
}
private List<BrandCrawlTaskEntity> listTaskEntities(Long userId) {
if (userId == null || userId <= 0) {
throw new BusinessException("userId 不合法");
@@ -496,7 +521,7 @@ public class BrandTaskService {
private void markTaskRunning(Long taskId, int totalCount) {
int started = brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper<BrandCrawlTaskEntity>()
.eq(BrandCrawlTaskEntity::getId, taskId)
.ne(BrandCrawlTaskEntity::getStatus, STATUS_CANCELLED)
.in(BrandCrawlTaskEntity::getStatus, STATUS_PENDING, STATUS_RUNNING)
.set(BrandCrawlTaskEntity::getStatus, STATUS_RUNNING)
.set(BrandCrawlTaskEntity::getProgressTotal, totalCount)
.set(BrandCrawlTaskEntity::getUpdatedAt, LocalDateTime.now())
@@ -505,6 +530,13 @@ public class BrandTaskService {
throw new BusinessException("任务已取消");
}
}
private void afterTaskStarted(Long taskId, int totalCount) {
saveBrandProgressSnapshot(taskId, STATUS_RUNNING, totalCount, 0, 0, null);
}
private void saveBrandProgressSnapshot(Long taskId, String status, int totalCount, int successCount, int failedCount, String message) {
taskProgressSnapshotService.save(taskId, MODULE_TYPE, status, totalCount, successCount, failedCount, null, message, null);
}
private BrandCrawlTaskEntity requireActiveTask(Long taskId) {
BrandCrawlTaskEntity task = requireTask(taskId);
@@ -533,12 +565,33 @@ public class BrandTaskService {
return;
}
try {
finalizeTask(taskId, strategy, sourceFiles, cachedByUrl, totalCount);
enqueueFinalizeTask(taskId, totalCount);
} finally {
brandTaskProgressCacheService.releaseFinalizeLock(taskId);
}
}
private void enqueueFinalizeTask(Long taskId, int totalCount) {
brandTaskProgressCacheService.updatePhase(taskId, BrandTaskProgressCacheService.PHASE_ASSEMBLING, totalCount, totalCount);
taskProgressSnapshotService.save(taskId, MODULE_TYPE, STATUS_RUNNING, totalCount, totalCount, 0,
null, "result file assembly queued", null);
taskFileJobService.enqueueAssembleResult(taskId, MODULE_TYPE, taskId, "task:" + taskId);
}
public void processResultFileJob(TaskFileJobEntity job) {
if (job == null || job.getTaskId() == null) {
throw new BusinessException("结果文件任务参数不完整");
}
BrandCrawlTaskEntity task = requireTask(job.getTaskId());
List<BrandSourceFileDto> sourceFiles = parseSourceFiles(task.getFilePaths());
if (sourceFiles.isEmpty()) {
throw new BusinessException("任务没有源文件");
}
List<BrandParsedFileCacheDto> cachedFiles = brandTaskStorageService.getParsedPayload(task.getId());
Map<String, BrandParsedFileCacheDto> cachedByUrl = indexCachedFiles(cachedFiles);
finalizeTask(task.getId(), normalizeStrategy(task.getStrategy()), sourceFiles, cachedByUrl, sourceFiles.size());
}
private void finalizeTask(Long taskId,
String strategy,
List<BrandSourceFileDto> sourceFiles,
@@ -601,6 +654,7 @@ public class BrandTaskService {
if (updated == 0) {
throw new BusinessException("任务已取消");
}
saveBrandProgressSnapshot(taskId, STATUS_SUCCESS, totalCount, totalCount, 0, null);
brandTaskStorageService.deleteTaskData(taskId);
brandTaskProgressCacheService.delete(taskId);
log.info("[brand-finalize] taskId={} finalized success files={} elapsedMs={}",
@@ -617,6 +671,7 @@ public class BrandTaskService {
.set(BrandCrawlTaskEntity::getProgressTotal, totalCount)
.set(BrandCrawlTaskEntity::getErrorMessage, ex.getMessage()));
brandTaskProgressCacheService.markFailed(taskId, ex.getMessage());
saveBrandProgressSnapshot(taskId, STATUS_FAILED, totalCount, brandTaskStorageService.countCompletedFiles(taskId), 1, ex.getMessage());
if (ex instanceof BusinessException businessException) {
throw businessException;
}
@@ -643,10 +698,23 @@ public class BrandTaskService {
}
}
private Map<String, BrandParsedFileCacheDto> indexCachedFiles(List<BrandParsedFileCacheDto> cachedFiles) {
Map<String, BrandParsedFileCacheDto> cachedByUrl = new LinkedHashMap<>();
if (cachedFiles == null) {
return cachedByUrl;
}
for (BrandParsedFileCacheDto cachedFile : cachedFiles) {
if (cachedFile != null && cachedFile.getFileUrl() != null) {
cachedByUrl.put(cachedFile.getFileUrl(), cachedFile);
}
}
return cachedByUrl;
}
private void updateTaskProgress(Long taskId, int finishedCount, int totalCount) {
brandCrawlTaskMapper.update(null, new LambdaUpdateWrapper<BrandCrawlTaskEntity>()
.eq(BrandCrawlTaskEntity::getId, taskId)
.ne(BrandCrawlTaskEntity::getStatus, STATUS_CANCELLED)
.in(BrandCrawlTaskEntity::getStatus, STATUS_PENDING, STATUS_RUNNING)
.set(BrandCrawlTaskEntity::getStatus, STATUS_RUNNING)
.set(BrandCrawlTaskEntity::getProgressCurrent, finishedCount)
.set(BrandCrawlTaskEntity::getProgressTotal, totalCount)
@@ -951,13 +1019,14 @@ public class BrandTaskService {
List<String> fullUrls = new ArrayList<>();
for (OutputEntry entry : entries) {
// 存储完整公开地址下载时会自动通过 resolveObjectKey 提取并重新签名 OssStorageService.resolveObjectKey
String objectKey = ossStorageService.uploadResultFile(entry.resultFile(), "BRAND");
String objectKey = ossStorageService.uploadResultFile(entry.resultFile(), MODULE_TYPE);
fullUrls.add(ossStorageService.getPublicUrl(objectKey));
}
Map<String, Object> result = new LinkedHashMap<>();
result.put("urls", fullUrls);
File zipFile = packageAsZip(taskId, entries);
String zipObjectKey = ossStorageService.uploadResultFile(zipFile, "BRAND");
String zipObjectKey = ossStorageService.uploadResultFile(zipFile, MODULE_TYPE);
result.put("zip_object_key", zipObjectKey);
result.put("zip_url", ossStorageService.getPublicUrl(zipObjectKey));
return result;
}
@@ -7,11 +7,11 @@ import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.brand.model.dto.BrandCrawlResultFileDto;
import com.nanri.aiimage.modules.brand.model.dto.BrandFileAggregateCacheDto;
import com.nanri.aiimage.modules.brand.model.dto.BrandParsedFileCacheDto;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import lombok.RequiredArgsConstructor;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.stereotype.Service;
@@ -30,12 +30,11 @@ import java.util.Map;
public class BrandTaskStorageService {
private static final String MODULE_TYPE = "BRAND";
private static final String OSS_POINTER_PREFIX = "oss:";
private final TaskChunkMapper taskChunkMapper;
private final TaskScopeStateMapper taskScopeStateMapper;
private final ObjectMapper objectMapper;
private final OssStorageService ossStorageService;
private final TransientPayloadStorageService transientPayloadStorageService;
@Transactional
public void saveParsedPayload(Long taskId, List<BrandParsedFileCacheDto> payload) {
@@ -73,7 +72,7 @@ public class BrandTaskStorageService {
if (state == null) {
throw new BusinessException("Save brand task parsed payload failed");
}
deleteParsedPayloadIfNeeded(state.getParsedPayloadJson(), parsedJson);
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(state.getParsedPayloadJson(), parsedJson);
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getId, state.getId())
.set(TaskScopeStateEntity::getScopeKey, scopeKey)
@@ -132,6 +131,7 @@ public class BrandTaskStorageService {
throw new BusinessException("Brand task chunk conflict: " + scopeKey + "#" + file.getChunkIndex());
}
String storedPayload = transientPayloadStorageService.storeChunkPayload(MODULE_TYPE, taskId, scopeHash, file.getChunkIndex(), payloadJson);
TaskChunkEntity entity = new TaskChunkEntity();
entity.setTaskId(taskId);
entity.setModuleType(MODULE_TYPE);
@@ -139,7 +139,7 @@ public class BrandTaskStorageService {
entity.setScopeHash(scopeHash);
entity.setChunkIndex(file.getChunkIndex());
entity.setChunkTotal(file.getChunkTotal());
entity.setPayloadJson(payloadJson);
entity.setPayloadJson(storedPayload);
entity.setPayloadHash(payloadHash);
entity.setCreatedAt(now);
entity.setUpdatedAt(now);
@@ -153,9 +153,11 @@ public class BrandTaskStorageService {
.eq(TaskChunkEntity::getChunkIndex, file.getChunkIndex())
.last("limit 1"));
if (concurrentChunk != null && payloadHash.equals(concurrentChunk.getPayloadHash())) {
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
BrandFileAggregateCacheDto aggregate = rebuildAggregate(taskId, scopeKey, scopeHash);
return new ChunkStoreResult(false, false, countCompletedFiles(taskId), aggregate);
}
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
throw new BusinessException("Brand task chunk conflict: " + scopeKey + "#" + file.getChunkIndex());
}
@@ -171,7 +173,8 @@ public class BrandTaskStorageService {
return null;
}
try {
return objectMapper.readValue(state.getStateJson(), BrandFileAggregateCacheDto.class);
String aggregateJson = transientPayloadStorageService.resolvePayload(state.getStateJson(), "read brand task aggregate failed");
return objectMapper.readValue(aggregateJson, BrandFileAggregateCacheDto.class);
} catch (Exception ex) {
throw new BusinessException("Read brand task aggregate failed");
}
@@ -195,7 +198,8 @@ public class BrandTaskStorageService {
continue;
}
try {
result.put(state.getScopeKey(), objectMapper.readValue(state.getStateJson(), BrandFileAggregateCacheDto.class));
String aggregateJson = transientPayloadStorageService.resolvePayload(state.getStateJson(), "read brand task aggregate failed");
result.put(state.getScopeKey(), objectMapper.readValue(aggregateJson, BrandFileAggregateCacheDto.class));
} catch (Exception ex) {
throw new BusinessException("Read brand task aggregate failed");
}
@@ -220,12 +224,22 @@ public class BrandTaskStorageService {
return;
}
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
.select(TaskScopeStateEntity::getParsedPayloadJson)
.select(TaskScopeStateEntity::getParsedPayloadJson, TaskScopeStateEntity::getStateJson)
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
if (states != null) {
for (TaskScopeStateEntity state : states) {
deleteParsedPayloadIfNeeded(state.getParsedPayloadJson(), null);
transientPayloadStorageService.deletePayloadIfPresent(state.getParsedPayloadJson());
transientPayloadStorageService.deletePayloadIfPresent(state.getStateJson());
}
}
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
.select(TaskChunkEntity::getPayloadJson)
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
if (chunks != null) {
for (TaskChunkEntity chunk : chunks) {
transientPayloadStorageService.deletePayloadIfPresent(chunk.getPayloadJson());
}
}
taskChunkMapper.delete(new LambdaQueryWrapper<TaskChunkEntity>()
@@ -246,12 +260,14 @@ public class BrandTaskStorageService {
if (state == null) {
throw new BusinessException("Brand task aggregate state missing: " + scopeKey);
}
String storedAggregate = transientPayloadStorageService.storeScopePayload(MODULE_TYPE, taskId, scopeHash, aggregateJson, true);
int chunkTotal = aggregate.getChunkTotal() == null ? 0 : aggregate.getChunkTotal();
int receivedCount = aggregate.getReceivedChunkCount() == null ? 0 : aggregate.getReceivedChunkCount();
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(state.getStateJson(), storedAggregate);
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getId, state.getId())
.set(TaskScopeStateEntity::getScopeKey, scopeKey)
.set(TaskScopeStateEntity::getStateJson, aggregateJson)
.set(TaskScopeStateEntity::getStateJson, storedAggregate)
.set(TaskScopeStateEntity::getChunkTotal, chunkTotal)
.set(TaskScopeStateEntity::getReceivedChunkCount, receivedCount)
.set(TaskScopeStateEntity::getCompleted, Boolean.TRUE.equals(aggregate.getCompleted()) ? 1 : 0)
@@ -362,7 +378,8 @@ public class BrandTaskStorageService {
private BrandCrawlResultFileDto readChunkPayload(String payloadJson) {
try {
return objectMapper.readValue(payloadJson, BrandCrawlResultFileDto.class);
String resolvedPayload = transientPayloadStorageService.resolvePayload(payloadJson, "read brand task chunk failed");
return objectMapper.readValue(resolvedPayload, BrandCrawlResultFileDto.class);
} catch (Exception ex) {
throw new BusinessException("Read brand task chunk failed");
}
@@ -382,56 +399,17 @@ public class BrandTaskStorageService {
}
private String storeParsedPayload(Long taskId, String scopeHash, String payloadJson) {
try {
String pointer = OSS_POINTER_PREFIX + ossStorageService.uploadTaskParsedPayload(MODULE_TYPE, taskId, scopeHash, payloadJson);
return objectMapper.writeValueAsString(pointer);
} catch (Exception ex) {
throw new BusinessException("Save brand task parsed payload failed");
}
return transientPayloadStorageService.storeParsedPayload(MODULE_TYPE, taskId, scopeHash, payloadJson, true);
}
private String resolveParsedPayload(String value) {
if (isBlank(value)) {
return value;
}
String ossPointer = extractOssPointer(value);
if (ossPointer == null) {
return value;
}
try {
return ossStorageService.readObjectAsString(ossPointer.substring(OSS_POINTER_PREFIX.length()));
return transientPayloadStorageService.resolvePayload(value, "Read brand task parsed payload failed");
} catch (Exception ex) {
throw new BusinessException("Read brand task parsed payload failed");
}
}
private void deleteParsedPayloadIfNeeded(String oldValue, String newValue) {
String oldPointer = extractOssPointer(oldValue);
String newPointer = extractOssPointer(newValue);
if (oldPointer == null || oldPointer.equals(newPointer)) {
return;
}
try {
ossStorageService.deleteObject(oldPointer.substring(OSS_POINTER_PREFIX.length()));
} catch (Exception ignored) {
}
}
private String extractOssPointer(String value) {
if (isBlank(value)) {
return null;
}
if (value.startsWith(OSS_POINTER_PREFIX)) {
return value;
}
try {
String decoded = objectMapper.readValue(value, String.class);
return decoded != null && decoded.startsWith(OSS_POINTER_PREFIX) ? decoded : null;
} catch (Exception ignored) {
return null;
}
}
private String writeJson(Object value, String message) {
try {
return objectMapper.writeValueAsString(value);
@@ -205,9 +205,7 @@ public class ConvertRunService {
vo.setResultId(entity.getId());
vo.setSourceFilename(entity.getSourceFilename());
vo.setOutputFilename(entity.getResultFilename());
vo.setDownloadUrl(entity.getSuccess() != null && entity.getSuccess() == 1
? ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl())
: null);
vo.setDownloadUrl(null);
vo.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1);
vo.setError(entity.getErrorMessage());
return vo;
@@ -229,9 +229,7 @@ public class DedupeRunService {
vo.setResultId(entity.getId());
vo.setSourceFilename(entity.getSourceFilename());
vo.setOutputFilename(entity.getResultFilename());
vo.setDownloadUrl(entity.getSuccess() != null && entity.getSuccess() == 1
? ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl())
: null);
vo.setDownloadUrl(null);
vo.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1);
vo.setError(entity.getErrorMessage());
return vo;
@@ -50,6 +50,10 @@ public class DeleteBrandResultItemVo {
@Schema(description = "下载地址")
private String downloadUrl;
private Long fileJobId;
private String fileStatus;
private String fileError;
private Boolean fileReady;
@Schema(description = "任务ID")
private Long taskId;
@@ -34,6 +34,8 @@ import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.ziniao.model.vo.ZiniaoShopMatchResultVo;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopIndexService;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
@@ -80,6 +82,7 @@ public class DeleteBrandRunService {
private final ObjectMapper objectMapper;
private final DeleteBrandProgressProperties deleteBrandProgressProperties;
private final TaskPressureProperties taskPressureProperties;
private final TaskFileJobService taskFileJobService;
public DeleteBrandRunVo run(DeleteBrandRunRequest request) {
if (request.getUserId() == null || request.getUserId() <= 0) {
@@ -271,7 +274,8 @@ public class DeleteBrandRunService {
item.setSourceFilename(entity.getSourceFilename());
item.setShopName(entity.getSourceFilename() == null ? null : FileUtil.mainName(entity.getSourceFilename()));
item.setOutputFilename(entity.getResultFilename());
item.setDownloadUrl(entity.getResultFileUrl() == null ? null : ossStorageService.getPublicUrl(entity.getResultFileUrl()));
item.setDownloadUrl(null);
attachFileJobState(item, entity);
item.setTotalRows(entity.getRowCount());
item.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1);
item.setError(entity.getErrorMessage());
@@ -320,6 +324,18 @@ public class DeleteBrandRunService {
return vo;
}
private void attachFileJobState(DeleteBrandResultItemVo item, FileResultEntity entity) {
TaskFileJobEntity job = taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId());
item.setFileReady(entity.getResultFileUrl() != null && !entity.getResultFileUrl().isBlank());
if (job == null) {
item.setFileStatus(Boolean.TRUE.equals(item.getFileReady()) ? "SUCCESS" : null);
return;
}
item.setFileJobId(job.getId());
item.setFileStatus(job.getStatus());
item.setFileError(job.getErrorMessage());
}
public DeleteBrandTaskDeletionStatusVo getTaskDeletionStatus(Long taskId, Long userId) {
FileTaskEntity task = loadTaskForExecution(taskId);
if (task == null || !MODULE_TYPE.equals(task.getModuleType()) || !userId.equals(task.getUserId())) {
@@ -373,12 +389,19 @@ public class DeleteBrandRunService {
Map<String, Map<String, DeleteBrandCountryAsinVo>> grouped = new LinkedHashMap<>();
Map<String, String> displayCountryNames = new LinkedHashMap<>();
for (int rowNum = 2; rowNum <= sheet.getLastRowNum(); rowNum++) {
int firstDataRow = pairs.stream()
.mapToInt(CountryColumnPair::dataStartRowIndex)
.min()
.orElse(2);
for (int rowNum = firstDataRow; rowNum <= sheet.getLastRowNum(); rowNum++) {
Row row = sheet.getRow(rowNum);
if (row == null) {
continue;
}
for (CountryColumnPair pair : pairs) {
if (rowNum < pair.dataStartRowIndex()) {
continue;
}
String country = pair.country();
String asin = normalizeCellText(formatter.formatCellValue(row.getCell(pair.asinColumnIndex())));
String status = normalizeCellText(formatter.formatCellValue(row.getCell(pair.statusColumnIndex())));
@@ -457,7 +480,10 @@ public class DeleteBrandRunService {
continue;
}
if ("删除ASIN".equals(firstHeader) && "状态".equals(secondHeader)) {
pairs.add(new CountryColumnPair(title, i, i + 1));
pairs.add(new CountryColumnPair(title, i, i + 1, 2));
i++;
} else if (!"店铺名".equals(title) && "状态".equals(secondHeader)) {
pairs.add(new CountryColumnPair(title, i, i + 1, 1));
i++;
}
}
@@ -486,7 +512,7 @@ public class DeleteBrandRunService {
.replaceAll("\\s+", " ");
}
private record CountryColumnPair(String country, int asinColumnIndex, int statusColumnIndex) {
private record CountryColumnPair(String country, int asinColumnIndex, int statusColumnIndex, int dataStartRowIndex) {
}
private String resolveFileIdentity(DeleteBrandSourceFileDto sourceFile) {
@@ -1085,58 +1111,111 @@ public class DeleteBrandRunService {
return;
}
Map<String, DeleteBrandParsedFileCacheDto> parsedPayload = deleteBrandTaskStorageService.loadParsedPayload(taskId);
if (parsedPayload == null || parsedPayload.isEmpty()) {
int expectedFiles = deleteBrandTaskStorageService.countParsedPayloadScopes(taskId);
if (expectedFiles <= 0) {
tryFinalizeTaskFromTerminalResults(task);
return;
}
int expectedFiles = parsedPayload.size();
// 移除 finishedFilesHint 的强依赖因为 Redis 里的 finished_files 可能是旧值
// 既然进入 tryFinalizeTask就说明有新分片到达应该以 loadMergedChunks 为准
Map<String, List<DeleteBrandResultFileDto>> mergedByFile = deleteBrandTaskStorageService.loadMergedChunks(taskId);
int finishedFiles = countCompletedFiles(mergedByFile);
int finishedFiles = deleteBrandTaskStorageService.countCompletedScopes(taskId);
if (finishedFiles < expectedFiles) {
Map<String, String> progress = new LinkedHashMap<>();
progress.put("finished_files", String.valueOf(finishedFiles));
// 定时补偿只负责再试一次 finalize不能把缺片任务重新续命
// 否则 stale-check 会一直看不到超时任务
if (!fromCompensation) {
progress.put("updated_at", String.valueOf(System.currentTimeMillis()));
}
deleteBrandTaskCacheService.saveProgress(taskId, progress);
task.setSuccessFileCount(finishedFiles);
if (!fromCompensation) {
task.setUpdatedAt(LocalDateTime.now());
}
fileTaskMapper.updateById(task);
deleteBrandTaskCacheService.saveTaskCache(task);
updateTaskFinalizeProgress(task, finishedFiles, fromCompensation);
log.info("[DeleteBrand] tryFinalizeTask skipping -> finishedFiles ({}) < expectedFiles ({})", finishedFiles, expectedFiles);
return;
}
Map<String, String> progress = new LinkedHashMap<>();
progress.put("finished_files", String.valueOf(finishedFiles));
progress.put("updated_at", String.valueOf(System.currentTimeMillis()));
if (fromCompensation) {
progress.put("last_heartbeat_at", String.valueOf(System.currentTimeMillis()));
updateTaskFinalizeProgress(task, finishedFiles, true);
Map<String, DeleteBrandParsedFileCacheDto> parsedPayload = deleteBrandTaskStorageService.loadParsedPayload(taskId);
if (parsedPayload == null || parsedPayload.isEmpty()) {
tryFinalizeTaskFromTerminalResults(task);
return;
}
deleteBrandTaskCacheService.saveProgress(taskId, progress, true);
task.setSuccessFileCount(finishedFiles);
task.setUpdatedAt(LocalDateTime.now());
fileTaskMapper.updateById(task);
deleteBrandTaskCacheService.saveTaskCache(task);
Map<String, List<DeleteBrandResultFileDto>> mergedByFile = deleteBrandTaskStorageService.loadMergedChunks(taskId);
log.info("[DeleteBrand] tryFinalizeTask calling finalizeTask -> finishedFiles: {}, expectedFiles: {}", finishedFiles, expectedFiles);
finalizeTask(task, parsedPayload, mergedByFile);
}
private void updateTaskFinalizeProgress(FileTaskEntity task, int finishedFiles, boolean fromCompensation) {
Map<String, String> progress = new LinkedHashMap<>();
progress.put("finished_files", String.valueOf(finishedFiles));
if (!fromCompensation) {
progress.put("updated_at", String.valueOf(System.currentTimeMillis()));
}
deleteBrandTaskCacheService.saveProgress(task.getId(), progress);
Integer currentFinished = task.getSuccessFileCount();
if (fromCompensation && currentFinished != null && currentFinished == finishedFiles) {
return;
}
task.setSuccessFileCount(finishedFiles);
if (!fromCompensation) {
task.setUpdatedAt(LocalDateTime.now());
}
fileTaskMapper.updateById(task);
deleteBrandTaskCacheService.saveTaskCache(task);
}
private void tryFinalizeTaskFromTerminalResults(FileTaskEntity task) {
if (task == null || task.getId() == null) {
return;
}
List<FileResultEntity> resultEntities = fileResultMapper.selectList(new LambdaQueryWrapper<FileResultEntity>()
.eq(FileResultEntity::getModuleType, MODULE_TYPE)
.eq(FileResultEntity::getTaskId, task.getId())
.orderByAsc(FileResultEntity::getId));
if (resultEntities == null || resultEntities.isEmpty()) {
return;
}
if (taskFileJobService.countUnfinishedAssembleJobs(task.getId(), MODULE_TYPE) > 0L) {
return;
}
int successCount = 0;
int failedCount = 0;
for (FileResultEntity entity : resultEntities) {
boolean success = (entity.getSuccess() != null && entity.getSuccess() == 1)
|| (entity.getResultFileUrl() != null && !entity.getResultFileUrl().isBlank());
boolean failed = !success && entity.getErrorMessage() != null && !entity.getErrorMessage().isBlank();
if (!success && !failed) {
return;
}
if (success) {
successCount++;
} else {
failedCount++;
}
}
if (task.getSourceFileCount() != null
&& task.getSourceFileCount() > 0
&& successCount + failedCount < task.getSourceFileCount()) {
return;
}
task.setStatus(successCount > 0 ? "SUCCESS" : "FAILED");
task.setSuccessFileCount(successCount);
task.setFailedFileCount(failedCount);
task.setErrorMessage(successCount > 0 ? null : "任务中间数据缺失,已按结果行状态自动收尾");
task.setUpdatedAt(LocalDateTime.now());
task.setFinishedAt(LocalDateTime.now());
fileTaskMapper.updateById(task);
deleteBrandTaskCacheService.saveTaskCache(task);
deleteBrandTaskCacheService.saveProgress(task.getId(), Map.of(
"phase", successCount > 0 ? "success" : DeleteBrandTaskCacheService.PHASE_FAILED,
"finished_files", String.valueOf(successCount),
"updated_at", String.valueOf(System.currentTimeMillis())
), true);
log.info("[DeleteBrand] finalized orphan running task from terminal results -> taskId: {}, status: {}, successFiles: {}, failedFiles: {}",
task.getId(), task.getStatus(), successCount, failedCount);
}
private int countCompletedFiles(Map<String, List<DeleteBrandResultFileDto>> mergedByFile) {
int finishedFiles = 0;
for (List<DeleteBrandResultFileDto> chunks : mergedByFile.values()) {
@@ -1205,28 +1284,22 @@ public class DeleteBrandRunService {
if (!isMergedFileCompleted(chunks)) {
throw new BusinessException("存在未完成的结果分片: " + parsedFile.getSourceFilename());
}
MergedDeleteBrandFile mergedFile = mergeChunks(parsedFile, chunks);
File outputFile = buildResultWorkbookPreserveLayout(task.getId(), parsedFile, mergedFile);
mergeChunks(parsedFile, chunks);
deleteBrandTaskCacheService.saveProgress(task.getId(), Map.of(
"phase", DeleteBrandTaskCacheService.PHASE_UPLOADING,
"file_name", blankToEmpty(parsedFile.getSourceFilename()),
"updated_at", String.valueOf(System.currentTimeMillis())
));
String objectKey = ossStorageService.uploadResultFile(outputFile, MODULE_TYPE);
FileResultEntity resultEntity = resultEntityByIdentity.get(fileIdentity);
if (resultEntity == null) {
throw new BusinessException("结果记录不存在: " + parsedFile.getSourceFilename());
}
resultEntity.setResultFilename(outputFile.getName());
resultEntity.setResultFileUrl(objectKey);
resultEntity.setResultFileSize(outputFile.length());
String outputFilename = buildResultFilename(parsedFile);
resultEntity.setResultFilename(outputFilename);
resultEntity.setResultFileUrl(null);
resultEntity.setResultFileSize(0L);
resultEntity.setResultContentType(CONTENT_TYPE_XLSX);
resultEntity.setRowCount(parsedFile.getTotalRows());
resultEntity.setSuccess(1);
resultEntity.setErrorMessage(null);
fileResultMapper.updateById(resultEntity);
taskFileJobService.enqueueAssembleResult(task.getId(), MODULE_TYPE, resultEntity.getId(), fileIdentity);
DeleteBrandResultItemVo item = new DeleteBrandResultItemVo();
item.setResultId(resultEntity.getId());
@@ -1235,8 +1308,8 @@ public class DeleteBrandRunService {
item.setSourceFilename(parsedFile.getSourceFilename());
item.setShopName(parsedFile.getShopName());
item.setCompanyName(parsedFile.getCompanyName());
item.setOutputFilename(outputFile.getName());
item.setDownloadUrl(ossStorageService.getPublicUrl(objectKey));
item.setOutputFilename(outputFilename);
item.setDownloadUrl(null);
item.setTotalRows(parsedFile.getTotalRows());
item.setCountryCount(parsedFile.getCountries() == null ? 0 : parsedFile.getCountries().size());
item.setCountries(parsedFile.getCountries());
@@ -1263,8 +1336,6 @@ public class DeleteBrandRunService {
task.setFinishedAt(LocalDateTime.now());
fileTaskMapper.updateById(task);
deleteBrandTaskCacheService.saveTaskCache(task);
deleteBrandTaskStorageService.deleteTaskData(task.getId());
deleteBrandTaskCacheService.saveProgress(task.getId(), Map.of(
"phase", "success",
"file_index", String.valueOf(parsedPayload.size()),
@@ -1293,6 +1364,75 @@ public class DeleteBrandRunService {
}
}
public void processResultFileJob(TaskFileJobEntity job) {
if (job == null || job.getTaskId() == null || job.getResultId() == null) {
throw new BusinessException("结果文件任务参数不完整");
}
FileTaskEntity task = fileTaskMapper.selectById(job.getTaskId());
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
throw new BusinessException("任务不存在");
}
FileResultEntity resultEntity = fileResultMapper.selectById(job.getResultId());
if (resultEntity == null || !MODULE_TYPE.equals(resultEntity.getModuleType())) {
throw new BusinessException("结果记录不存在");
}
Map<String, DeleteBrandParsedFileCacheDto> parsedPayload = deleteBrandTaskStorageService.loadParsedPayload(job.getTaskId());
Map<String, List<DeleteBrandResultFileDto>> mergedByFile = deleteBrandTaskStorageService.loadMergedChunks(job.getTaskId());
String fileIdentity = blankToEmpty(job.getScopeKey());
if (fileIdentity.isBlank()) {
fileIdentity = blankToEmpty(resultEntity.getSourceFileUrl());
}
if (fileIdentity.isBlank()) {
fileIdentity = blankToEmpty(resultEntity.getSourceFilename());
}
DeleteBrandParsedFileCacheDto parsedFile = parsedPayload.get(fileIdentity);
if (parsedFile == null) {
throw new BusinessException("任务原始数据不存在: " + fileIdentity);
}
List<DeleteBrandResultFileDto> chunks = mergedByFile.get(fileIdentity);
if (!isMergedFileCompleted(chunks)) {
throw new BusinessException("结果分片未完整: " + fileIdentity);
}
MergedDeleteBrandFile mergedFile = mergeChunks(parsedFile, chunks);
File outputFile = buildResultWorkbookPreserveLayout(job.getTaskId(), parsedFile, mergedFile);
try {
deleteBrandTaskCacheService.saveProgress(job.getTaskId(), Map.of(
"phase", DeleteBrandTaskCacheService.PHASE_UPLOADING,
"file_name", blankToEmpty(parsedFile.getSourceFilename()),
"updated_at", String.valueOf(System.currentTimeMillis())
));
String objectKey = ossStorageService.uploadResultFile(outputFile, MODULE_TYPE);
resultEntity.setResultFilename(outputFile.getName());
resultEntity.setResultFileUrl(objectKey);
resultEntity.setResultFileSize(outputFile.length());
resultEntity.setResultContentType(CONTENT_TYPE_XLSX);
resultEntity.setRowCount(parsedFile.getTotalRows());
resultEntity.setSuccess(1);
resultEntity.setErrorMessage(null);
fileResultMapper.updateById(resultEntity);
} finally {
deleteQuietly(outputFile);
}
}
public void cleanupResultFileJob(TaskFileJobEntity job) {
if (job == null || job.getTaskId() == null) {
return;
}
if (taskFileJobService.countUnfinishedAssembleJobs(job.getTaskId(), MODULE_TYPE) == 0L) {
deleteBrandTaskStorageService.deleteTaskData(job.getTaskId());
}
}
private String buildResultFilename(DeleteBrandParsedFileCacheDto parsedFile) {
String filename = blankToDefault(parsedFile.getSourceFilename(), "delete-brand-result.xlsx");
String lower = filename.toLowerCase(Locale.ROOT);
if (!lower.endsWith(".xlsx") && !lower.endsWith(".xls")) {
return filename + ".xlsx";
}
return filename;
}
private MergedDeleteBrandFile mergeChunks(DeleteBrandParsedFileCacheDto parsedFile, List<DeleteBrandResultFileDto> chunks) {
Integer chunkTotal = chunks.get(0).getChunkTotal();
for (DeleteBrandResultFileDto chunk : chunks) {
@@ -27,7 +27,6 @@ import java.nio.file.attribute.FileTime;
import java.time.Duration;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.util.List;
import java.util.Map;
@@ -47,6 +46,7 @@ public class DeleteBrandStaleTaskService {
private final FileTaskMapper fileTaskMapper;
private final DeleteBrandTaskCacheService deleteBrandTaskCacheService;
private final DeleteBrandTaskStorageService deleteBrandTaskStorageService;
private final DeleteBrandRunService deleteBrandRunService;
private final ProductRiskTaskService productRiskTaskService;
private final ProductRiskTaskCacheService productRiskTaskCacheService;
@@ -110,7 +110,13 @@ public class DeleteBrandStaleTaskService {
}
private void failStaleDeleteBrandTasks() {
LocalDateTime threshold = LocalDateTime.now().minusMinutes(deleteBrandProgressProperties.getHeartbeatTimeoutMinutes());
long minutes = Math.max(1L, deleteBrandProgressProperties.getHeartbeatTimeoutMinutes());
long initialMinutes = Math.max(minutes, deleteBrandProgressProperties.getDeleteBrandInitialTimeoutMinutes());
long staleTimeoutMillis = Duration.ofMinutes(minutes).toMillis();
long nowMillis = System.currentTimeMillis();
LocalDateTime now = LocalDateTime.now();
LocalDateTime threshold = now.minusMinutes(minutes);
LocalDateTime initialThreshold = now.minusMinutes(initialMinutes);
java.util.List<FileTaskEntity> runningTasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getModuleType, MODULE_TYPE_DELETE_BRAND)
@@ -120,36 +126,35 @@ public class DeleteBrandStaleTaskService {
for (FileTaskEntity task : runningTasks) {
Map<Object, Object> progress = deleteBrandTaskCacheService.getProgress(task.getId());
boolean hasProgress = progress != null && !progress.isEmpty() && progress.containsKey("last_heartbeat_at");
boolean isActive = false;
if (progress != null && progress.containsKey("has_progress")) {
Object hpObj = progress.get("has_progress");
if (hpObj instanceof Boolean) {
isActive = (Boolean) hpObj;
} else if (hpObj instanceof String) {
isActive = Boolean.parseBoolean((String) hpObj);
}
}
long lastHeartbeatAt = 0L;
if (hasProgress) {
if (progress != null && !progress.isEmpty() && progress.containsKey("last_heartbeat_at")) {
try {
lastHeartbeatAt = Long.parseLong(String.valueOf(progress.get("last_heartbeat_at")));
} catch (Exception ignored) {
}
}
boolean hasStartedProgress = lastHeartbeatAt > 0L;
if (!hasStartedProgress) {
hasStartedProgress = deleteBrandTaskStorageService.countCompletedScopes(task.getId()) > 0
|| deleteBrandTaskStorageService.countParsedPayloadScopes(task.getId()) > 0;
}
if (lastHeartbeatAt > 0 && isActive) {
LocalDateTime lastActivityTime = LocalDateTime.ofInstant(Instant.ofEpochMilli(lastHeartbeatAt), ZoneId.systemDefault());
if (lastActivityTime.isAfter(threshold)) {
continue;
}
} else {
LocalDateTime queuedThreshold = LocalDateTime.now().minusHours(12);
if (task.getCreatedAt() != null && task.getCreatedAt().isAfter(queuedThreshold)) {
if (lastHeartbeatAt > 0L && nowMillis - lastHeartbeatAt < staleTimeoutMillis) {
continue;
}
if (!hasStartedProgress && task.getCreatedAt() != null && task.getCreatedAt().isAfter(initialThreshold)) {
continue;
}
try {
deleteBrandRunService.tryFinalizeTask(task.getId(), true);
FileTaskEntity refreshed = fileTaskMapper.selectById(task.getId());
if (refreshed != null && !"RUNNING".equals(refreshed.getStatus())) {
deleteBrandTaskCacheService.saveTaskCache(refreshed);
continue;
}
} catch (Exception ex) {
log.warn("[stale-check] delete-brand finalize threw taskId={} msg={}", task.getId(), ex.getMessage());
}
int updated = fileTaskMapper.update(null, new LambdaUpdateWrapper<FileTaskEntity>()
@@ -185,20 +190,24 @@ public class DeleteBrandStaleTaskService {
if (runningTasks.isEmpty()) {
return stats;
}
Map<Long, Long> heartbeatByTaskId = productRiskTaskCacheService.getTaskHeartbeatMillisBatch(
runningTasks.stream().map(FileTaskEntity::getId).toList());
log.info("[stale-check] product-risk candidates={} threshold={} timeoutMinutes={} initialThreshold={} initialTimeoutMinutes={}",
runningTasks.size(), threshold, minutes, initialThreshold, initialMinutes);
for (FileTaskEntity task : runningTasks) {
boolean hasUploadedPayload = productRiskTaskCacheService.hasAnyShopMergedPayload(task.getId());
long lastPayloadHeartbeatMillis = productRiskTaskCacheService.getTaskHeartbeatMillis(task.getId());
long lastPayloadHeartbeatMillis = heartbeatByTaskId.getOrDefault(task.getId(), 0L);
boolean hasFinishedRows = (task.getSuccessFileCount() != null && task.getSuccessFileCount() > 0)
|| (task.getFailedFileCount() != null && task.getFailedFileCount() > 0);
boolean hasStartedProgress = hasUploadedPayload || hasFinishedRows;
if (lastPayloadHeartbeatMillis > 0L && nowMillis - lastPayloadHeartbeatMillis < staleTimeoutMillis) {
stats.skippedTaskCount++;
log.info("[stale-check] product-risk skip recent-task-heartbeat taskId={} lastPayloadHeartbeatMillis={} timeoutMinutes={} updatedAt={}",
task.getId(), lastPayloadHeartbeatMillis, minutes, task.getUpdatedAt());
continue;
}
boolean hasStartedProgress = lastPayloadHeartbeatMillis > 0L || hasFinishedRows;
if (!hasStartedProgress) {
hasStartedProgress = productRiskTaskCacheService.hasAnyShopMergedPayload(task.getId());
}
if (!hasStartedProgress && task.getCreatedAt() != null && task.getCreatedAt().isAfter(initialThreshold)) {
stats.skippedTaskCount++;
log.info("[stale-check] product-risk skip initial-grace taskId={} createdAt={} initialThreshold={}",
@@ -250,18 +259,22 @@ public class DeleteBrandStaleTaskService {
if (runningTasks.isEmpty()) {
return stats;
}
Map<Long, Long> heartbeatByTaskId = priceTrackTaskCacheService.getTaskHeartbeatMillisBatch(
runningTasks.stream().map(FileTaskEntity::getId).toList());
log.info("[stale-check] price-track candidates={} threshold={} timeoutMinutes={} initialThreshold={} initialTimeoutMinutes={}",
runningTasks.size(), threshold, minutes, initialThreshold, initialMinutes);
for (FileTaskEntity task : runningTasks) {
long lastHeartbeatMillis = priceTrackTaskCacheService.getTaskHeartbeatMillis(task.getId());
boolean hasUploadedPayload = priceTrackTaskCacheService.hasAnyShopMergedPayload(task.getId());
long lastHeartbeatMillis = heartbeatByTaskId.getOrDefault(task.getId(), 0L);
boolean hasFinishedRows = (task.getSuccessFileCount() != null && task.getSuccessFileCount() > 0)
|| (task.getFailedFileCount() != null && task.getFailedFileCount() > 0);
boolean hasStartedProgress = lastHeartbeatMillis > 0L || hasUploadedPayload || hasFinishedRows;
if (lastHeartbeatMillis > 0L && nowMillis - lastHeartbeatMillis < staleTimeoutMillis) {
stats.skippedTaskCount++;
continue;
}
boolean hasStartedProgress = lastHeartbeatMillis > 0L || hasFinishedRows;
if (!hasStartedProgress) {
hasStartedProgress = priceTrackTaskCacheService.hasAnyShopMergedPayload(task.getId());
}
if (!hasStartedProgress && task.getCreatedAt() != null && task.getCreatedAt().isAfter(initialThreshold)) {
stats.skippedTaskCount++;
continue;
@@ -312,20 +325,24 @@ public class DeleteBrandStaleTaskService {
if (runningTasks.isEmpty()) {
return stats;
}
Map<Long, Long> heartbeatByTaskId = shopMatchTaskCacheService.getTaskHeartbeatMillisBatch(
runningTasks.stream().map(FileTaskEntity::getId).toList());
log.info("[stale-check] shop-match candidates={} threshold={} timeoutMinutes={} initialThreshold={} initialTimeoutMinutes={}",
runningTasks.size(), threshold, minutes, initialThreshold, initialMinutes);
for (FileTaskEntity task : runningTasks) {
long lastHeartbeatMillis = shopMatchTaskCacheService.getTaskHeartbeatMillis(task.getId());
boolean hasUploadedPayload = shopMatchTaskCacheService.hasAnyShopMergedPayload(task.getId());
long lastHeartbeatMillis = heartbeatByTaskId.getOrDefault(task.getId(), 0L);
boolean hasFinishedRows = (task.getSuccessFileCount() != null && task.getSuccessFileCount() > 0)
|| (task.getFailedFileCount() != null && task.getFailedFileCount() > 0);
boolean hasStartedProgress = lastHeartbeatMillis > 0L || hasUploadedPayload || hasFinishedRows;
if (lastHeartbeatMillis > 0L && nowMillis - lastHeartbeatMillis < staleTimeoutMillis) {
stats.skippedTaskCount++;
log.info("[stale-check] shop-match skip recent-task-heartbeat taskId={} lastHeartbeatMillis={} timeoutMinutes={} updatedAt={}",
task.getId(), lastHeartbeatMillis, minutes, task.getUpdatedAt());
continue;
}
boolean hasStartedProgress = lastHeartbeatMillis > 0L || hasFinishedRows;
if (!hasStartedProgress) {
hasStartedProgress = shopMatchTaskCacheService.hasAnyShopMergedPayload(task.getId());
}
if (!hasStartedProgress && task.getCreatedAt() != null && task.getCreatedAt().isAfter(initialThreshold)) {
stats.skippedTaskCount++;
log.info("[stale-check] shop-match skip initial-grace taskId={} createdAt={} initialThreshold={}",
@@ -378,16 +395,20 @@ public class DeleteBrandStaleTaskService {
if (runningTasks.isEmpty()) {
return stats;
}
Map<Long, Long> heartbeatByTaskId = patrolDeleteTaskCacheService.getTaskHeartbeatMillisBatch(
runningTasks.stream().map(FileTaskEntity::getId).toList());
for (FileTaskEntity task : runningTasks) {
long lastHeartbeatMillis = patrolDeleteTaskCacheService.getTaskHeartbeatMillis(task.getId());
boolean hasUploadedPayload = patrolDeleteTaskCacheService.hasAnyShopMergedPayload(task.getId());
long lastHeartbeatMillis = heartbeatByTaskId.getOrDefault(task.getId(), 0L);
boolean hasFinishedRows = (task.getSuccessFileCount() != null && task.getSuccessFileCount() > 0)
|| (task.getFailedFileCount() != null && task.getFailedFileCount() > 0);
boolean hasStartedProgress = lastHeartbeatMillis > 0L || hasUploadedPayload || hasFinishedRows;
if (lastHeartbeatMillis > 0L && nowMillis - lastHeartbeatMillis < staleTimeoutMillis) {
stats.skippedTaskCount++;
continue;
}
boolean hasStartedProgress = lastHeartbeatMillis > 0L || hasFinishedRows;
if (!hasStartedProgress) {
hasStartedProgress = patrolDeleteTaskCacheService.hasAnyShopMergedPayload(task.getId());
}
if (!hasStartedProgress && task.getCreatedAt() != null && task.getCreatedAt().isAfter(initialThreshold)) {
stats.skippedTaskCount++;
continue;
@@ -431,6 +452,10 @@ public class DeleteBrandStaleTaskService {
java.util.List<FileTaskEntity> runningTasks = fileTaskMapper.selectList(new LambdaQueryWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getModuleType, MODULE_TYPE_DELETE_BRAND)
.eq(FileTaskEntity::getStatus, "RUNNING")
.and(wrapper -> wrapper
.gt(FileTaskEntity::getSuccessFileCount, 0)
.or()
.gt(FileTaskEntity::getFailedFileCount, 0))
.orderByAsc(FileTaskEntity::getUpdatedAt)
.last("limit 100"));
@@ -3,6 +3,7 @@ package com.nanri.aiimage.modules.deletebrand.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.TaskPressureProperties;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
@@ -11,6 +12,7 @@ import java.util.concurrent.ConcurrentHashMap;
@Service
@RequiredArgsConstructor
@Slf4j
public class DeleteBrandTaskCacheService {
public static final String PHASE_CRAWLING = "crawling";
@@ -52,9 +54,13 @@ public class DeleteBrandTaskCacheService {
}
String key = buildProgressKey(taskId);
stringRedisTemplate.opsForHash().putAll(key, merged);
stringRedisTemplate.expire(key, Duration.ofHours(PAYLOAD_TTL_HOURS));
progressRedisFlushAt.put(taskId, now);
try {
stringRedisTemplate.opsForHash().putAll(key, merged);
stringRedisTemplate.expire(key, Duration.ofHours(PAYLOAD_TTL_HOURS));
progressRedisFlushAt.put(taskId, now);
} catch (Exception ex) {
log.warn("[delete-brand-cache] save progress degraded taskId={} msg={}", taskId, ex.getMessage());
}
}
public java.util.Map<Object, Object> getProgress(Long taskId) {
@@ -63,7 +69,13 @@ public class DeleteBrandTaskCacheService {
if (cached != null && cached.isFresh(now)) {
return new java.util.LinkedHashMap<>(cached.values());
}
java.util.Map<Object, Object> values = stringRedisTemplate.opsForHash().entries(buildProgressKey(taskId));
java.util.Map<Object, Object> values;
try {
values = stringRedisTemplate.opsForHash().entries(buildProgressKey(taskId));
} catch (Exception ex) {
log.warn("[delete-brand-cache] load progress degraded taskId={} msg={}", taskId, ex.getMessage());
return java.util.Collections.emptyMap();
}
if (!values.isEmpty()) {
progressLocalCache.put(taskId, new LocalProgressCacheEntry(now, toStringMap(values)));
}
@@ -98,15 +110,24 @@ public class DeleteBrandTaskCacheService {
return result;
}
java.util.List<Object> pipelineResults = stringRedisTemplate.executePipelined(
(org.springframework.data.redis.core.RedisCallback<Object>) connection -> {
org.springframework.data.redis.serializer.RedisSerializer<String> serializer =
stringRedisTemplate.getStringSerializer();
for (Long taskId : missingTaskIds) {
connection.hGetAll(serializer.serialize(buildProgressKey(taskId)));
}
return null;
});
java.util.List<Object> pipelineResults;
try {
pipelineResults = stringRedisTemplate.executePipelined(
(org.springframework.data.redis.core.RedisCallback<Object>) connection -> {
org.springframework.data.redis.serializer.RedisSerializer<String> serializer =
stringRedisTemplate.getStringSerializer();
for (Long taskId : missingTaskIds) {
connection.hGetAll(serializer.serialize(buildProgressKey(taskId)));
}
return null;
});
} catch (Exception ex) {
log.warn("[delete-brand-cache] batch load progress degraded taskIds={} msg={}", missingTaskIds, ex.getMessage());
for (Long taskId : missingTaskIds) {
result.put(taskId, java.util.Collections.emptyMap());
}
return result;
}
for (int i = 0; i < missingTaskIds.size(); i++) {
Long taskId = missingTaskIds.get(i);
@@ -129,8 +150,12 @@ public class DeleteBrandTaskCacheService {
progressLocalCache.remove(taskId);
progressRedisFlushAt.remove(taskId);
taskEntityLocalCache.remove(taskId);
stringRedisTemplate.delete(buildProgressKey(taskId));
stringRedisTemplate.delete(buildTaskEntityKey(taskId));
try {
stringRedisTemplate.delete(buildProgressKey(taskId));
stringRedisTemplate.delete(buildTaskEntityKey(taskId));
} catch (Exception ex) {
log.warn("[delete-brand-cache] delete degraded taskId={} msg={}", taskId, ex.getMessage());
}
}
public void saveTaskCache(com.nanri.aiimage.modules.task.model.entity.FileTaskEntity task) {
@@ -180,7 +205,13 @@ public class DeleteBrandTaskCacheService {
}
java.util.List<String> keys = missingIds.stream().map(this::buildTaskEntityKey).toList();
java.util.List<String> values = stringRedisTemplate.opsForValue().multiGet(keys);
java.util.List<String> values;
try {
values = stringRedisTemplate.opsForValue().multiGet(keys);
} catch (Exception ex) {
log.warn("[delete-brand-cache] batch load task cache degraded taskIds={} msg={}", missingIds, ex.getMessage());
return result;
}
for (int i = 0; i < missingIds.size(); i++) {
Long taskId = missingIds.get(i);
@@ -6,11 +6,11 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.deletebrand.model.cache.DeleteBrandParsedFileCacheDto;
import com.nanri.aiimage.modules.deletebrand.model.dto.DeleteBrandResultFileDto;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.dao.DuplicateKeyException;
@@ -32,12 +32,11 @@ import java.util.Map;
public class DeleteBrandTaskStorageService {
private static final String MODULE_TYPE = "DELETE_BRAND";
private static final String OSS_POINTER_PREFIX = "oss:";
private final TaskChunkMapper taskChunkMapper;
private final TaskScopeStateMapper taskScopeStateMapper;
private final ObjectMapper objectMapper;
private final OssStorageService ossStorageService;
private final TransientPayloadStorageService transientPayloadStorageService;
@Transactional
public void saveParsedPayload(Long taskId, Map<String, DeleteBrandParsedFileCacheDto> parsedPayloadByScope) {
@@ -81,7 +80,7 @@ public class DeleteBrandTaskStorageService {
if (existing == null) {
throw new BusinessException("failed to save delete-brand parsed payload");
}
deleteParsedPayloadIfNeeded(existing.getParsedPayloadJson(), parsedPayloadPointer);
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(existing.getParsedPayloadJson(), parsedPayloadPointer);
taskScopeStateMapper.update(null, new LambdaUpdateWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getId, existing.getId())
.set(TaskScopeStateEntity::getScopeKey, scopeKey)
@@ -120,6 +119,28 @@ public class DeleteBrandTaskStorageService {
return result;
}
public int countParsedPayloadScopes(Long taskId) {
if (taskId == null || taskId <= 0) {
return 0;
}
Long count = taskScopeStateMapper.selectCount(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.isNotNull(TaskScopeStateEntity::getParsedPayloadJson));
return count == null ? 0 : count.intValue();
}
public int countCompletedScopes(Long taskId) {
if (taskId == null || taskId <= 0) {
return 0;
}
Long count = taskScopeStateMapper.selectCount(new LambdaQueryWrapper<TaskScopeStateEntity>()
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE)
.eq(TaskScopeStateEntity::getCompleted, 1));
return count == null ? 0 : count.intValue();
}
@Transactional
public boolean storeResultChunkIfChanged(Long taskId, String scopeKey, Integer chunkIndex, DeleteBrandResultFileDto chunk) {
if (taskId == null || taskId <= 0) {
@@ -145,6 +166,7 @@ public class DeleteBrandTaskStorageService {
.last("limit 1"));
boolean changed = true;
if (existingChunk == null) {
String storedPayload = transientPayloadStorageService.storeChunkPayload(MODULE_TYPE, taskId, scopeHash, chunkIndex, payloadJson);
TaskChunkEntity entity = new TaskChunkEntity();
entity.setTaskId(taskId);
entity.setModuleType(MODULE_TYPE);
@@ -152,7 +174,7 @@ public class DeleteBrandTaskStorageService {
entity.setScopeHash(scopeHash);
entity.setChunkIndex(chunkIndex);
entity.setChunkTotal(chunk.getChunkTotal() == null ? 0 : chunk.getChunkTotal());
entity.setPayloadJson(payloadJson);
entity.setPayloadJson(storedPayload);
entity.setPayloadHash(payloadHash);
entity.setCreatedAt(now);
entity.setUpdatedAt(now);
@@ -169,11 +191,13 @@ public class DeleteBrandTaskStorageService {
changed = !payloadHash.equals(concurrentChunk.getPayloadHash());
concurrentChunk.setScopeKey(normalizedScopeKey);
concurrentChunk.setChunkTotal(chunk.getChunkTotal() == null ? 0 : chunk.getChunkTotal());
concurrentChunk.setPayloadJson(payloadJson);
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(concurrentChunk.getPayloadJson(), storedPayload);
concurrentChunk.setPayloadJson(storedPayload);
concurrentChunk.setPayloadHash(payloadHash);
concurrentChunk.setUpdatedAt(now);
taskChunkMapper.updateById(concurrentChunk);
} else {
transientPayloadStorageService.deletePayloadIfPresent(storedPayload);
throw ex;
}
}
@@ -181,7 +205,9 @@ public class DeleteBrandTaskStorageService {
changed = !payloadHash.equals(existingChunk.getPayloadHash());
existingChunk.setScopeKey(normalizedScopeKey);
existingChunk.setChunkTotal(chunk.getChunkTotal() == null ? 0 : chunk.getChunkTotal());
existingChunk.setPayloadJson(payloadJson);
String storedPayload = transientPayloadStorageService.storeChunkPayload(MODULE_TYPE, taskId, scopeHash, chunkIndex, payloadJson);
transientPayloadStorageService.deleteReplacedPayloadIfNeeded(existingChunk.getPayloadJson(), storedPayload);
existingChunk.setPayloadJson(storedPayload);
existingChunk.setPayloadHash(payloadHash);
existingChunk.setUpdatedAt(now);
taskChunkMapper.updateById(existingChunk);
@@ -210,7 +236,8 @@ public class DeleteBrandTaskStorageService {
continue;
}
try {
DeleteBrandResultFileDto dto = objectMapper.readValue(entity.getPayloadJson(), DeleteBrandResultFileDto.class);
String payloadJson = transientPayloadStorageService.resolvePayload(entity.getPayloadJson(), "failed to load delete-brand result chunks");
DeleteBrandResultFileDto dto = objectMapper.readValue(payloadJson, DeleteBrandResultFileDto.class);
grouped.computeIfAbsent(entity.getScopeKey(), ignored -> new ArrayList<>()).add(dto);
} catch (Exception ex) {
throw new BusinessException("failed to load delete-brand result chunks");
@@ -228,12 +255,22 @@ public class DeleteBrandTaskStorageService {
return;
}
List<TaskScopeStateEntity> states = taskScopeStateMapper.selectList(new LambdaQueryWrapper<TaskScopeStateEntity>()
.select(TaskScopeStateEntity::getParsedPayloadJson)
.select(TaskScopeStateEntity::getParsedPayloadJson, TaskScopeStateEntity::getStateJson)
.eq(TaskScopeStateEntity::getTaskId, taskId)
.eq(TaskScopeStateEntity::getModuleType, MODULE_TYPE));
if (states != null) {
for (TaskScopeStateEntity state : states) {
deleteParsedPayloadIfNeeded(state.getParsedPayloadJson(), null);
transientPayloadStorageService.deletePayloadIfPresent(state.getParsedPayloadJson());
transientPayloadStorageService.deletePayloadIfPresent(state.getStateJson());
}
}
List<TaskChunkEntity> chunks = taskChunkMapper.selectList(new LambdaQueryWrapper<TaskChunkEntity>()
.select(TaskChunkEntity::getPayloadJson)
.eq(TaskChunkEntity::getTaskId, taskId)
.eq(TaskChunkEntity::getModuleType, MODULE_TYPE));
if (chunks != null) {
for (TaskChunkEntity chunk : chunks) {
transientPayloadStorageService.deletePayloadIfPresent(chunk.getPayloadJson());
}
}
taskChunkMapper.delete(new LambdaQueryWrapper<TaskChunkEntity>()
@@ -322,56 +359,17 @@ public class DeleteBrandTaskStorageService {
}
private String storeParsedPayload(Long taskId, String scopeHash, String payloadJson) {
try {
String pointer = OSS_POINTER_PREFIX + ossStorageService.uploadTaskParsedPayload(MODULE_TYPE, taskId, scopeHash, payloadJson);
return objectMapper.writeValueAsString(pointer);
} catch (Exception ex) {
throw new BusinessException("failed to save delete-brand parsed payload");
}
return transientPayloadStorageService.storeParsedPayload(MODULE_TYPE, taskId, scopeHash, payloadJson, true);
}
private String resolveParsedPayload(String value) {
if (isBlank(value)) {
return value;
}
String ossPointer = extractOssPointer(value);
if (ossPointer == null) {
return value;
}
try {
return ossStorageService.readObjectAsString(ossPointer.substring(OSS_POINTER_PREFIX.length()));
return transientPayloadStorageService.resolvePayload(value, "failed to load delete-brand parsed payload");
} catch (Exception ex) {
throw new BusinessException("failed to load delete-brand parsed payload");
}
}
private void deleteParsedPayloadIfNeeded(String oldValue, String newValue) {
String oldPointer = extractOssPointer(oldValue);
String newPointer = extractOssPointer(newValue);
if (oldPointer == null || oldPointer.equals(newPointer)) {
return;
}
try {
ossStorageService.deleteObject(oldPointer.substring(OSS_POINTER_PREFIX.length()));
} catch (Exception ignored) {
}
}
private String extractOssPointer(String value) {
if (isBlank(value)) {
return null;
}
if (value.startsWith(OSS_POINTER_PREFIX)) {
return value;
}
try {
String decoded = objectMapper.readValue(value, String.class);
return decoded != null && decoded.startsWith(OSS_POINTER_PREFIX) ? decoded : null;
} catch (Exception ignored) {
return null;
}
}
private String writeJson(Object value, String message) {
try {
return objectMapper.writeValueAsString(value);
@@ -0,0 +1,85 @@
package com.nanri.aiimage.modules.file.service.object;
import com.nanri.aiimage.config.TransientStorageProperties;
import io.minio.GetObjectArgs;
import io.minio.MinioClient;
import io.minio.PutObjectArgs;
import io.minio.RemoveObjectArgs;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import java.io.ByteArrayInputStream;
import java.nio.charset.StandardCharsets;
import java.util.Objects;
@Service
@RequiredArgsConstructor
public class RustfsObjectStorageService {
private final TransientStorageProperties properties;
public boolean isConfigured() {
return notBlank(properties.getEndpoint())
&& notBlank(properties.getBucket())
&& notBlank(properties.getAccessKeyId())
&& notBlank(properties.getAccessKeySecret());
}
public String uploadText(String objectKey, String content) {
if (!isConfigured()) {
throw new IllegalStateException("transient storage is not configured");
}
try (ByteArrayInputStream stream = new ByteArrayInputStream(
Objects.requireNonNullElse(content, "").getBytes(StandardCharsets.UTF_8))) {
buildClient().putObject(PutObjectArgs.builder()
.bucket(properties.getBucket())
.object(objectKey)
.stream(stream, stream.available(), -1)
.contentType("application/json")
.build());
return objectKey;
} catch (Exception ex) {
throw new IllegalStateException("failed to upload payload to transient storage", ex);
}
}
public String readObjectAsString(String objectKey) {
if (!isConfigured()) {
throw new IllegalStateException("transient storage is not configured");
}
try (var stream = buildClient().getObject(GetObjectArgs.builder()
.bucket(properties.getBucket())
.object(objectKey)
.build())) {
return new String(stream.readAllBytes(), StandardCharsets.UTF_8);
} catch (Exception ex) {
throw new IllegalStateException("failed to read payload from transient storage", ex);
}
}
public void deleteObject(String objectKey) {
if (!isConfigured() || !notBlank(objectKey)) {
return;
}
try {
buildClient().removeObject(RemoveObjectArgs.builder()
.bucket(properties.getBucket())
.object(objectKey)
.build());
} catch (Exception ex) {
throw new IllegalStateException("failed to delete payload from transient storage", ex);
}
}
private MinioClient buildClient() {
return MinioClient.builder()
.endpoint(properties.getEndpoint())
.credentials(properties.getAccessKeyId(), properties.getAccessKeySecret())
.region(properties.getRegion())
.build();
}
private boolean notBlank(String value) {
return value != null && !value.isBlank();
}
}
@@ -43,6 +43,10 @@ public class PatrolDeleteResultItemVo {
private LocalDateTime finishedAt;
private String outputFilename;
private String downloadUrl;
private Long fileJobId;
private String fileStatus;
private String fileError;
private Boolean fileReady;
@JsonProperty("countrySections")
private List<PatrolDeleteCountrySectionDto> countrySections = new ArrayList<>();
@@ -6,6 +6,7 @@ import com.nanri.aiimage.modules.patroldelete.model.dto.PatrolDeleteShopPayloadD
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.service.TaskScopePayloadStorageService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
@@ -13,11 +14,13 @@ import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@Service
@RequiredArgsConstructor
@Slf4j
public class PatrolDeleteTaskCacheService {
private static final String MODULE_TYPE = "PATROL_DELETE";
@@ -60,7 +63,13 @@ public class PatrolDeleteTaskCacheService {
if (taskId == null || taskId <= 0) {
return 0L;
}
String raw = stringRedisTemplate.opsForValue().get(buildTaskHeartbeatKey(taskId));
String raw;
try {
raw = stringRedisTemplate.opsForValue().get(buildTaskHeartbeatKey(taskId));
} catch (Exception ex) {
log.warn("[patrol-delete-cache] get heartbeat degraded taskId={} msg={}", taskId, ex.getMessage());
return 0L;
}
if (raw == null || raw.isBlank()) {
return 0L;
}
@@ -71,11 +80,50 @@ public class PatrolDeleteTaskCacheService {
}
}
public Map<Long, Long> getTaskHeartbeatMillisBatch(List<Long> taskIds) {
Map<Long, Long> result = new LinkedHashMap<>();
if (taskIds == null || taskIds.isEmpty()) {
return result;
}
List<Long> normalized = taskIds.stream()
.filter(id -> id != null && id > 0)
.distinct()
.toList();
if (normalized.isEmpty()) {
return result;
}
List<String> keys = normalized.stream().map(this::buildTaskHeartbeatKey).toList();
List<String> values;
try {
values = stringRedisTemplate.opsForValue().multiGet(keys);
} catch (Exception ex) {
log.warn("[patrol-delete-cache] batch get heartbeat degraded taskIds={} msg={}", normalized, ex.getMessage());
return result;
}
for (int i = 0; i < normalized.size(); i++) {
String raw = values != null && i < values.size() ? values.get(i) : null;
if (raw == null || raw.isBlank()) {
result.put(normalized.get(i), 0L);
continue;
}
try {
result.put(normalized.get(i), Long.parseLong(raw));
} catch (NumberFormatException ignored) {
result.put(normalized.get(i), 0L);
}
}
return result;
}
public void touchTaskHeartbeat(Long taskId) {
stringRedisTemplate.opsForValue().set(
buildTaskHeartbeatKey(taskId),
String.valueOf(Instant.now().toEpochMilli()),
Duration.ofHours(PAYLOAD_TTL_HOURS));
try {
stringRedisTemplate.opsForValue().set(
buildTaskHeartbeatKey(taskId),
String.valueOf(Instant.now().toEpochMilli()),
Duration.ofHours(PAYLOAD_TTL_HOURS));
} catch (Exception ex) {
log.warn("[patrol-delete-cache] touch heartbeat degraded taskId={} msg={}", taskId, ex.getMessage());
}
}
public void deleteTaskCache(Long taskId) {
@@ -83,8 +131,12 @@ public class PatrolDeleteTaskCacheService {
return;
}
taskEntityLocalCache.remove(taskId);
stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId));
stringRedisTemplate.delete(buildTaskEntityKey(taskId));
try {
stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId));
stringRedisTemplate.delete(buildTaskEntityKey(taskId));
} catch (Exception ex) {
log.warn("[patrol-delete-cache] delete cache degraded taskId={} msg={}", taskId, ex.getMessage());
}
taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE);
}
@@ -133,7 +185,13 @@ public class PatrolDeleteTaskCacheService {
return result;
}
java.util.List<String> keys = missingIds.stream().map(this::buildTaskEntityKey).toList();
java.util.List<String> values = stringRedisTemplate.opsForValue().multiGet(keys);
java.util.List<String> values;
try {
values = stringRedisTemplate.opsForValue().multiGet(keys);
} catch (Exception ex) {
log.warn("[patrol-delete-cache] batch get task cache degraded taskIds={} msg={}", missingIds, ex.getMessage());
return result;
}
for (int i = 0; i < missingIds.size(); i++) {
Long taskId = missingIds.get(i);
String val = values != null && i < values.size() ? values.get(i) : null;
@@ -24,6 +24,10 @@ import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import com.nanri.aiimage.modules.task.service.TaskResultItemService;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@@ -45,6 +49,9 @@ public class PatrolDeleteTaskService {
private static final String MODULE_TYPE = "PATROL_DELETE";
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
private static final int RESULT_PENDING = -1;
private static final int RESULT_FAILED = 0;
private static final int RESULT_SUCCESS = 1;
private static final String INTERRUPTED_MESSAGE = "Python 在该店铺结果提交完成前中断";
private final FileTaskMapper fileTaskMapper;
@@ -56,6 +63,9 @@ public class PatrolDeleteTaskService {
private final ZiniaoShopSwitchService ziniaoShopSwitchService;
private final ObjectMapper objectMapper;
private final TaskPressureProperties taskPressureProperties;
private final TaskFileJobService taskFileJobService;
private final TaskResultItemService taskResultItemService;
private final TaskProgressSnapshotService taskProgressSnapshotService;
private FileTaskEntity loadTaskForExecution(Long taskId) {
Map<Long, FileTaskEntity> cachedTasks = taskCacheService.getTaskCacheBatch(List.of(taskId));
@@ -185,8 +195,7 @@ public class PatrolDeleteTaskService {
batch.getMissingTaskIds().add(taskId);
continue;
}
List<PatrolDeleteResultItemVo> snapshots = buildSnapshotFromDb(task, taskRows);
batch.getItems().addAll(snapshots);
batch.getItems().addAll(buildProgressItems(task, taskRows));
}
return batch;
}
@@ -258,12 +267,13 @@ public class PatrolDeleteTaskService {
result.setSourceFilename(normalizedShopName);
result.setSourceFileUrl(item.getShopId());
result.setUserId(request.getUserId());
result.setSuccess(null);
result.setSuccess(RESULT_PENDING);
result.setCreatedAt(now);
fileResultMapper.insert(result);
snapshots.add(toSnapshotVo(result, item, task.getStatus(), null));
}
persistTaskJson(task, uniqueItems, snapshots);
taskCacheService.saveTaskCache(task);
PatrolDeleteCreateTaskVo vo = new PatrolDeleteCreateTaskVo();
vo.setTaskId(task.getId());
@@ -320,6 +330,7 @@ public class PatrolDeleteTaskService {
persistSnapshotJson(task, snapshots);
fileTaskMapper.updateById(task);
taskCacheService.saveTaskCache(task);
tryFinalizeTask(taskId, false);
}
@@ -396,6 +407,7 @@ public class PatrolDeleteTaskService {
persistSnapshotJson(task, snapshots);
fileTaskMapper.updateById(task);
taskCacheService.saveTaskCache(task);
return changed;
}
@@ -442,6 +454,7 @@ public class PatrolDeleteTaskService {
updateTaskStatusFromRows(task, rows);
persistSnapshotJson(task, buildSnapshotFromDb(task, rows));
fileTaskMapper.updateById(task);
taskCacheService.saveTaskCache(task);
}
private long countTasks(Long userId, List<String> statuses) {
@@ -496,7 +509,11 @@ public class PatrolDeleteTaskService {
private Map<Long, Map<Long, PatrolDeleteResultItemVo>> buildSnapshotMap(Map<Long, FileTaskEntity> taskMap) {
Map<Long, Map<Long, PatrolDeleteResultItemVo>> out = new LinkedHashMap<>();
for (Map.Entry<Long, FileTaskEntity> entry : taskMap.entrySet()) {
out.put(entry.getKey(), indexSnapshotByResultId(parseTaskSnapshots(entry.getValue().getResultJson())));
List<PatrolDeleteResultItemVo> snapshots = parseTaskSnapshots(entry.getValue().getResultJson());
if (snapshots.isEmpty()) {
snapshots = taskResultItemService.listResultSnapshots(entry.getKey(), MODULE_TYPE, PatrolDeleteResultItemVo.class);
}
out.put(entry.getKey(), indexSnapshotByResultId(snapshots));
}
return out;
}
@@ -508,14 +525,13 @@ public class PatrolDeleteTaskService {
item.setShopName(firstNonBlank(item.getShopName(), entity.getSourceFilename()));
item.setShopId(firstNonBlank(item.getShopId(), entity.getSourceFileUrl()));
item.setTaskStatus(task != null ? task.getStatus() : item.getTaskStatus());
item.setSuccess(entity.getSuccess() == null ? item.getSuccess() : entity.getSuccess() == 1);
item.setSuccess(toSuccessFlag(entity.getSuccess(), item.getSuccess()));
item.setError(!blank(entity.getErrorMessage()) ? entity.getErrorMessage() : item.getError());
item.setCreatedAt(entity.getCreatedAt());
item.setFinishedAt(task != null ? task.getFinishedAt() : item.getFinishedAt());
item.setOutputFilename(firstNonBlank(item.getOutputFilename(), entity.getResultFilename()));
item.setDownloadUrl(blank(entity.getResultFileUrl())
? item.getDownloadUrl()
: ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl()));
item.setDownloadUrl(null);
attachFileJobState(item, entity);
if (item.getCountrySections() == null) {
item.setCountrySections(new ArrayList<>());
}
@@ -525,6 +541,18 @@ public class PatrolDeleteTaskService {
return item;
}
private void attachFileJobState(PatrolDeleteResultItemVo item, FileResultEntity entity) {
TaskFileJobEntity job = taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId());
item.setFileReady(!blank(entity.getResultFileUrl()));
if (job == null) {
item.setFileStatus(Boolean.TRUE.equals(item.getFileReady()) ? "SUCCESS" : null);
return;
}
item.setFileJobId(job.getId());
item.setFileStatus(job.getStatus());
item.setFileError(job.getErrorMessage());
}
private List<PatrolDeleteTaskItemDto> dedupeItems(List<PatrolDeleteTaskItemDto> items) {
LinkedHashMap<String, PatrolDeleteTaskItemDto> map = new LinkedHashMap<>();
for (PatrolDeleteTaskItemDto item : items) {
@@ -554,7 +582,7 @@ public class PatrolDeleteTaskService {
vo.setMatchStatus(item.getMatchStatus());
vo.setMatchMessage(item.getMatchMessage());
vo.setTaskStatus(taskStatus);
vo.setSuccess(result.getSuccess() == null ? null : result.getSuccess() == 1);
vo.setSuccess(toSuccessFlag(result.getSuccess(), null));
vo.setError(result.getErrorMessage());
vo.setCreatedAt(result.getCreatedAt());
vo.setFinishedAt(finishedAt);
@@ -569,6 +597,7 @@ public class PatrolDeleteTaskService {
try {
task.setRequestJson(objectMapper.writeValueAsString(requestItems));
task.setResultJson(objectMapper.writeValueAsString(snapshots));
syncSnapshotTables(task, snapshots);
fileTaskMapper.updateById(task);
} catch (Exception ex) {
throw new BusinessException("巡店删除任务快照保存失败");
@@ -585,6 +614,27 @@ public class PatrolDeleteTaskService {
return list;
}
private List<PatrolDeleteResultItemVo> buildProgressItems(FileTaskEntity task, List<FileResultEntity> rows) {
List<PatrolDeleteResultItemVo> list = new ArrayList<>();
for (FileResultEntity row : rows) {
PatrolDeleteResultItemVo item = new PatrolDeleteResultItemVo();
item.setResultId(row.getId());
item.setTaskId(row.getTaskId());
item.setShopName(row.getSourceFilename());
item.setShopId(row.getSourceFileUrl());
item.setTaskStatus(task == null ? null : task.getStatus());
item.setSuccess(toSuccessFlag(row.getSuccess(), null));
item.setError(row.getErrorMessage());
item.setCreatedAt(row.getCreatedAt());
item.setFinishedAt(task == null ? null : task.getFinishedAt());
item.setOutputFilename(row.getResultFilename());
item.setDownloadUrl(null);
attachFileJobState(item, row);
list.add(item);
}
return list;
}
private Map<Long, PatrolDeleteResultItemVo> indexSnapshotByResultId(List<PatrolDeleteResultItemVo> snapshots) {
Map<Long, PatrolDeleteResultItemVo> map = new LinkedHashMap<>();
if (snapshots == null) {
@@ -599,8 +649,8 @@ public class PatrolDeleteTaskService {
}
private void updateTaskStatusFromRows(FileTaskEntity task, List<FileResultEntity> rows) {
long successCount = rows.stream().filter(row -> Integer.valueOf(1).equals(row.getSuccess())).count();
long failedCount = rows.stream().filter(row -> Integer.valueOf(0).equals(row.getSuccess())).count();
long successCount = rows.stream().filter(row -> Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess())).count();
long failedCount = rows.stream().filter(row -> Integer.valueOf(RESULT_FAILED).equals(row.getSuccess())).count();
long pendingCount = rows.stream().filter(row -> !isResultFinished(row)).count();
task.setSuccessFileCount((int) successCount);
task.setFailedFileCount((int) failedCount);
@@ -753,27 +803,25 @@ public class PatrolDeleteTaskService {
.filter(item -> Boolean.TRUE.equals(item.getSuccess()))
.toList();
if (!successItems.isEmpty()) {
File workRoot = FileUtil.mkdir(FileUtil.file(System.getProperty("java.io.tmpdir"), "patrol-delete-result", String.valueOf(task.getId())));
String filename = safeFileStem("巡店删除-" + task.getId()) + ".xlsx";
File xlsx = FileUtil.file(workRoot, filename);
try {
excelAssemblyService.writeWorkbook(xlsx, successItems);
String objectKey = ossStorageService.uploadResultFile(xlsx, MODULE_TYPE);
long fileSize = xlsx.length();
int rowCount = excelAssemblyService.countRows(successItems);
String filename = buildTaskWorkbookFilename(task);
int rowCount = excelAssemblyService.countRows(successItems);
FileResultEntity firstSuccessRow = null;
for (FileResultEntity row : rows) {
if (Integer.valueOf(1).equals(row.getSuccess())) {
if (Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess())) {
row.setResultFilename(filename);
row.setResultFileUrl(objectKey);
row.setResultFileSize(fileSize);
row.setResultFileUrl(null);
row.setResultFileSize(0L);
row.setResultContentType(CONTENT_TYPE_XLSX);
row.setRowCount(rowCount);
fileResultMapper.updateById(row);
if (firstSuccessRow == null) {
firstSuccessRow = row;
}
}
}
if (firstSuccessRow != null) {
taskFileJobService.enqueueAssembleResult(task.getId(), MODULE_TYPE, firstSuccessRow.getId(), "task:" + task.getId());
snapshots = buildSnapshotFromDb(task, rows);
} finally {
FileUtil.del(xlsx);
}
}
@@ -782,14 +830,56 @@ public class PatrolDeleteTaskService {
taskCacheService.deleteTaskCache(task.getId());
}
public void processResultFileJob(TaskFileJobEntity job) {
if (job == null || job.getTaskId() == null) {
throw new BusinessException("结果文件任务参数不完整");
}
FileTaskEntity task = fileTaskMapper.selectById(job.getTaskId());
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
throw new BusinessException("任务不存在");
}
List<FileResultEntity> rows = listTaskRows(job.getTaskId());
List<PatrolDeleteResultItemVo> snapshots = taskResultItemService.listResultSnapshots(job.getTaskId(), MODULE_TYPE, PatrolDeleteResultItemVo.class);
if (snapshots.isEmpty()) {
snapshots = buildSnapshotFromDb(task, rows);
}
List<PatrolDeleteResultItemVo> successItems = snapshots.stream()
.filter(item -> Boolean.TRUE.equals(item.getSuccess()))
.toList();
if (successItems.isEmpty()) {
throw new BusinessException("没有可生成的巡店删除结果");
}
File workRoot = FileUtil.mkdir(FileUtil.file(System.getProperty("java.io.tmpdir"), "patrol-delete-result", String.valueOf(task.getId())));
String filename = buildTaskWorkbookFilename(task);
File xlsx = FileUtil.file(workRoot, filename);
try {
excelAssemblyService.writeWorkbook(xlsx, successItems);
String objectKey = ossStorageService.uploadResultFile(xlsx, MODULE_TYPE);
long fileSize = xlsx.length();
int rowCount = excelAssemblyService.countRows(successItems);
for (FileResultEntity row : rows) {
if (Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess())) {
row.setResultFilename(filename);
row.setResultFileUrl(objectKey);
row.setResultFileSize(fileSize);
row.setResultContentType(CONTENT_TYPE_XLSX);
row.setRowCount(rowCount);
fileResultMapper.updateById(row);
}
}
} finally {
FileUtil.del(xlsx);
}
}
private void markResultSuccess(FileResultEntity row) {
row.setSuccess(1);
row.setSuccess(RESULT_SUCCESS);
row.setErrorMessage(null);
fileResultMapper.updateById(row);
}
private void markResultFailed(FileResultEntity row, String message) {
row.setSuccess(0);
row.setSuccess(RESULT_FAILED);
row.setErrorMessage(blankToNull(message));
row.setResultFilename(null);
row.setResultFileUrl(null);
@@ -800,7 +890,14 @@ public class PatrolDeleteTaskService {
}
private boolean isResultFinished(FileResultEntity row) {
return Integer.valueOf(1).equals(row.getSuccess()) || Integer.valueOf(0).equals(row.getSuccess());
return Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess()) || Integer.valueOf(RESULT_FAILED).equals(row.getSuccess());
}
private Boolean toSuccessFlag(Integer dbValue, Boolean fallback) {
if (dbValue == null || Integer.valueOf(RESULT_PENDING).equals(dbValue)) {
return fallback;
}
return Integer.valueOf(RESULT_SUCCESS).equals(dbValue);
}
private List<PatrolDeleteResultItemVo> parseTaskSnapshots(String json) {
@@ -818,11 +915,39 @@ public class PatrolDeleteTaskService {
private void persistSnapshotJson(FileTaskEntity task, List<PatrolDeleteResultItemVo> snapshots) {
try {
task.setResultJson(objectMapper.writeValueAsString(snapshots == null ? List.of() : snapshots));
syncSnapshotTables(task, snapshots);
} catch (Exception ex) {
throw new BusinessException("巡店删除任务快照保存失败");
}
}
private void syncSnapshotTables(FileTaskEntity task, List<PatrolDeleteResultItemVo> snapshots) {
List<PatrolDeleteResultItemVo> safe = snapshots == null ? List.of() : snapshots;
taskResultItemService.replaceTaskSnapshots(task.getId(), MODULE_TYPE, safe, new TaskResultItemService.SnapshotKeyResolver() {
@Override
public Long resultId(Object snapshot) {
return ((PatrolDeleteResultItemVo) snapshot).getResultId();
}
@Override
public String scopeKey(Object snapshot) {
PatrolDeleteResultItemVo item = (PatrolDeleteResultItemVo) snapshot;
return firstNonBlank(item.getShopName(), "result:" + item.getResultId());
}
});
int successCount = 0;
int failedCount = 0;
for (PatrolDeleteResultItemVo item : safe) {
if (Boolean.TRUE.equals(item.getSuccess())) {
successCount++;
} else if (Boolean.FALSE.equals(item.getSuccess())) {
failedCount++;
}
}
taskProgressSnapshotService.save(task.getId(), MODULE_TYPE, firstNonBlank(task.getStatus(), "RUNNING"),
safe.size(), successCount, failedCount, null, task.getErrorMessage(), null);
}
private List<PatrolDeleteCountrySectionDto> copyCountrySections(List<PatrolDeleteCountrySectionDto> sections) {
List<PatrolDeleteCountrySectionDto> copy = new ArrayList<>();
if (sections == null) {
@@ -861,6 +986,11 @@ public class PatrolDeleteTaskService {
return !blank(first) ? first : second;
}
private String buildTaskWorkbookFilename(FileTaskEntity task) {
Long taskId = task == null ? null : task.getId();
return safeFileStem("巡店删除-" + (taskId == null ? "result" : taskId)) + ".xlsx";
}
private String safeFileStem(String value) {
String raw = value == null ? "result" : value.trim();
String safe = raw.replaceAll("[\\\\/:*?\"<>|]+", "_");
@@ -1,16 +1,20 @@
package com.nanri.aiimage.modules.permission.service;
import jakarta.annotation.PostConstruct;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.event.EventListener;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Component;
import java.util.List;
import java.util.concurrent.CompletableFuture;
@Component
@RequiredArgsConstructor
@Slf4j
@ConditionalOnProperty(prefix = "aiimage.permission-schema-init", name = "enabled", havingValue = "true")
public class PermissionMenuSchemaInitializer {
private final JdbcTemplate jdbcTemplate;
@@ -33,8 +37,12 @@ public class PermissionMenuSchemaInitializer {
new DefaultAdminMenu("版本管理", "admin_version", "version", 80)
);
@PostConstruct
@EventListener(ApplicationReadyEvent.class)
public void initialize() {
CompletableFuture.runAsync(this::initializeInternal);
}
private void initializeInternal() {
executeQuietly("""
CREATE TABLE IF NOT EXISTS columns (
id INT AUTO_INCREMENT PRIMARY KEY,
@@ -215,7 +215,7 @@ public class PriceTrackController {
+ "未完成/处理中:本次只传增量行数据,shop.success 不传或传 null,后端仅合并缓存继续等待。"
+ "已完成:传最后一批数据并设置 shop.success=true,后端会使用该店铺累计后的完整数据出结果。"
+ "失败:传 shop.success=false 或传非空 shop.error。"
+ "行级数据只保留表头字段:shopMallName、asin、price、recommendedPrice、minimumPrice、firstPlace、secondPlace、cartShopName、priceChangeStatus、modifyCount、status。"
+ "行级数据只保留表头字段:shopMallName、asin、price、recommendedPrice、shippingFee、minimumPrice、firstPlace、secondPlace、cartShopName、priceChangeStatus、modifyCount、status。"
)
public ApiResponse<Void> submitResult(
@Parameter(
@@ -73,6 +73,9 @@ public class PriceTrackSubmitResultRequest {
@Schema(description = "推荐价", example = "18.99")
private String recommendedPrice;
@Schema(description = "运费", example = "2.50")
private String shippingFee;
@Schema(description = "最低价", example = "18.50")
private String minimumPrice;
@@ -52,6 +52,10 @@ public class PriceTrackResultItemVo {
@Schema(description = "预签名下载 URL,列表可能带回,也可通过下载接口获取")
private String downloadUrl;
private Long fileJobId;
private String fileStatus;
private String fileError;
private Boolean fileReady;
@Schema(description = "所属循环任务 ID", example = "1")
private Long loopRunId;
@@ -24,6 +24,7 @@ public class PriceTrackExcelAssemblyService {
"ASIN",
"价格",
"推荐价",
"运费",
"最低价",
"第一名",
"第二名",
@@ -56,13 +57,14 @@ public class PriceTrackExcelAssemblyService {
row.createCell(1).setCellValue(valueOf(item.getAsin()));
row.createCell(2).setCellValue(valueOf(item.getPrice()));
row.createCell(3).setCellValue(valueOf(item.getRecommendedPrice()));
row.createCell(4).setCellValue(valueOf(item.getMinimumPrice()));
row.createCell(5).setCellValue(valueOf(item.getFirstPlace()));
row.createCell(6).setCellValue(valueOf(item.getSecondPlace()));
row.createCell(7).setCellValue(valueOf(item.getCartShopName()));
row.createCell(8).setCellValue(valueOf(item.getPriceChangeStatus()));
row.createCell(9).setCellValue(valueOf(item.getModifyCount()));
row.createCell(10).setCellValue(valueOf(item.getStatus()));
row.createCell(4).setCellValue(valueOf(item.getShippingFee()));
row.createCell(5).setCellValue(valueOf(item.getMinimumPrice()));
row.createCell(6).setCellValue(valueOf(item.getFirstPlace()));
row.createCell(7).setCellValue(valueOf(item.getSecondPlace()));
row.createCell(8).setCellValue(valueOf(item.getCartShopName()));
row.createCell(9).setCellValue(valueOf(item.getPriceChangeStatus()));
row.createCell(10).setCellValue(valueOf(item.getModifyCount()));
row.createCell(11).setCellValue(valueOf(item.getStatus()));
}
applyDefaultColumnWidths(sheet);
}
@@ -112,7 +114,7 @@ public class PriceTrackExcelAssemblyService {
}
private void applyDefaultColumnWidths(Sheet sheet) {
int[] widths = {22, 18, 12, 12, 12, 20, 20, 22, 18, 12, 12};
int[] widths = {22, 18, 12, 12, 12, 12, 20, 20, 22, 18, 12, 12};
for (int i = 0; i < widths.length; i++) {
sheet.setColumnWidth(i, widths[i] * 256);
}
@@ -6,6 +6,7 @@ import com.nanri.aiimage.modules.pricetrack.model.dto.PriceTrackSubmitResultRequ
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.service.TaskScopePayloadStorageService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
@@ -13,11 +14,13 @@ import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@Service
@RequiredArgsConstructor
@Slf4j
public class PriceTrackTaskCacheService {
private static final String MODULE_TYPE = "PRICE_TRACK";
@@ -32,17 +35,27 @@ public class PriceTrackTaskCacheService {
if (taskId == null || taskId <= 0) {
return;
}
stringRedisTemplate.opsForValue().set(
buildTaskHeartbeatKey(taskId),
String.valueOf(Instant.now().toEpochMilli()),
Duration.ofHours(HEARTBEAT_TTL_HOURS));
try {
stringRedisTemplate.opsForValue().set(
buildTaskHeartbeatKey(taskId),
String.valueOf(Instant.now().toEpochMilli()),
Duration.ofHours(HEARTBEAT_TTL_HOURS));
} catch (Exception ex) {
log.warn("[price-track-cache] touch heartbeat degraded taskId={} msg={}", taskId, ex.getMessage());
}
}
public long getTaskHeartbeatMillis(Long taskId) {
if (taskId == null || taskId <= 0) {
return 0L;
}
String raw = stringRedisTemplate.opsForValue().get(buildTaskHeartbeatKey(taskId));
String raw;
try {
raw = stringRedisTemplate.opsForValue().get(buildTaskHeartbeatKey(taskId));
} catch (Exception ex) {
log.warn("[price-track-cache] get heartbeat degraded taskId={} msg={}", taskId, ex.getMessage());
return 0L;
}
if (raw == null || raw.isBlank()) {
return 0L;
}
@@ -53,6 +66,41 @@ public class PriceTrackTaskCacheService {
}
}
public Map<Long, Long> getTaskHeartbeatMillisBatch(List<Long> taskIds) {
Map<Long, Long> result = new LinkedHashMap<>();
if (taskIds == null || taskIds.isEmpty()) {
return result;
}
List<Long> normalized = taskIds.stream()
.filter(id -> id != null && id > 0)
.distinct()
.toList();
if (normalized.isEmpty()) {
return result;
}
List<String> keys = normalized.stream().map(this::buildTaskHeartbeatKey).toList();
List<String> values;
try {
values = stringRedisTemplate.opsForValue().multiGet(keys);
} catch (Exception ex) {
log.warn("[price-track-cache] batch get heartbeat degraded taskIds={} msg={}", normalized, ex.getMessage());
return result;
}
for (int i = 0; i < normalized.size(); i++) {
String raw = values != null && i < values.size() ? values.get(i) : null;
if (raw == null || raw.isBlank()) {
result.put(normalized.get(i), 0L);
continue;
}
try {
result.put(normalized.get(i), Long.parseLong(raw));
} catch (NumberFormatException ignored) {
result.put(normalized.get(i), 0L);
}
}
return result;
}
public PriceTrackSubmitResultRequest.ShopResult getShopMergedPayload(Long taskId, String shopKey) {
return taskScopePayloadStorageService.getScopePayload(taskId, MODULE_TYPE, shopKey, PriceTrackSubmitResultRequest.ShopResult.class);
}
@@ -82,8 +130,12 @@ public class PriceTrackTaskCacheService {
return;
}
taskEntityLocalCache.remove(taskId);
stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId));
stringRedisTemplate.delete(buildTaskEntityKey(taskId));
try {
stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId));
stringRedisTemplate.delete(buildTaskEntityKey(taskId));
} catch (Exception ex) {
log.warn("[price-track-cache] delete cache degraded taskId={} msg={}", taskId, ex.getMessage());
}
taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE);
}
@@ -132,7 +184,13 @@ public class PriceTrackTaskCacheService {
return result;
}
java.util.List<String> keys = missingIds.stream().map(this::buildTaskEntityKey).toList();
java.util.List<String> values = stringRedisTemplate.opsForValue().multiGet(keys);
java.util.List<String> values;
try {
values = stringRedisTemplate.opsForValue().multiGet(keys);
} catch (Exception ex) {
log.warn("[price-track-cache] batch get task cache degraded taskIds={} msg={}", missingIds, ex.getMessage());
return result;
}
for (int i = 0; i < missingIds.size(); i++) {
Long taskId = missingIds.get(i);
String val = values != null && i < values.size() ? values.get(i) : null;
@@ -26,6 +26,9 @@ import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskResultPayloadService;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
import cn.hutool.core.util.IdUtil;
import lombok.RequiredArgsConstructor;
@@ -66,6 +69,8 @@ public class PriceTrackTaskService {
private final PriceTrackTaskCacheService priceTrackTaskCacheService;
private final PriceTrackLoopRunService priceTrackLoopRunService;
private final TaskPressureProperties taskPressureProperties;
private final TaskResultPayloadService taskResultPayloadService;
private final TaskFileJobService taskFileJobService;
private FileTaskEntity loadTaskForExecution(Long taskId) {
Map<Long, FileTaskEntity> cachedTasks = priceTrackTaskCacheService.getTaskCacheBatch(List.of(taskId));
@@ -435,7 +440,7 @@ public class PriceTrackTaskService {
continue;
}
try {
assembleShopResult(fr, shopKey, merged);
enqueueResultFileAssembly(fr, shopKey, merged);
priceTrackTaskCacheService.removeShopMergedPayload(taskId, shopKey);
} catch (Exception ex) {
markResultFailed(fr, ex.getMessage() == null ? "assemble price-track result failed" : ex.getMessage());
@@ -509,7 +514,7 @@ public class PriceTrackTaskService {
continue;
}
try {
assembleShopResult(fr, shopKey, cachedPayload);
enqueueResultFileAssembly(fr, shopKey, cachedPayload);
priceTrackTaskCacheService.removeShopMergedPayload(taskId, shopKey);
changed = true;
log.warn("[price-track] stale finalize assembled taskId={} shop={} fromCompensation={} completedFlag={} rowCount={}",
@@ -618,6 +623,49 @@ public class PriceTrackTaskService {
}
}
public void processResultFileJob(TaskFileJobEntity job) {
if (job == null || job.getTaskId() == null || job.getResultId() == null || job.getScopeKey() == null) {
throw new BusinessException("结果文件任务参数不完整");
}
FileResultEntity result = fileResultMapper.selectById(job.getResultId());
if (result == null || !MODULE_TYPE.equals(result.getModuleType())) {
throw new BusinessException("结果记录不存在");
}
PriceTrackSubmitResultRequest.ShopResult payload = taskResultPayloadService.getLatest(
job.getTaskId(), MODULE_TYPE, job.getScopeKey(), PriceTrackSubmitResultRequest.ShopResult.class);
if (payload == null) {
throw new BusinessException("结果文件载荷不存在");
}
assembleShopResult(result, job.getScopeKey(), payload);
}
private void enqueueResultFileAssembly(FileResultEntity result,
String shopKey,
PriceTrackSubmitResultRequest.ShopResult payload) {
taskResultPayloadService.saveLatest(result.getTaskId(), MODULE_TYPE, shopKey, payload);
markResultFilePending(result, shopKey, payload);
taskFileJobService.enqueueAssembleResult(result.getTaskId(), MODULE_TYPE, result.getId(), shopKey);
}
private void markResultFilePending(FileResultEntity result,
String shopKey,
PriceTrackSubmitResultRequest.ShopResult payload) {
Map<String, List<PriceTrackSubmitResultRequest.AsinResult>> countries =
excelAssemblyService.normalizeCountriesMap(payload.getCountries());
String displayName = payload.getShopName() != null && !payload.getShopName().isBlank()
? payload.getShopName().trim()
: shopKey;
String stem = safeFileStem(displayName);
result.setSuccess(1);
result.setErrorMessage(null);
result.setResultFilename(stem + ".xlsx");
result.setResultFileUrl(null);
result.setResultFileSize(0L);
result.setResultContentType(CONTENT_TYPE_XLSX);
result.setRowCount(excelAssemblyService.countRows(countries));
fileResultMapper.updateById(result);
}
public Map<String, List<Map<String, String>>> parseMatchAsinRowsByCountry(List<String> asinFiles, List<String> countryCodes) {
return parseAsinRowsByCountry(asinFiles, countryCodes);
}
@@ -901,8 +949,8 @@ public class PriceTrackTaskService {
vo.setSuccess(ok);
vo.setError(entity.getErrorMessage());
vo.setOutputFilename(entity.getResultFilename());
vo.setDownloadUrl(entity.getResultFileUrl() == null || entity.getResultFileUrl().isBlank()
? null : ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl()));
vo.setDownloadUrl(null);
attachFileJobState(vo, entity);
vo.setMatched(true);
if (entity.getTaskId() != null) {
mergeQueueFieldsFromRequest(vo, entity.getTaskId());
@@ -910,6 +958,18 @@ public class PriceTrackTaskService {
return vo;
}
private void attachFileJobState(PriceTrackResultItemVo vo, FileResultEntity entity) {
TaskFileJobEntity job = taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId());
vo.setFileReady(entity.getResultFileUrl() != null && !entity.getResultFileUrl().isBlank());
if (job == null) {
vo.setFileStatus(Boolean.TRUE.equals(vo.getFileReady()) ? "SUCCESS" : null);
return;
}
vo.setFileJobId(job.getId());
vo.setFileStatus(job.getStatus());
vo.setFileError(job.getErrorMessage());
}
private PriceTrackTaskDetailVo buildTaskDetail(FileTaskEntity task) {
PriceTrackTaskDetailVo detail = new PriceTrackTaskDetailVo();
detail.setTask(toTaskItemVo(task));
@@ -1074,6 +1134,7 @@ public class PriceTrackTaskService {
merged.setAsin(firstNonBlank(incoming.getAsin(), merged.getAsin()));
merged.setPrice(firstNonBlank(incoming.getPrice(), merged.getPrice()));
merged.setRecommendedPrice(firstNonBlank(incoming.getRecommendedPrice(), merged.getRecommendedPrice()));
merged.setShippingFee(firstNonBlank(incoming.getShippingFee(), merged.getShippingFee()));
merged.setMinimumPrice(firstNonBlank(incoming.getMinimumPrice(), merged.getMinimumPrice()));
merged.setFirstPlace(firstNonBlank(incoming.getFirstPlace(), merged.getFirstPlace()));
merged.setSecondPlace(firstNonBlank(incoming.getSecondPlace(), merged.getSecondPlace()));
@@ -1097,6 +1158,7 @@ public class PriceTrackTaskService {
out.setAsin(row.getAsin());
out.setPrice(row.getPrice());
out.setRecommendedPrice(row.getRecommendedPrice());
out.setShippingFee(row.getShippingFee());
out.setMinimumPrice(row.getMinimumPrice());
out.setFirstPlace(row.getFirstPlace());
out.setSecondPlace(row.getSecondPlace());
@@ -57,6 +57,10 @@ public class ProductRiskResultItemVo {
@JsonProperty("downloadUrl")
@Schema(description = "预签名下载 URL(列表里可能带;直链下载也可用 GET /results/{resultId}/download")
private String downloadUrl;
private Long fileJobId;
private String fileStatus;
private String fileError;
private Boolean fileReady;
@JsonProperty("scheduledAt")
@Schema(description = "定时执行时间,未设置时为空")
@@ -6,6 +6,7 @@ import com.nanri.aiimage.modules.productrisk.model.dto.ProductRiskShopPayloadDto
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.service.TaskScopePayloadStorageService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
@@ -13,11 +14,13 @@ import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@Service
@RequiredArgsConstructor
@Slf4j
public class ProductRiskTaskCacheService {
private static final String MODULE_TYPE = "PRODUCT_RISK_RESOLVE";
@@ -60,7 +63,13 @@ public class ProductRiskTaskCacheService {
if (taskId == null || taskId <= 0) {
return 0L;
}
String raw = stringRedisTemplate.opsForValue().get(buildTaskHeartbeatKey(taskId));
String raw;
try {
raw = stringRedisTemplate.opsForValue().get(buildTaskHeartbeatKey(taskId));
} catch (Exception ex) {
log.warn("[product-risk-cache] get heartbeat degraded taskId={} msg={}", taskId, ex.getMessage());
return 0L;
}
if (raw == null || raw.isBlank()) {
return 0L;
}
@@ -71,13 +80,52 @@ public class ProductRiskTaskCacheService {
}
}
public Map<Long, Long> getTaskHeartbeatMillisBatch(List<Long> taskIds) {
Map<Long, Long> result = new LinkedHashMap<>();
if (taskIds == null || taskIds.isEmpty()) {
return result;
}
List<Long> normalized = taskIds.stream()
.filter(id -> id != null && id > 0)
.distinct()
.toList();
if (normalized.isEmpty()) {
return result;
}
List<String> keys = normalized.stream().map(this::buildTaskHeartbeatKey).toList();
List<String> values;
try {
values = stringRedisTemplate.opsForValue().multiGet(keys);
} catch (Exception ex) {
log.warn("[product-risk-cache] batch get heartbeat degraded taskIds={} msg={}", normalized, ex.getMessage());
return result;
}
for (int i = 0; i < normalized.size(); i++) {
String raw = values != null && i < values.size() ? values.get(i) : null;
if (raw == null || raw.isBlank()) {
result.put(normalized.get(i), 0L);
continue;
}
try {
result.put(normalized.get(i), Long.parseLong(raw));
} catch (NumberFormatException ignored) {
result.put(normalized.get(i), 0L);
}
}
return result;
}
public void deleteTaskCache(Long taskId) {
if (taskId == null || taskId <= 0) {
return;
}
taskEntityLocalCache.remove(taskId);
stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId));
stringRedisTemplate.delete(buildTaskEntityKey(taskId));
try {
stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId));
stringRedisTemplate.delete(buildTaskEntityKey(taskId));
} catch (Exception ex) {
log.warn("[product-risk-cache] delete cache degraded taskId={} msg={}", taskId, ex.getMessage());
}
taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE);
}
@@ -126,7 +174,13 @@ public class ProductRiskTaskCacheService {
return result;
}
java.util.List<String> keys = missingIds.stream().map(this::buildTaskEntityKey).toList();
java.util.List<String> values = stringRedisTemplate.opsForValue().multiGet(keys);
java.util.List<String> values;
try {
values = stringRedisTemplate.opsForValue().multiGet(keys);
} catch (Exception ex) {
log.warn("[product-risk-cache] batch get task cache degraded taskIds={} msg={}", missingIds, ex.getMessage());
return result;
}
for (int i = 0; i < missingIds.size(); i++) {
Long taskId = missingIds.get(i);
String val = values != null && i < values.size() ? values.get(i) : null;
@@ -144,10 +198,14 @@ public class ProductRiskTaskCacheService {
}
public void touchTaskHeartbeat(Long taskId) {
stringRedisTemplate.opsForValue().set(
buildTaskHeartbeatKey(taskId),
String.valueOf(Instant.now().toEpochMilli()),
Duration.ofHours(PAYLOAD_TTL_HOURS));
try {
stringRedisTemplate.opsForValue().set(
buildTaskHeartbeatKey(taskId),
String.valueOf(Instant.now().toEpochMilli()),
Duration.ofHours(PAYLOAD_TTL_HOURS));
} catch (Exception ex) {
log.warn("[product-risk-cache] touch heartbeat degraded taskId={} msg={}", taskId, ex.getMessage());
}
}
private String buildTaskHeartbeatKey(Long taskId) {
@@ -24,6 +24,11 @@ import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import com.nanri.aiimage.modules.task.service.TaskResultItemService;
import com.nanri.aiimage.modules.task.service.TaskResultPayloadService;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.util.IdUtil;
@@ -59,6 +64,10 @@ public class ProductRiskTaskService {
private final ObjectMapper objectMapper;
private final ProductRiskTaskCacheService productRiskTaskCacheService;
private final TaskPressureProperties taskPressureProperties;
private final TaskResultPayloadService taskResultPayloadService;
private final TaskFileJobService taskFileJobService;
private final TaskResultItemService taskResultItemService;
private final TaskProgressSnapshotService taskProgressSnapshotService;
private FileTaskEntity loadTaskForExecution(Long taskId) {
Map<Long, FileTaskEntity> cachedTasks = productRiskTaskCacheService.getTaskCacheBatch(List.of(taskId));
@@ -553,7 +562,7 @@ public class ProductRiskTaskService {
}
try {
assembleShopResult(fr, shopKey, mergedPayload, workRoot);
enqueueResultFileAssembly(fr, shopKey, mergedPayload);
assembledCount++;
log.info("[product-risk] shop assembled taskId={} shop={} rowCount={} file={}",
taskId, shopKey, fr.getRowCount(), fr.getResultFilename());
@@ -698,7 +707,7 @@ public class ProductRiskTaskService {
}
try {
assembleShopResult(fr, shopKey, cachedPayload, workRoot);
enqueueResultFileAssembly(fr, shopKey, cachedPayload);
productRiskTaskCacheService.removeShopMergedPayload(taskId, shopKey);
changed = true;
log.warn("[product-risk] stale finalize assembled taskId={} shop={} fromCompensation={} completedFlag={} rowCount={}",
@@ -778,6 +787,49 @@ public class ProductRiskTaskService {
}
}
public void processResultFileJob(TaskFileJobEntity job) {
if (job == null || job.getTaskId() == null || job.getResultId() == null || job.getScopeKey() == null) {
throw new BusinessException("结果文件任务参数不完整");
}
FileResultEntity result = fileResultMapper.selectById(job.getResultId());
if (result == null || !MODULE_TYPE.equals(result.getModuleType())) {
throw new BusinessException("结果记录不存在");
}
ProductRiskShopPayloadDto payload = taskResultPayloadService.getLatest(
job.getTaskId(), MODULE_TYPE, job.getScopeKey(), ProductRiskShopPayloadDto.class);
if (payload == null) {
payload = taskResultItemService.getResultSnapshot(job.getTaskId(), MODULE_TYPE, job.getResultId(), ProductRiskShopPayloadDto.class);
}
if (payload == null) {
throw new BusinessException("结果文件载荷不存在");
}
File workRoot = FileUtil.mkdir(FileUtil.file(System.getProperty("java.io.tmpdir"), "product-risk-result", String.valueOf(job.getTaskId())));
assembleShopResult(result, job.getScopeKey(), payload, workRoot);
}
private void enqueueResultFileAssembly(FileResultEntity result, String shopKey, ProductRiskShopPayloadDto payload) {
taskResultPayloadService.saveLatest(result.getTaskId(), MODULE_TYPE, shopKey, payload);
taskResultItemService.replaceResultSnapshot(result.getTaskId(), MODULE_TYPE, result.getId(), shopKey, payload);
markResultFilePending(result, shopKey, payload);
taskFileJobService.enqueueAssembleResult(result.getTaskId(), MODULE_TYPE, result.getId(), shopKey);
}
private void markResultFilePending(FileResultEntity result, String shopKey, ProductRiskShopPayloadDto payload) {
Map<String, List<ProductRiskRowDto>> countries = excelAssemblyService.normalizeCountriesMap(payload.getCountries());
String displayName = payload.getShopName() != null && !payload.getShopName().isBlank()
? payload.getShopName().trim()
: shopKey;
String stem = safeFileStem(displayName);
result.setResultFilename(stem + ".zip");
result.setResultFileUrl(null);
result.setResultFileSize(0L);
result.setResultContentType(CONTENT_TYPE_ZIP);
result.setRowCount(excelAssemblyService.countRows(countries));
result.setSuccess(1);
result.setErrorMessage(null);
fileResultMapper.updateById(result);
}
private void updateTaskStatusFromLatestRows(FileTaskEntity task,
List<FileResultEntity> latest,
List<String> batchErrors) {
@@ -826,6 +878,8 @@ public class ProductRiskTaskService {
log.warn("[product-risk] compact result json failed: {}", ex.getMessage());
}
fileTaskMapper.updateById(task);
taskProgressSnapshotService.save(task.getId(), MODULE_TYPE, task.getStatus(),
latest.size(), ok, fail, null, task.getErrorMessage(), null);
log.warn("[product-risk] status updated from latest rows taskId={} oldStatus={} newStatus={} ok={} fail={} allDone={} batchErrors={} errorMessage={}",
task.getId(), oldStatus, task.getStatus(), ok, fail, allDone, batchErrors, task.getErrorMessage());
cleanupTaskCacheIfTerminal(task.getId(), task.getStatus());
@@ -910,9 +964,8 @@ public class ProductRiskTaskService {
vo.setSuccess(ok);
vo.setError(entity.getErrorMessage());
vo.setOutputFilename(entity.getResultFilename());
vo.setDownloadUrl(entity.getResultFileUrl() == null || entity.getResultFileUrl().isBlank()
? null
: ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl()));
vo.setDownloadUrl(null);
attachFileJobState(vo, entity);
vo.setMatched(true);
if (entity.getTaskId() != null) {
mergeQueueFieldsFromRequest(vo, entity.getTaskId());
@@ -920,6 +973,18 @@ public class ProductRiskTaskService {
return vo;
}
private void attachFileJobState(ProductRiskResultItemVo vo, FileResultEntity entity) {
TaskFileJobEntity job = taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId());
vo.setFileReady(entity.getResultFileUrl() != null && !entity.getResultFileUrl().isBlank());
if (job == null) {
vo.setFileStatus(Boolean.TRUE.equals(vo.getFileReady()) ? "SUCCESS" : null);
return;
}
vo.setFileJobId(job.getId());
vo.setFileStatus(job.getStatus());
vo.setFileError(job.getErrorMessage());
}
private void mergeQueueFieldsFromRequest(ProductRiskResultItemVo vo, Long taskId) {
FileTaskEntity t = loadTaskForExecution(taskId);
if (t == null || t.getRequestJson() == null || t.getRequestJson().isBlank()) {
@@ -43,6 +43,10 @@ public class QueryAsinResultItemVo {
private LocalDateTime finishedAt;
private String outputFilename;
private String downloadUrl;
private Long fileJobId;
private String fileStatus;
private String fileError;
private Boolean fileReady;
@JsonProperty("queryAsins")
private List<QueryAsinCountryAsinsDto> queryAsins = new ArrayList<>();
@@ -6,6 +6,7 @@ import com.nanri.aiimage.modules.queryasin.model.dto.QueryAsinShopPayloadDto;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.service.TaskScopePayloadStorageService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
@@ -18,6 +19,7 @@ import java.util.concurrent.ConcurrentHashMap;
@Service
@RequiredArgsConstructor
@Slf4j
public class QueryAsinTaskCacheService {
private static final String MODULE_TYPE = "QUERY_ASIN";
@@ -60,7 +62,13 @@ public class QueryAsinTaskCacheService {
if (taskId == null || taskId <= 0) {
return 0L;
}
String raw = stringRedisTemplate.opsForValue().get(buildTaskHeartbeatKey(taskId));
String raw;
try {
raw = stringRedisTemplate.opsForValue().get(buildTaskHeartbeatKey(taskId));
} catch (Exception ex) {
log.warn("[query-asin-cache] get heartbeat degraded taskId={} msg={}", taskId, ex.getMessage());
return 0L;
}
if (raw == null || raw.isBlank()) {
return 0L;
}
@@ -72,10 +80,14 @@ public class QueryAsinTaskCacheService {
}
public void touchTaskHeartbeat(Long taskId) {
stringRedisTemplate.opsForValue().set(
buildTaskHeartbeatKey(taskId),
String.valueOf(Instant.now().toEpochMilli()),
Duration.ofHours(PAYLOAD_TTL_HOURS));
try {
stringRedisTemplate.opsForValue().set(
buildTaskHeartbeatKey(taskId),
String.valueOf(Instant.now().toEpochMilli()),
Duration.ofHours(PAYLOAD_TTL_HOURS));
} catch (Exception ex) {
log.warn("[query-asin-cache] touch heartbeat degraded taskId={} msg={}", taskId, ex.getMessage());
}
}
public void deleteTaskCache(Long taskId) {
@@ -83,8 +95,12 @@ public class QueryAsinTaskCacheService {
return;
}
taskEntityLocalCache.remove(taskId);
stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId));
stringRedisTemplate.delete(buildTaskEntityKey(taskId));
try {
stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId));
stringRedisTemplate.delete(buildTaskEntityKey(taskId));
} catch (Exception ex) {
log.warn("[query-asin-cache] delete cache degraded taskId={} msg={}", taskId, ex.getMessage());
}
taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE);
}
@@ -133,7 +149,13 @@ public class QueryAsinTaskCacheService {
return result;
}
java.util.List<String> keys = missingIds.stream().map(this::buildTaskEntityKey).toList();
java.util.List<String> values = stringRedisTemplate.opsForValue().multiGet(keys);
java.util.List<String> values;
try {
values = stringRedisTemplate.opsForValue().multiGet(keys);
} catch (Exception ex) {
log.warn("[query-asin-cache] batch get task cache degraded taskIds={} msg={}", missingIds, ex.getMessage());
return result;
}
for (int i = 0; i < missingIds.size(); i++) {
Long taskId = missingIds.get(i);
String val = values != null && i < values.size() ? values.get(i) : null;
@@ -24,6 +24,10 @@ import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import com.nanri.aiimage.modules.task.service.TaskResultItemService;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@@ -45,6 +49,9 @@ public class QueryAsinTaskService {
private static final String MODULE_TYPE = "QUERY_ASIN";
private static final String CONTENT_TYPE_XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
private static final int RESULT_PENDING = -1;
private static final int RESULT_FAILED = 0;
private static final int RESULT_SUCCESS = 1;
private static final String INTERRUPTED_MESSAGE = "Python 在该店铺结果提交完成前中断";
private final FileTaskMapper fileTaskMapper;
@@ -56,6 +63,9 @@ public class QueryAsinTaskService {
private final ZiniaoShopSwitchService ziniaoShopSwitchService;
private final ObjectMapper objectMapper;
private final TaskPressureProperties taskPressureProperties;
private final TaskFileJobService taskFileJobService;
private final TaskResultItemService taskResultItemService;
private final TaskProgressSnapshotService taskProgressSnapshotService;
private FileTaskEntity loadTaskForExecution(Long taskId) {
Map<Long, FileTaskEntity> cachedTasks = taskCacheService.getTaskCacheBatch(List.of(taskId));
@@ -185,8 +195,7 @@ public class QueryAsinTaskService {
batch.getMissingTaskIds().add(taskId);
continue;
}
List<QueryAsinResultItemVo> snapshots = buildSnapshotFromDb(task, taskRows);
batch.getItems().addAll(snapshots);
batch.getItems().addAll(buildProgressItems(task, taskRows));
}
return batch;
}
@@ -264,12 +273,13 @@ public class QueryAsinTaskService {
result.setSourceFilename(normalizedShopName);
result.setSourceFileUrl(item.getShopId());
result.setUserId(request.getUserId());
result.setSuccess(null);
result.setSuccess(RESULT_PENDING);
result.setCreatedAt(now);
fileResultMapper.insert(result);
snapshots.add(toSnapshotVo(result, item, task.getStatus(), null));
}
persistTaskJson(task, uniqueItems, snapshots);
taskCacheService.saveTaskCache(task);
QueryAsinCreateTaskVo vo = new QueryAsinCreateTaskVo();
vo.setTaskId(task.getId());
@@ -326,6 +336,7 @@ public class QueryAsinTaskService {
persistSnapshotJson(task, snapshots);
fileTaskMapper.updateById(task);
taskCacheService.saveTaskCache(task);
tryFinalizeTask(taskId, false);
}
@@ -402,6 +413,7 @@ public class QueryAsinTaskService {
persistSnapshotJson(task, snapshots);
fileTaskMapper.updateById(task);
taskCacheService.saveTaskCache(task);
return changed;
}
@@ -448,6 +460,7 @@ public class QueryAsinTaskService {
updateTaskStatusFromRows(task, rows);
persistSnapshotJson(task, buildSnapshotFromDb(task, rows));
fileTaskMapper.updateById(task);
taskCacheService.saveTaskCache(task);
}
private long countTasks(Long userId, List<String> statuses) {
@@ -502,7 +515,11 @@ public class QueryAsinTaskService {
private Map<Long, Map<Long, QueryAsinResultItemVo>> buildSnapshotMap(Map<Long, FileTaskEntity> taskMap) {
Map<Long, Map<Long, QueryAsinResultItemVo>> out = new LinkedHashMap<>();
for (Map.Entry<Long, FileTaskEntity> entry : taskMap.entrySet()) {
out.put(entry.getKey(), indexSnapshotByResultId(parseTaskSnapshots(entry.getValue().getResultJson())));
List<QueryAsinResultItemVo> snapshots = parseTaskSnapshots(entry.getValue().getResultJson());
if (snapshots.isEmpty()) {
snapshots = taskResultItemService.listResultSnapshots(entry.getKey(), MODULE_TYPE, QueryAsinResultItemVo.class);
}
out.put(entry.getKey(), indexSnapshotByResultId(snapshots));
}
return out;
}
@@ -514,14 +531,13 @@ public class QueryAsinTaskService {
item.setShopName(firstNonBlank(item.getShopName(), entity.getSourceFilename()));
item.setShopId(firstNonBlank(item.getShopId(), entity.getSourceFileUrl()));
item.setTaskStatus(task != null ? task.getStatus() : item.getTaskStatus());
item.setSuccess(entity.getSuccess() == null ? item.getSuccess() : entity.getSuccess() == 1);
item.setSuccess(toSuccessFlag(entity.getSuccess(), item.getSuccess()));
item.setError(!blank(entity.getErrorMessage()) ? entity.getErrorMessage() : item.getError());
item.setCreatedAt(entity.getCreatedAt());
item.setFinishedAt(task != null ? task.getFinishedAt() : item.getFinishedAt());
item.setOutputFilename(firstNonBlank(item.getOutputFilename(), entity.getResultFilename()));
item.setDownloadUrl(blank(entity.getResultFileUrl())
? item.getDownloadUrl()
: ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl()));
item.setDownloadUrl(null);
attachFileJobState(item, entity);
if (item.getCountryResults() == null) {
item.setCountryResults(new ArrayList<>());
}
@@ -531,6 +547,18 @@ public class QueryAsinTaskService {
return item;
}
private void attachFileJobState(QueryAsinResultItemVo item, FileResultEntity entity) {
TaskFileJobEntity job = taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId());
item.setFileReady(!blank(entity.getResultFileUrl()));
if (job == null) {
item.setFileStatus(Boolean.TRUE.equals(item.getFileReady()) ? "SUCCESS" : null);
return;
}
item.setFileJobId(job.getId());
item.setFileStatus(job.getStatus());
item.setFileError(job.getErrorMessage());
}
private List<QueryAsinTaskItemDto> dedupeItems(List<QueryAsinTaskItemDto> items) {
LinkedHashMap<String, QueryAsinTaskItemDto> map = new LinkedHashMap<>();
for (QueryAsinTaskItemDto item : items) {
@@ -560,7 +588,7 @@ public class QueryAsinTaskService {
vo.setMatchStatus(item.getMatchStatus());
vo.setMatchMessage(item.getMatchMessage());
vo.setTaskStatus(taskStatus);
vo.setSuccess(result.getSuccess() == null ? null : result.getSuccess() == 1);
vo.setSuccess(toSuccessFlag(result.getSuccess(), null));
vo.setError(result.getErrorMessage());
vo.setCreatedAt(result.getCreatedAt());
vo.setFinishedAt(finishedAt);
@@ -575,6 +603,7 @@ public class QueryAsinTaskService {
try {
task.setRequestJson(objectMapper.writeValueAsString(requestItems));
task.setResultJson(objectMapper.writeValueAsString(snapshots));
syncSnapshotTables(task, snapshots);
fileTaskMapper.updateById(task);
} catch (Exception ex) {
throw new BusinessException("查询ASIN任务快照保存失败");
@@ -591,6 +620,27 @@ public class QueryAsinTaskService {
return list;
}
private List<QueryAsinResultItemVo> buildProgressItems(FileTaskEntity task, List<FileResultEntity> rows) {
List<QueryAsinResultItemVo> list = new ArrayList<>();
for (FileResultEntity row : rows) {
QueryAsinResultItemVo item = new QueryAsinResultItemVo();
item.setResultId(row.getId());
item.setTaskId(row.getTaskId());
item.setShopName(row.getSourceFilename());
item.setShopId(row.getSourceFileUrl());
item.setTaskStatus(task == null ? null : task.getStatus());
item.setSuccess(toSuccessFlag(row.getSuccess(), null));
item.setError(row.getErrorMessage());
item.setCreatedAt(row.getCreatedAt());
item.setFinishedAt(task == null ? null : task.getFinishedAt());
item.setOutputFilename(row.getResultFilename());
item.setDownloadUrl(null);
attachFileJobState(item, row);
list.add(item);
}
return list;
}
private Map<Long, QueryAsinResultItemVo> indexSnapshotByResultId(List<QueryAsinResultItemVo> snapshots) {
Map<Long, QueryAsinResultItemVo> map = new LinkedHashMap<>();
if (snapshots == null) {
@@ -605,8 +655,8 @@ public class QueryAsinTaskService {
}
private void updateTaskStatusFromRows(FileTaskEntity task, List<FileResultEntity> rows) {
long successCount = rows.stream().filter(row -> Integer.valueOf(1).equals(row.getSuccess())).count();
long failedCount = rows.stream().filter(row -> Integer.valueOf(0).equals(row.getSuccess())).count();
long successCount = rows.stream().filter(row -> Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess())).count();
long failedCount = rows.stream().filter(row -> Integer.valueOf(RESULT_FAILED).equals(row.getSuccess())).count();
long pendingCount = rows.stream().filter(row -> !isResultFinished(row)).count();
task.setSuccessFileCount((int) successCount);
task.setFailedFileCount((int) failedCount);
@@ -773,27 +823,25 @@ public class QueryAsinTaskService {
.filter(item -> Boolean.TRUE.equals(item.getSuccess()))
.toList();
if (!successItems.isEmpty()) {
File workRoot = FileUtil.mkdir(FileUtil.file(System.getProperty("java.io.tmpdir"), "query-asin-result", String.valueOf(task.getId())));
String filename = safeFileStem("查询ASIN-" + task.getId()) + ".xlsx";
File xlsx = FileUtil.file(workRoot, filename);
try {
excelAssemblyService.writeWorkbook(xlsx, successItems);
String objectKey = ossStorageService.uploadResultFile(xlsx, MODULE_TYPE);
long fileSize = xlsx.length();
int rowCount = excelAssemblyService.countRows(successItems);
String filename = buildTaskWorkbookFilename(task);
int rowCount = excelAssemblyService.countRows(successItems);
FileResultEntity firstSuccessRow = null;
for (FileResultEntity row : rows) {
if (Integer.valueOf(1).equals(row.getSuccess())) {
if (Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess())) {
row.setResultFilename(filename);
row.setResultFileUrl(objectKey);
row.setResultFileSize(fileSize);
row.setResultFileUrl(null);
row.setResultFileSize(0L);
row.setResultContentType(CONTENT_TYPE_XLSX);
row.setRowCount(rowCount);
fileResultMapper.updateById(row);
if (firstSuccessRow == null) {
firstSuccessRow = row;
}
}
}
if (firstSuccessRow != null) {
taskFileJobService.enqueueAssembleResult(task.getId(), MODULE_TYPE, firstSuccessRow.getId(), "task:" + task.getId());
snapshots = buildSnapshotFromDb(task, rows);
} finally {
FileUtil.del(xlsx);
}
}
@@ -802,14 +850,56 @@ public class QueryAsinTaskService {
taskCacheService.deleteTaskCache(task.getId());
}
public void processResultFileJob(TaskFileJobEntity job) {
if (job == null || job.getTaskId() == null) {
throw new BusinessException("结果文件任务参数不完整");
}
FileTaskEntity task = fileTaskMapper.selectById(job.getTaskId());
if (task == null || !MODULE_TYPE.equals(task.getModuleType())) {
throw new BusinessException("任务不存在");
}
List<FileResultEntity> rows = listTaskRows(job.getTaskId());
List<QueryAsinResultItemVo> snapshots = taskResultItemService.listResultSnapshots(job.getTaskId(), MODULE_TYPE, QueryAsinResultItemVo.class);
if (snapshots.isEmpty()) {
snapshots = buildSnapshotFromDb(task, rows);
}
List<QueryAsinResultItemVo> successItems = snapshots.stream()
.filter(item -> Boolean.TRUE.equals(item.getSuccess()))
.toList();
if (successItems.isEmpty()) {
throw new BusinessException("没有可生成的查询ASIN结果");
}
File workRoot = FileUtil.mkdir(FileUtil.file(System.getProperty("java.io.tmpdir"), "query-asin-result", String.valueOf(task.getId())));
String filename = buildTaskWorkbookFilename(task);
File xlsx = FileUtil.file(workRoot, filename);
try {
excelAssemblyService.writeWorkbook(xlsx, successItems);
String objectKey = ossStorageService.uploadResultFile(xlsx, MODULE_TYPE);
long fileSize = xlsx.length();
int rowCount = excelAssemblyService.countRows(successItems);
for (FileResultEntity row : rows) {
if (Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess())) {
row.setResultFilename(filename);
row.setResultFileUrl(objectKey);
row.setResultFileSize(fileSize);
row.setResultContentType(CONTENT_TYPE_XLSX);
row.setRowCount(rowCount);
fileResultMapper.updateById(row);
}
}
} finally {
FileUtil.del(xlsx);
}
}
private void markResultSuccess(FileResultEntity row) {
row.setSuccess(1);
row.setSuccess(RESULT_SUCCESS);
row.setErrorMessage(null);
fileResultMapper.updateById(row);
}
private void markResultFailed(FileResultEntity row, String message) {
row.setSuccess(0);
row.setSuccess(RESULT_FAILED);
row.setErrorMessage(blankToNull(message));
row.setResultFilename(null);
row.setResultFileUrl(null);
@@ -820,7 +910,14 @@ public class QueryAsinTaskService {
}
private boolean isResultFinished(FileResultEntity row) {
return Integer.valueOf(1).equals(row.getSuccess()) || Integer.valueOf(0).equals(row.getSuccess());
return Integer.valueOf(RESULT_SUCCESS).equals(row.getSuccess()) || Integer.valueOf(RESULT_FAILED).equals(row.getSuccess());
}
private Boolean toSuccessFlag(Integer dbValue, Boolean fallback) {
if (dbValue == null || Integer.valueOf(RESULT_PENDING).equals(dbValue)) {
return fallback;
}
return Integer.valueOf(RESULT_SUCCESS).equals(dbValue);
}
private List<QueryAsinResultItemVo> parseTaskSnapshots(String json) {
@@ -838,11 +935,39 @@ public class QueryAsinTaskService {
private void persistSnapshotJson(FileTaskEntity task, List<QueryAsinResultItemVo> snapshots) {
try {
task.setResultJson(objectMapper.writeValueAsString(snapshots == null ? List.of() : snapshots));
syncSnapshotTables(task, snapshots);
} catch (Exception ex) {
throw new BusinessException("查询ASIN任务快照保存失败");
}
}
private void syncSnapshotTables(FileTaskEntity task, List<QueryAsinResultItemVo> snapshots) {
List<QueryAsinResultItemVo> safe = snapshots == null ? List.of() : snapshots;
taskResultItemService.replaceTaskSnapshots(task.getId(), MODULE_TYPE, safe, new TaskResultItemService.SnapshotKeyResolver() {
@Override
public Long resultId(Object snapshot) {
return ((QueryAsinResultItemVo) snapshot).getResultId();
}
@Override
public String scopeKey(Object snapshot) {
QueryAsinResultItemVo item = (QueryAsinResultItemVo) snapshot;
return firstNonBlank(item.getShopName(), "result:" + item.getResultId());
}
});
int successCount = 0;
int failedCount = 0;
for (QueryAsinResultItemVo item : safe) {
if (Boolean.TRUE.equals(item.getSuccess())) {
successCount++;
} else if (Boolean.FALSE.equals(item.getSuccess())) {
failedCount++;
}
}
taskProgressSnapshotService.save(task.getId(), MODULE_TYPE, firstNonBlank(task.getStatus(), "RUNNING"),
safe.size(), successCount, failedCount, null, task.getErrorMessage(), null);
}
private List<QueryAsinCountryResultDto> copyCountryResults(List<QueryAsinCountryResultDto> results) {
List<QueryAsinCountryResultDto> copy = new ArrayList<>();
if (results == null) {
@@ -923,6 +1048,11 @@ public class QueryAsinTaskService {
return asin == null ? "" : asin.trim().toUpperCase();
}
private String buildTaskWorkbookFilename(FileTaskEntity task) {
Long taskId = task == null ? null : task.getId();
return safeFileStem("查询ASIN-" + (taskId == null ? "result" : taskId)) + ".xlsx";
}
private String safeFileStem(String value) {
String raw = value == null ? "result" : value.trim();
String safe = raw.replaceAll("[\\\\/:*?\"<>|]+", "_");
@@ -15,6 +15,7 @@ import com.nanri.aiimage.modules.shopkey.model.vo.QueryAsinItemVo;
import com.nanri.aiimage.modules.shopkey.model.vo.QueryAsinPageVo;
import com.nanri.aiimage.modules.shopkey.model.vo.ShopManageGroupItemVo;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.multipart.MultipartFile;
@@ -30,9 +31,11 @@ import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Predicate;
@Service
@RequiredArgsConstructor
@Slf4j
public class QueryAsinService {
private static final List<String> SUPPORTED_COUNTRIES = List.of("DE", "UK", "FR", "IT", "ES");
@@ -364,13 +367,32 @@ public class QueryAsinService {
}
private String normalizeCountry(String country) {
String normalized = normalizeBlank(country).toUpperCase(Locale.ROOT);
String normalized = normalizeCountryAlias(country);
if (!SUPPORTED_COUNTRIES.contains(normalized)) {
throw new BusinessException("国家参数不支持: " + country);
}
return normalized;
}
private String normalizeCountryAlias(String country) {
String normalized = normalizeBlank(country);
if (normalized.isEmpty()) {
return "";
}
String upper = normalized.toUpperCase(Locale.ROOT);
if (SUPPORTED_COUNTRIES.contains(upper)) {
return upper;
}
return switch (normalizeHeaderKey(normalized)) {
case "\u5fb7\u56fd", "\u5fb7", "de", "germany", "german", "deutschland" -> "DE";
case "\u82f1\u56fd", "\u82f1", "uk", "gb", "greatbritain", "britain", "unitedkingdom" -> "UK";
case "\u6cd5\u56fd", "\u6cd5", "fr", "france", "french" -> "FR";
case "\u610f\u5927\u5229", "\u610f", "it", "italy", "italian" -> "IT";
case "\u897f\u73ed\u7259", "\u897f", "es", "spain", "spanish", "espana" -> "ES";
default -> "";
};
}
private String normalizeAsin(String asin) {
String normalized = normalizeRequired(asin, "ASIN 不能为空").toUpperCase(Locale.ROOT);
if (normalized.length() > 64) {
@@ -505,13 +527,19 @@ public class QueryAsinService {
}
}
headerLoaded = true;
if (findGroupIndex() == null && (fallbackGroupId == null || fallbackGroupId <= 0)) {
Integer resolvedGroupIndex = resolveGroupHeaderIndex();
Integer resolvedShopIndex = resolveShopHeaderIndex();
boolean hasWideCountries = hasWideCountryColumns();
boolean hasSingleCountryAsin = hasSingleCountryAsinColumns();
log.info("[query-asin-import] headerMap={} normalizedKeys={} groupIndex={} shopIndex={} hasWideCountries={} hasSingleCountryAsin={}",
headerMap, headerIndexByKey.keySet(), resolvedGroupIndex, resolvedShopIndex, hasWideCountries, hasSingleCountryAsin);
if (resolvedGroupIndex == null && (fallbackGroupId == null || fallbackGroupId <= 0)) {
throw new BusinessException("Excel 缺少分组列,请在页面选择分组或提供分组列");
}
if (findShopIndex() == null) {
if (resolvedShopIndex == null) {
throw new BusinessException("Excel 缺少店铺名列");
}
if (SUPPORTED_COUNTRIES.stream().noneMatch(country -> findCountryIndex(country) != null)) {
if (!hasWideCountries && !hasSingleCountryAsin) {
throw new BusinessException("Excel 至少需要包含一个国家 ASIN 列");
}
}
@@ -524,7 +552,7 @@ public class QueryAsinService {
progress.setProcessedRows(rowIndex);
Long groupId = resolveGroupId(rowMap);
String shopName = cell(rowMap, findShopIndex());
String shopName = cell(rowMap, resolveShopHeaderIndex());
if (groupId == null || shopName.isEmpty()) {
skippedCount++;
updateProgress();
@@ -532,27 +560,8 @@ public class QueryAsinService {
}
QueryAsinEntity row = newImportEntity(groupId, shopName);
int rowAsinCount = 0;
int rowAcceptedCount = 0;
for (String country : SUPPORTED_COUNTRIES) {
Integer index = findCountryIndex(country);
if (index == null) {
continue;
}
String asin = normalizeBlank(cell(rowMap, index)).toUpperCase(Locale.ROOT);
if (asin.isEmpty()) {
continue;
}
if (asin.length() > 64) {
skippedCount++;
continue;
}
asinCount++;
rowAsinCount++;
setCountryAsin(row, country, asin);
rowAcceptedCount++;
}
if (rowAsinCount == 0) {
int rowAcceptedCount = collectRowCountryAsins(row, rowMap);
if (countCountryCells(row) == 0) {
skippedCount++;
}
if (rowAcceptedCount > 0) {
@@ -754,7 +763,7 @@ public class QueryAsinService {
}
private Long resolveGroupId(Map<Integer, String> rowMap) {
Integer groupIndex = findGroupIndex();
Integer groupIndex = resolveGroupHeaderIndex();
String groupText = cell(rowMap, groupIndex);
if (groupText.isEmpty()) {
return fallbackGroupId != null && fallbackGroupId > 0 ? fallbackGroupId : null;
@@ -767,6 +776,51 @@ public class QueryAsinService {
return id;
}
private int collectRowCountryAsins(QueryAsinEntity row, Map<Integer, String> rowMap) {
int acceptedCount = 0;
for (String country : SUPPORTED_COUNTRIES) {
Integer index = resolveCountryHeaderIndex(country);
if (index == null) {
continue;
}
acceptedCount += acceptCountryAsin(row, country, cell(rowMap, index));
}
Integer singleCountryIndex = findSingleCountryIndex();
Integer singleAsinIndex = findSingleAsinIndex();
if (singleCountryIndex != null && singleAsinIndex != null) {
acceptedCount += acceptCountryAsin(
row,
normalizeCountryAlias(cell(rowMap, singleCountryIndex)),
cell(rowMap, singleAsinIndex));
}
return acceptedCount;
}
private int acceptCountryAsin(QueryAsinEntity row, String country, String asin) {
String normalizedAsin = normalizeBlank(asin).toUpperCase(Locale.ROOT);
if (normalizedAsin.isEmpty()) {
return 0;
}
asinCount++;
if (country == null || country.isEmpty() || !SUPPORTED_COUNTRIES.contains(country)) {
skippedCount++;
return 0;
}
if (normalizedAsin.length() > 64) {
skippedCount++;
return 0;
}
String existing = getCountryAsin(row, country);
if (!existing.isEmpty()) {
if (!existing.equals(normalizedAsin)) {
skippedCount++;
}
return 0;
}
setCountryAsin(row, country, normalizedAsin);
return 1;
}
private void loadGroups() {
if (groupsLoaded) {
return;
@@ -801,6 +855,88 @@ public class QueryAsinService {
};
}
private Integer resolveGroupHeaderIndex() {
Integer index = findGroupIndex();
if (index != null) {
return index;
}
index = firstHeader("groupname", "\u5206\u7ec4id", "\u5206\u7ec4");
if (index != null) {
return index;
}
return findHeaderByPredicate(key -> key.contains("\u5206\u7ec4") || key.contains("group"));
}
private Integer resolveShopHeaderIndex() {
Integer index = findShopIndex();
if (index != null) {
return index;
}
index = firstHeader(
"\u5e97\u94fa\u540d", "\u5e97\u94fa\u540d\u79f0", "\u5e97\u94fa\u8d26\u53f7", "\u8d26\u53f7",
"account", "selleraccount", "storeaccount", "storename");
if (index != null) {
return index;
}
return findHeaderByPredicate(key ->
(key.contains("\u5e97\u94fa") && (key.contains("\u540d") || key.contains("\u8d26\u53f7")))
|| (key.contains("shop") && (key.contains("name") || key.contains("account")))
|| key.contains("selleraccount")
|| key.contains("storeaccount"));
}
private Integer resolveCountryHeaderIndex(String country) {
Integer index = findCountryIndex(country);
if (index != null) {
return index;
}
return switch (country) {
case "DE" -> firstHeader("\u5fb7\u56fd", "\u5fb7", "\u5fb7\u56fdasin");
case "UK" -> firstHeader("\u82f1\u56fd", "\u82f1", "\u82f1\u56fdasin");
case "FR" -> firstHeader("\u6cd5\u56fd", "\u6cd5", "\u6cd5\u56fdasin");
case "IT" -> firstHeader("\u610f\u5927\u5229", "\u610f", "\u610f\u5927\u5229asin");
case "ES" -> firstHeader("\u897f\u73ed\u7259", "\u897f", "\u897f\u73ed\u7259asin");
default -> null;
};
}
private Integer findSingleCountryIndex() {
Integer index = firstHeader(
"\u4e2d\u6587\u56fd\u5bb6\u540d", "\u56fd\u5bb6", "\u56fd\u5bb6\u540d", "\u56fd\u5bb6\u7ad9\u70b9",
"\u7ad9\u70b9", "country", "countryname", "marketplace", "site");
if (index != null) {
return index;
}
return findHeaderByPredicate(key ->
key.contains("\u56fd\u5bb6")
|| key.contains("\u7ad9\u70b9")
|| key.contains("country")
|| key.contains("marketplace")
|| key.equals("site"));
}
private Integer findSingleAsinIndex() {
return firstHeader("asin", "\u67e5\u8be2asin", "\u4ea7\u54c1asin", "\u94fe\u63a5asin");
}
private boolean hasWideCountryColumns() {
return SUPPORTED_COUNTRIES.stream().anyMatch(country -> resolveCountryHeaderIndex(country) != null);
}
private boolean hasSingleCountryAsinColumns() {
return findSingleCountryIndex() != null && findSingleAsinIndex() != null;
}
private Integer findHeaderByPredicate(Predicate<String> predicate) {
for (Map.Entry<String, Integer> entry : headerIndexByKey.entrySet()) {
String key = entry.getKey();
if (key != null && predicate.test(key)) {
return entry.getValue();
}
}
return null;
}
private Integer firstHeader(String... names) {
for (String name : names) {
Integer index = headerIndexByKey.get(normalizeHeaderKey(name));
@@ -81,13 +81,32 @@ public class ShopManageGroupService {
.map(ShopManageGroupEntity::getId)
.filter(id -> id != null && id > 0)
.collect(Collectors.toCollection(LinkedHashSet::new));
groupIds.addAll(groupMemberMapper.selectList(new LambdaQueryWrapper<ShopManageGroupMemberEntity>()
List<Long> memberGroupIds = groupMemberMapper.selectList(new LambdaQueryWrapper<ShopManageGroupMemberEntity>()
.select(ShopManageGroupMemberEntity::getGroupId)
.eq(ShopManageGroupMemberEntity::getUserId, normalizedOperatorId))
.stream()
.map(ShopManageGroupMemberEntity::getGroupId)
.filter(id -> id != null && id > 0)
.collect(Collectors.toCollection(LinkedHashSet::new)));
.toList();
groupIds.addAll(memberGroupIds);
if (!memberGroupIds.isEmpty()) {
LinkedHashSet<Long> leaderIds = groupMapper.selectList(new LambdaQueryWrapper<ShopManageGroupEntity>()
.select(ShopManageGroupEntity::getCreatedById)
.in(ShopManageGroupEntity::getId, memberGroupIds))
.stream()
.map(ShopManageGroupEntity::getCreatedById)
.filter(id -> id != null && id > 0)
.collect(Collectors.toCollection(LinkedHashSet::new));
if (!leaderIds.isEmpty()) {
groupIds.addAll(groupMapper.selectList(new LambdaQueryWrapper<ShopManageGroupEntity>()
.select(ShopManageGroupEntity::getId)
.in(ShopManageGroupEntity::getCreatedById, leaderIds))
.stream()
.map(ShopManageGroupEntity::getId)
.filter(id -> id != null && id > 0)
.toList());
}
}
return groupIds;
}
@@ -178,9 +197,7 @@ public class ShopManageGroupService {
return;
}
Long normalizedOperatorId = normalizePositiveId(operatorId, "操作人不能为空");
boolean createdBySelf = entity.getCreatedById() != null && entity.getCreatedById().equals(normalizedOperatorId);
boolean memberOfGroup = isGroupMember(entity.getId(), normalizedOperatorId);
if (!createdBySelf && !memberOfGroup) {
if (!listAccessibleGroupIds(normalizedOperatorId, false).contains(entity.getId())) {
throw new BusinessException("只能操作自己有权限的分组数据");
}
}
@@ -18,6 +18,7 @@ import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@@ -38,17 +39,27 @@ public class ShopMatchTaskCacheService {
if (taskId == null || taskId <= 0) {
return;
}
stringRedisTemplate.opsForValue().set(
buildTaskHeartbeatKey(taskId),
String.valueOf(Instant.now().toEpochMilli()),
Duration.ofHours(PAYLOAD_TTL_HOURS));
try {
stringRedisTemplate.opsForValue().set(
buildTaskHeartbeatKey(taskId),
String.valueOf(Instant.now().toEpochMilli()),
Duration.ofHours(PAYLOAD_TTL_HOURS));
} catch (Exception ex) {
log.warn("[shop-match-cache] touch heartbeat degraded taskId={} msg={}", taskId, ex.getMessage());
}
}
public long getTaskHeartbeatMillis(Long taskId) {
if (taskId == null || taskId <= 0) {
return 0L;
}
String raw = stringRedisTemplate.opsForValue().get(buildTaskHeartbeatKey(taskId));
String raw;
try {
raw = stringRedisTemplate.opsForValue().get(buildTaskHeartbeatKey(taskId));
} catch (Exception ex) {
log.warn("[shop-match-cache] get heartbeat degraded taskId={} msg={}", taskId, ex.getMessage());
return 0L;
}
if (raw == null || raw.isBlank()) {
return 0L;
}
@@ -59,6 +70,41 @@ public class ShopMatchTaskCacheService {
}
}
public Map<Long, Long> getTaskHeartbeatMillisBatch(List<Long> taskIds) {
Map<Long, Long> result = new LinkedHashMap<>();
if (taskIds == null || taskIds.isEmpty()) {
return result;
}
List<Long> normalized = taskIds.stream()
.filter(id -> id != null && id > 0)
.distinct()
.toList();
if (normalized.isEmpty()) {
return result;
}
List<String> keys = normalized.stream().map(this::buildTaskHeartbeatKey).toList();
List<String> values;
try {
values = stringRedisTemplate.opsForValue().multiGet(keys);
} catch (Exception ex) {
log.warn("[shop-match-cache] batch get heartbeat degraded taskIds={} msg={}", normalized, ex.getMessage());
return result;
}
for (int i = 0; i < normalized.size(); i++) {
String raw = values != null && i < values.size() ? values.get(i) : null;
if (raw == null || raw.isBlank()) {
result.put(normalized.get(i), 0L);
continue;
}
try {
result.put(normalized.get(i), Long.parseLong(raw));
} catch (NumberFormatException ignored) {
result.put(normalized.get(i), 0L);
}
}
return result;
}
public ShopMatchShopPayloadDto getShopMergedPayload(Long taskId, String shopKey) {
return taskScopePayloadStorageService.getScopePayload(taskId, MODULE_TYPE, shopKey, ShopMatchShopPayloadDto.class);
}
@@ -88,7 +134,11 @@ public class ShopMatchTaskCacheService {
return;
}
taskEntityLocalCache.remove(taskId);
stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId));
try {
stringRedisTemplate.delete(buildTaskHeartbeatKey(taskId));
} catch (Exception ex) {
log.warn("[shop-match-cache] delete heartbeat degraded taskId={} msg={}", taskId, ex.getMessage());
}
taskScopePayloadStorageService.deleteTaskScopePayloads(taskId, MODULE_TYPE);
try {
Path taskDir = buildTaskDir(taskId);
@@ -28,6 +28,9 @@ import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import com.nanri.aiimage.modules.task.service.TaskResultPayloadService;
import com.nanri.aiimage.modules.ziniao.service.ZiniaoShopSwitchService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@@ -62,6 +65,8 @@ public class ShopMatchTaskService {
private final ObjectMapper objectMapper;
private final ShopMatchTaskCacheService shopMatchTaskCacheService;
private final TaskPressureProperties taskPressureProperties;
private final TaskResultPayloadService taskResultPayloadService;
private final TaskFileJobService taskFileJobService;
private FileTaskEntity loadTaskForExecution(Long taskId) {
Map<Long, FileTaskEntity> cachedTasks = shopMatchTaskCacheService.getTaskCacheBatch(List.of(taskId));
@@ -409,6 +414,12 @@ public class ShopMatchTaskService {
if (state.getScheduleTimes() == null || stageIndex >= state.getScheduleTimes().size()) {
throw new BusinessException("轮次配置缺失");
}
FileTaskEntity runningTask = findOtherRunningTask(userId, taskId);
if (runningTask != null) {
log.warn("[shop-match] activate serialized taskId={} stageIndex={} blockedByTaskId={} blockedUpdatedAt={}",
taskId, stageIndex, runningTask.getId(), runningTask.getUpdatedAt());
throw new BusinessException("当前已有定时匹配任务执行中,请等待上一任务执行结束后再重试");
}
state.setActiveStageIndex(stageIndex);
task.setStatus("RUNNING");
task.setUpdatedAt(now);
@@ -511,7 +522,7 @@ public class ShopMatchTaskService {
continue;
}
try {
assembleShopResult(result, shopKey, merged, workRoot);
enqueueResultFileAssembly(result, shopKey, merged);
shopMatchTaskCacheService.removeShopMergedPayload(taskId, shopKey);
} catch (Exception ex) {
markResultFailed(result, ex.getMessage() == null ? "结果组装失败" : ex.getMessage());
@@ -603,7 +614,7 @@ public class ShopMatchTaskService {
}
try {
assembleShopResult(result, shopKey, cachedPayload, workRoot);
enqueueResultFileAssembly(result, shopKey, cachedPayload);
shopMatchTaskCacheService.removeShopMergedPayload(taskId, shopKey);
changed = true;
log.warn("[shop-match] stale finalize assembled taskId={} shop={} fromCompensation={} completedFlag={} rowCount={}",
@@ -651,6 +662,43 @@ public class ShopMatchTaskService {
}
}
public void processResultFileJob(TaskFileJobEntity job) {
if (job == null || job.getTaskId() == null || job.getResultId() == null || job.getScopeKey() == null) {
throw new BusinessException("结果文件任务参数不完整");
}
FileResultEntity result = fileResultMapper.selectById(job.getResultId());
if (result == null || !MODULE_TYPE.equals(result.getModuleType())) {
throw new BusinessException("结果记录不存在");
}
ShopMatchShopPayloadDto payload = taskResultPayloadService.getLatest(
job.getTaskId(), MODULE_TYPE, job.getScopeKey(), ShopMatchShopPayloadDto.class);
if (payload == null) {
throw new BusinessException("结果文件载荷不存在");
}
File workRoot = FileUtil.mkdir(FileUtil.file(System.getProperty("java.io.tmpdir"), "shop-match-result", String.valueOf(job.getTaskId())));
assembleShopResult(result, job.getScopeKey(), payload, workRoot);
}
private void enqueueResultFileAssembly(FileResultEntity result, String shopKey, ShopMatchShopPayloadDto payload) {
taskResultPayloadService.saveLatest(result.getTaskId(), MODULE_TYPE, shopKey, payload);
markResultFilePending(result, shopKey, payload);
taskFileJobService.enqueueAssembleResult(result.getTaskId(), MODULE_TYPE, result.getId(), shopKey);
}
private void markResultFilePending(FileResultEntity result, String shopKey, ShopMatchShopPayloadDto payload) {
Map<String, List<ShopMatchRowDto>> countries = excelAssemblyService.normalizeCountriesMap(payload.getCountries());
String displayName = payload.getShopName() != null && !payload.getShopName().isBlank() ? payload.getShopName().trim() : shopKey;
String stem = safeFileStem(displayName);
result.setResultFilename(stem + ".xlsx");
result.setResultFileUrl(null);
result.setResultFileSize(0L);
result.setResultContentType(CONTENT_TYPE_XLSX);
result.setRowCount(excelAssemblyService.countRows(countries));
result.setSuccess(1);
result.setErrorMessage(null);
fileResultMapper.updateById(result);
}
private void markResultFailed(FileResultEntity result, String message) {
result.setSuccess(0);
result.setErrorMessage(message);
@@ -816,9 +864,8 @@ public class ShopMatchTaskService {
vo.setSuccess(success);
vo.setError(entity.getErrorMessage());
vo.setOutputFilename(entity.getResultFilename());
vo.setDownloadUrl(entity.getResultFileUrl() == null || entity.getResultFileUrl().isBlank()
? null
: ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl()));
vo.setDownloadUrl(null);
attachFileJobState(vo, entity);
vo.setMatched(true);
if (entity.getTaskId() != null) {
mergeQueueFieldsFromRequest(vo, entity.getTaskId());
@@ -826,6 +873,18 @@ public class ShopMatchTaskService {
return vo;
}
private void attachFileJobState(ProductRiskResultItemVo vo, FileResultEntity entity) {
TaskFileJobEntity job = taskFileJobService.findAssembleJob(entity.getTaskId(), MODULE_TYPE, entity.getId());
vo.setFileReady(entity.getResultFileUrl() != null && !entity.getResultFileUrl().isBlank());
if (job == null) {
vo.setFileStatus(Boolean.TRUE.equals(vo.getFileReady()) ? "SUCCESS" : null);
return;
}
vo.setFileJobId(job.getId());
vo.setFileStatus(job.getStatus());
vo.setFileError(job.getErrorMessage());
}
private List<ShopMatchTaskStageVo> buildStageVos(FileTaskEntity task, ShopMatchCreateTaskRequest request) {
List<ShopMatchTaskStageVo> stages = new ArrayList<>();
List<LocalDateTime> scheduleTimes = request.getScheduleTimes() == null ? List.of() : request.getScheduleTimes();
@@ -942,6 +1001,19 @@ public class ShopMatchTaskService {
shopMatchTaskCacheService.saveTaskCache(task);
}
private FileTaskEntity findOtherRunningTask(Long userId, Long taskId) {
if (userId == null || userId <= 0) {
return null;
}
return fileTaskMapper.selectOne(new LambdaQueryWrapper<FileTaskEntity>()
.eq(FileTaskEntity::getModuleType, MODULE_TYPE)
.eq(FileTaskEntity::getUserId, userId)
.eq(FileTaskEntity::getStatus, "RUNNING")
.ne(taskId != null, FileTaskEntity::getId, taskId)
.orderByAsc(FileTaskEntity::getUpdatedAt)
.last("limit 1"));
}
private static String fmt(LocalDateTime time) {
return time == null ? null : time.toString();
}
@@ -173,9 +173,7 @@ public class SplitRunService {
vo.setResultId(entity.getId());
vo.setSourceFilename(entity.getSourceFilename());
vo.setOutputFilename(entity.getResultFilename());
vo.setDownloadUrl(entity.getSuccess() != null && entity.getSuccess() == 1
? ossStorageService.generateFreshDownloadUrl(entity.getResultFileUrl())
: null);
vo.setDownloadUrl(null);
vo.setRowCount(entity.getRowCount());
vo.setSuccess(entity.getSuccess() != null && entity.getSuccess() == 1);
vo.setError(entity.getErrorMessage());
@@ -0,0 +1,52 @@
package com.nanri.aiimage.modules.task.controller;
import com.nanri.aiimage.common.api.ApiResponse;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import lombok.RequiredArgsConstructor;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import java.util.List;
import java.util.Map;
@RestController
@RequiredArgsConstructor
@RequestMapping("/api/task-file-jobs")
@Tag(name = "任务结果文件 Job")
public class TaskFileJobController {
private final TaskFileJobService taskFileJobService;
@GetMapping
@Operation(summary = "查询结果文件生成 Job")
public ApiResponse<List<TaskFileJobEntity>> list(
@RequestParam(value = "status", required = false) String status,
@RequestParam(value = "module_type", required = false) String moduleType,
@RequestParam(value = "task_id", required = false) Long taskId,
@RequestParam(value = "limit", defaultValue = "100") int limit) {
return ApiResponse.success(taskFileJobService.listJobs(status, moduleType, taskId, limit));
}
@PostMapping("/{jobId}/retry")
@Operation(summary = "手动重试结果文件生成 Job")
public ApiResponse<Map<String, Object>> retry(@PathVariable Long jobId) {
boolean updated = taskFileJobService.retry(jobId);
return ApiResponse.success(Map.of("jobId", jobId, "updated", updated));
}
@PostMapping("/stuck/reset")
@Operation(summary = "手动扫描并复位卡住的 RUNNING Job")
public ApiResponse<Map<String, Object>> resetStuck(
@RequestParam(value = "timeout_minutes", defaultValue = "30") int timeoutMinutes,
@RequestParam(value = "limit", defaultValue = "100") int limit) {
int reset = taskFileJobService.resetStuckRunningJobs(timeoutMinutes, limit);
return ApiResponse.success(Map.of("reset", reset));
}
}
@@ -0,0 +1,9 @@
package com.nanri.aiimage.modules.task.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface TaskFileJobMapper extends BaseMapper<TaskFileJobEntity> {
}
@@ -0,0 +1,9 @@
package com.nanri.aiimage.modules.task.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskProgressSnapshotEntity;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface TaskProgressSnapshotMapper extends BaseMapper<TaskProgressSnapshotEntity> {
}
@@ -0,0 +1,9 @@
package com.nanri.aiimage.modules.task.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskResultItemEntity;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface TaskResultItemMapper extends BaseMapper<TaskResultItemEntity> {
}
@@ -0,0 +1,9 @@
package com.nanri.aiimage.modules.task.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskResultPayloadEntity;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface TaskResultPayloadMapper extends BaseMapper<TaskResultPayloadEntity> {
}
@@ -0,0 +1,14 @@
package com.nanri.aiimage.modules.task.model.dto;
import java.time.LocalDateTime;
public record TaskFileJobDispatchEvent(
Long jobId,
Long taskId,
String moduleType,
Long resultId,
String scopeKey,
String jobType,
LocalDateTime createdAt
) {
}
@@ -0,0 +1,16 @@
package com.nanri.aiimage.modules.task.model.dto;
import lombok.Data;
import java.time.LocalDateTime;
@Data
public class TaskFileJobMessage {
private Long jobId;
private Long taskId;
private String moduleType;
private Long resultId;
private String scopeKey;
private String jobType;
private LocalDateTime createdAt;
}
@@ -0,0 +1,28 @@
package com.nanri.aiimage.modules.task.model.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.time.LocalDateTime;
@Data
@TableName("biz_task_file_job")
public class TaskFileJobEntity {
@TableId(type = IdType.AUTO)
private Long id;
private Long taskId;
private String moduleType;
private Long resultId;
private String scopeKey;
private String jobType;
private String status;
private Integer retryCount;
private String errorMessage;
private String resultFileUrl;
private LocalDateTime createdAt;
private LocalDateTime updatedAt;
private LocalDateTime finishedAt;
}
@@ -0,0 +1,28 @@
package com.nanri.aiimage.modules.task.model.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.time.LocalDateTime;
@Data
@TableName("biz_task_progress_snapshot")
public class TaskProgressSnapshotEntity {
@TableId(type = IdType.AUTO)
private Long id;
private Long taskId;
private String moduleType;
private String status;
private Integer totalCount;
private Integer successCount;
private Integer failedCount;
private Integer pendingCount;
private String currentScopeKey;
private String message;
private String snapshotJson;
private LocalDateTime createdAt;
private LocalDateTime updatedAt;
}
@@ -0,0 +1,29 @@
package com.nanri.aiimage.modules.task.model.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.time.LocalDateTime;
@Data
@TableName("biz_task_result_item")
public class TaskResultItemEntity {
@TableId(type = IdType.AUTO)
private Long id;
private Long taskId;
private String moduleType;
private Long resultId;
private String scopeKey;
private String scopeHash;
private String itemKey;
private String countryCode;
private String asin;
private String status;
private String payloadJson;
private String payloadHash;
private LocalDateTime createdAt;
private LocalDateTime updatedAt;
}
@@ -0,0 +1,26 @@
package com.nanri.aiimage.modules.task.model.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.time.LocalDateTime;
@Data
@TableName("biz_task_result_payload")
public class TaskResultPayloadEntity {
@TableId(type = IdType.AUTO)
private Long id;
private Long taskId;
private String moduleType;
private String scopeKey;
private String scopeHash;
private String submissionId;
private String payloadJson;
private String payloadHash;
private Long version;
private LocalDateTime createdAt;
private LocalDateTime updatedAt;
}
@@ -0,0 +1,39 @@
package com.nanri.aiimage.modules.task.service;
import com.nanri.aiimage.modules.task.mapper.TaskFileJobMapper;
import com.nanri.aiimage.modules.task.model.dto.TaskFileJobMessage;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.rocketmq.spring.annotation.RocketMQMessageListener;
import org.apache.rocketmq.spring.core.RocketMQListener;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
@Slf4j
@Service
@RequiredArgsConstructor
@ConditionalOnProperty(prefix = "aiimage.result-file-job", name = "mq-enabled", havingValue = "true")
@RocketMQMessageListener(
topic = "${aiimage.result-file-job.topic:aiimage-result-file-job}",
consumerGroup = "${aiimage.result-file-job.consumer-group:aiimage-result-file-job-consumer}"
)
public class TaskFileJobConsumer implements RocketMQListener<TaskFileJobMessage> {
private final TaskFileJobMapper taskFileJobMapper;
private final TaskResultFileJobWorker taskResultFileJobWorker;
@Override
public void onMessage(TaskFileJobMessage message) {
if (message == null || message.getJobId() == null) {
return;
}
TaskFileJobEntity job = taskFileJobMapper.selectById(message.getJobId());
if (job == null) {
log.warn("[task-file-job] mq message ignored, job not found jobId={} taskId={} moduleType={}",
message.getJobId(), message.getTaskId(), message.getModuleType());
return;
}
taskResultFileJobWorker.process(job);
}
}
@@ -0,0 +1,42 @@
package com.nanri.aiimage.modules.task.service;
import com.nanri.aiimage.modules.task.model.dto.TaskFileJobDispatchEvent;
import com.nanri.aiimage.modules.task.model.dto.TaskFileJobMessage;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.transaction.event.TransactionPhase;
import org.springframework.transaction.event.TransactionalEventListener;
@Slf4j
@Component
@RequiredArgsConstructor
public class TaskFileJobDispatchCoordinator {
private final TaskFileJobPublisher taskFileJobPublisher;
private final TaskFileJobLocalDispatcher taskFileJobLocalDispatcher;
@TransactionalEventListener(phase = TransactionPhase.AFTER_COMMIT, fallbackExecution = true)
public void onTaskFileJobDispatch(TaskFileJobDispatchEvent event) {
if (event == null || event.jobId() == null) {
return;
}
TaskFileJobMessage message = new TaskFileJobMessage();
message.setJobId(event.jobId());
message.setTaskId(event.taskId());
message.setModuleType(event.moduleType());
message.setResultId(event.resultId());
message.setScopeKey(event.scopeKey());
message.setJobType(event.jobType());
message.setCreatedAt(event.createdAt());
boolean published = taskFileJobPublisher.publish(message);
if (published) {
return;
}
boolean dispatched = taskFileJobLocalDispatcher.dispatch(event.jobId(), event.taskId(), event.moduleType());
if (dispatched) {
log.info("[task-file-job] local async dispatch scheduled jobId={} taskId={} moduleType={}",
event.jobId(), event.taskId(), event.moduleType());
}
}
}
@@ -0,0 +1,53 @@
package com.nanri.aiimage.modules.task.service;
import com.nanri.aiimage.modules.task.mapper.TaskFileJobMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.ObjectProvider;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.task.TaskExecutor;
import org.springframework.stereotype.Service;
@Slf4j
@Service
@RequiredArgsConstructor
public class TaskFileJobLocalDispatcher {
private final TaskFileJobMapper taskFileJobMapper;
private final ObjectProvider<TaskResultFileJobWorker> taskResultFileJobWorkerProvider;
@Qualifier("taskFileJobDispatchExecutor")
private final TaskExecutor taskFileJobDispatchExecutor;
@Value("${aiimage.result-file-job.local-dispatch-enabled:true}")
private boolean localDispatchEnabled;
public boolean dispatch(Long jobId, Long taskId, String moduleType) {
if (!localDispatchEnabled || jobId == null || jobId <= 0) {
return false;
}
taskFileJobDispatchExecutor.execute(() -> {
try {
TaskFileJobEntity job = taskFileJobMapper.selectById(jobId);
if (job == null) {
log.warn("[task-file-job] local dispatch ignored, job not found jobId={} taskId={} moduleType={}",
jobId, taskId, moduleType);
return;
}
TaskResultFileJobWorker taskResultFileJobWorker = taskResultFileJobWorkerProvider.getIfAvailable();
if (taskResultFileJobWorker == null) {
log.warn("[task-file-job] local dispatch ignored, worker unavailable jobId={} taskId={} moduleType={}",
jobId, taskId, moduleType);
return;
}
taskResultFileJobWorker.process(job);
} catch (Exception ex) {
log.warn("[task-file-job] local dispatch failed jobId={} taskId={} moduleType={} msg={}",
jobId, taskId, moduleType, ex.getMessage(), ex);
}
});
return true;
}
}
@@ -0,0 +1,50 @@
package com.nanri.aiimage.modules.task.service;
import com.nanri.aiimage.modules.task.model.dto.TaskFileJobMessage;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.rocketmq.spring.core.RocketMQTemplate;
import org.springframework.beans.factory.ObjectProvider;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
@Slf4j
@Service
@RequiredArgsConstructor
public class TaskFileJobPublisher {
private final ObjectProvider<RocketMQTemplate> rocketMQTemplateProvider;
@Value("${aiimage.result-file-job.mq-enabled:false}")
private boolean mqEnabled;
@Value("${aiimage.result-file-job.topic:aiimage-result-file-job}")
private String topic;
public boolean publish(TaskFileJobMessage message) {
if (message == null || message.getJobId() == null) {
return false;
}
if (!mqEnabled) {
log.info("[task-file-job] mq disabled, fallback to local dispatcher/scheduled worker jobId={} taskId={} moduleType={}",
message.getJobId(), message.getTaskId(), message.getModuleType());
return false;
}
RocketMQTemplate template = rocketMQTemplateProvider.getIfAvailable();
if (template == null) {
log.warn("[task-file-job] RocketMQTemplate unavailable, jobId={} taskId={} moduleType={}",
message.getJobId(), message.getTaskId(), message.getModuleType());
return false;
}
try {
template.convertAndSend(topic, message);
log.info("[task-file-job] mq published jobId={} taskId={} moduleType={} topic={}",
message.getJobId(), message.getTaskId(), message.getModuleType(), topic);
return true;
} catch (Exception ex) {
log.warn("[task-file-job] publish RocketMQ failed, fallback to local dispatcher/scheduled worker jobId={} taskId={} moduleType={} msg={}",
message.getJobId(), message.getTaskId(), message.getModuleType(), ex.getMessage());
return false;
}
}
}
@@ -0,0 +1,236 @@
package com.nanri.aiimage.modules.task.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.nanri.aiimage.modules.task.mapper.TaskFileJobMapper;
import com.nanri.aiimage.modules.task.model.dto.TaskFileJobDispatchEvent;
import com.nanri.aiimage.modules.task.model.entity.TaskFileJobEntity;
import lombok.RequiredArgsConstructor;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.time.LocalDateTime;
import java.util.List;
@Service
@RequiredArgsConstructor
public class TaskFileJobService {
public static final String JOB_TYPE_ASSEMBLE_RESULT = "ASSEMBLE_RESULT";
private final TaskFileJobMapper taskFileJobMapper;
private final ApplicationEventPublisher applicationEventPublisher;
@Transactional
public TaskFileJobEntity enqueueAssembleResult(Long taskId, String moduleType, Long resultId, String scopeKey) {
if (taskId == null || taskId <= 0 || moduleType == null || moduleType.isBlank() || resultId == null || resultId <= 0) {
return null;
}
LocalDateTime now = LocalDateTime.now();
TaskFileJobEntity existing = findJob(taskId, moduleType, resultId, JOB_TYPE_ASSEMBLE_RESULT);
if (existing == null) {
TaskFileJobEntity entity = new TaskFileJobEntity();
entity.setTaskId(taskId);
entity.setModuleType(moduleType);
entity.setResultId(resultId);
entity.setScopeKey(scopeKey);
entity.setJobType(JOB_TYPE_ASSEMBLE_RESULT);
entity.setStatus("PENDING");
entity.setRetryCount(0);
entity.setCreatedAt(now);
entity.setUpdatedAt(now);
try {
taskFileJobMapper.insert(entity);
publishDispatchEvent(entity);
return entity;
} catch (DuplicateKeyException ignored) {
existing = findJob(taskId, moduleType, resultId, JOB_TYPE_ASSEMBLE_RESULT);
}
}
if (existing == null) {
return null;
}
if ("SUCCESS".equals(existing.getStatus()) || "RUNNING".equals(existing.getStatus())) {
return existing;
}
taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getId, existing.getId())
.set(TaskFileJobEntity::getScopeKey, scopeKey)
.set(TaskFileJobEntity::getStatus, "PENDING")
.set(TaskFileJobEntity::getErrorMessage, null)
.set(TaskFileJobEntity::getUpdatedAt, now)
.set(TaskFileJobEntity::getFinishedAt, null));
TaskFileJobEntity refreshed = taskFileJobMapper.selectById(existing.getId());
publishDispatchEvent(refreshed);
return refreshed;
}
public List<TaskFileJobEntity> listRunnableJobs(int limit) {
return taskFileJobMapper.selectList(new LambdaQueryWrapper<TaskFileJobEntity>()
.in(TaskFileJobEntity::getStatus, List.of("PENDING", "FAILED"))
.lt(TaskFileJobEntity::getRetryCount, 5)
.orderByAsc(TaskFileJobEntity::getUpdatedAt)
.last("limit " + Math.max(1, Math.min(limit, 100))));
}
public List<TaskFileJobEntity> listJobs(String status, String moduleType, Long taskId, int limit) {
LambdaQueryWrapper<TaskFileJobEntity> wrapper = new LambdaQueryWrapper<TaskFileJobEntity>()
.orderByDesc(TaskFileJobEntity::getUpdatedAt)
.last("limit " + Math.max(1, Math.min(limit, 500)));
if (status != null && !status.isBlank()) {
wrapper.eq(TaskFileJobEntity::getStatus, status.trim());
}
if (moduleType != null && !moduleType.isBlank()) {
wrapper.eq(TaskFileJobEntity::getModuleType, moduleType.trim());
}
if (taskId != null && taskId > 0) {
wrapper.eq(TaskFileJobEntity::getTaskId, taskId);
}
return taskFileJobMapper.selectList(wrapper);
}
@Transactional
public boolean retry(Long jobId) {
if (jobId == null || jobId <= 0) {
return false;
}
int updated = taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getId, jobId)
.ne(TaskFileJobEntity::getStatus, "SUCCESS")
.set(TaskFileJobEntity::getStatus, "PENDING")
.set(TaskFileJobEntity::getRetryCount, 0)
.set(TaskFileJobEntity::getErrorMessage, null)
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())
.set(TaskFileJobEntity::getFinishedAt, null));
if (updated > 0) {
TaskFileJobEntity refreshed = taskFileJobMapper.selectById(jobId);
publishDispatchEvent(refreshed);
return true;
}
return false;
}
@Transactional
public int resetStuckRunningJobs(int stuckMinutes, int limit) {
LocalDateTime threshold = LocalDateTime.now().minusMinutes(Math.max(5, stuckMinutes));
List<TaskFileJobEntity> jobs = taskFileJobMapper.selectList(new LambdaQueryWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getStatus, "RUNNING")
.lt(TaskFileJobEntity::getUpdatedAt, threshold)
.orderByAsc(TaskFileJobEntity::getUpdatedAt)
.last("limit " + Math.max(1, Math.min(limit, 200))));
int reset = 0;
for (TaskFileJobEntity job : jobs) {
int retryCount = job.getRetryCount() == null ? 0 : job.getRetryCount();
if (retryCount >= 5) {
taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getId, job.getId())
.eq(TaskFileJobEntity::getStatus, "RUNNING")
.set(TaskFileJobEntity::getStatus, "FAILED")
.set(TaskFileJobEntity::getErrorMessage, "文件生成任务运行超时")
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now()));
continue;
}
int updated = taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getId, job.getId())
.eq(TaskFileJobEntity::getStatus, "RUNNING")
.set(TaskFileJobEntity::getStatus, "PENDING")
.set(TaskFileJobEntity::getErrorMessage, "文件生成任务运行超时,已重新排队")
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now()));
if (updated > 0) {
reset++;
publishDispatchEvent(taskFileJobMapper.selectById(job.getId()));
}
}
return reset;
}
@Transactional
public boolean markRunning(Long jobId) {
if (jobId == null || jobId <= 0) {
return false;
}
return taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getId, jobId)
.in(TaskFileJobEntity::getStatus, List.of("PENDING", "FAILED"))
.set(TaskFileJobEntity::getStatus, "RUNNING")
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())) > 0;
}
@Transactional
public void touchRunning(Long jobId) {
if (jobId == null || jobId <= 0) {
return;
}
taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getId, jobId)
.eq(TaskFileJobEntity::getStatus, "RUNNING")
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now()));
}
@Transactional
public void markSuccess(TaskFileJobEntity job, String resultFileUrl) {
taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getId, job.getId())
.set(TaskFileJobEntity::getStatus, "SUCCESS")
.set(TaskFileJobEntity::getErrorMessage, null)
.set(TaskFileJobEntity::getResultFileUrl, resultFileUrl)
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now())
.set(TaskFileJobEntity::getFinishedAt, LocalDateTime.now()));
}
@Transactional
public void markFailed(TaskFileJobEntity job, String message) {
int retryCount = job.getRetryCount() == null ? 0 : job.getRetryCount() + 1;
taskFileJobMapper.update(null, new LambdaUpdateWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getId, job.getId())
.set(TaskFileJobEntity::getStatus, "FAILED")
.set(TaskFileJobEntity::getRetryCount, retryCount)
.set(TaskFileJobEntity::getErrorMessage, message == null ? "结果文件生成失败" : message)
.set(TaskFileJobEntity::getUpdatedAt, LocalDateTime.now()));
}
public TaskFileJobEntity findAssembleJob(Long taskId, String moduleType, Long resultId) {
return findJob(taskId, moduleType, resultId, JOB_TYPE_ASSEMBLE_RESULT);
}
public long countUnfinishedAssembleJobs(Long taskId, String moduleType) {
if (taskId == null || taskId <= 0 || moduleType == null || moduleType.isBlank()) {
return 0L;
}
Long count = taskFileJobMapper.selectCount(new LambdaQueryWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getTaskId, taskId)
.eq(TaskFileJobEntity::getModuleType, moduleType)
.eq(TaskFileJobEntity::getJobType, JOB_TYPE_ASSEMBLE_RESULT)
.ne(TaskFileJobEntity::getStatus, "SUCCESS"));
return count == null ? 0L : count;
}
private TaskFileJobEntity findJob(Long taskId, String moduleType, Long resultId, String jobType) {
if (taskId == null || moduleType == null || moduleType.isBlank() || resultId == null || jobType == null || jobType.isBlank()) {
return null;
}
return taskFileJobMapper.selectOne(new LambdaQueryWrapper<TaskFileJobEntity>()
.eq(TaskFileJobEntity::getTaskId, taskId)
.eq(TaskFileJobEntity::getModuleType, moduleType)
.eq(TaskFileJobEntity::getResultId, resultId)
.eq(TaskFileJobEntity::getJobType, jobType)
.last("limit 1"));
}
private void publishDispatchEvent(TaskFileJobEntity entity) {
if (entity == null || entity.getId() == null) {
return;
}
applicationEventPublisher.publishEvent(new TaskFileJobDispatchEvent(
entity.getId(),
entity.getTaskId(),
entity.getModuleType(),
entity.getResultId(),
entity.getScopeKey(),
entity.getJobType(),
LocalDateTime.now()
));
}
}
@@ -0,0 +1,98 @@
package com.nanri.aiimage.modules.task.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.common.exception.BusinessException;
import com.nanri.aiimage.modules.task.mapper.TaskProgressSnapshotMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskProgressSnapshotEntity;
import lombok.RequiredArgsConstructor;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.time.LocalDateTime;
@Service
@RequiredArgsConstructor
public class TaskProgressSnapshotService {
private final TaskProgressSnapshotMapper taskProgressSnapshotMapper;
private final ObjectMapper objectMapper;
@Transactional
public void save(Long taskId,
String moduleType,
String status,
int totalCount,
int successCount,
int failedCount,
String currentScopeKey,
String message,
Object snapshot) {
if (taskId == null || taskId <= 0 || isBlank(moduleType) || isBlank(status)) {
return;
}
int pendingCount = Math.max(0, totalCount - successCount - failedCount);
String snapshotJson = snapshot == null ? null : writeJson(snapshot);
LocalDateTime now = LocalDateTime.now();
TaskProgressSnapshotEntity existing = find(taskId, moduleType);
if (existing == null) {
TaskProgressSnapshotEntity entity = new TaskProgressSnapshotEntity();
entity.setTaskId(taskId);
entity.setModuleType(moduleType);
entity.setStatus(status);
entity.setTotalCount(totalCount);
entity.setSuccessCount(successCount);
entity.setFailedCount(failedCount);
entity.setPendingCount(pendingCount);
entity.setCurrentScopeKey(currentScopeKey);
entity.setMessage(message);
entity.setSnapshotJson(snapshotJson);
entity.setCreatedAt(now);
entity.setUpdatedAt(now);
try {
taskProgressSnapshotMapper.insert(entity);
return;
} catch (DuplicateKeyException ignored) {
existing = find(taskId, moduleType);
}
}
if (existing == null) {
throw new BusinessException("保存任务进度快照失败");
}
taskProgressSnapshotMapper.update(null, new LambdaUpdateWrapper<TaskProgressSnapshotEntity>()
.eq(TaskProgressSnapshotEntity::getId, existing.getId())
.set(TaskProgressSnapshotEntity::getStatus, status)
.set(TaskProgressSnapshotEntity::getTotalCount, totalCount)
.set(TaskProgressSnapshotEntity::getSuccessCount, successCount)
.set(TaskProgressSnapshotEntity::getFailedCount, failedCount)
.set(TaskProgressSnapshotEntity::getPendingCount, pendingCount)
.set(TaskProgressSnapshotEntity::getCurrentScopeKey, currentScopeKey)
.set(TaskProgressSnapshotEntity::getMessage, message)
.set(TaskProgressSnapshotEntity::getSnapshotJson, snapshotJson)
.set(TaskProgressSnapshotEntity::getUpdatedAt, now));
}
public TaskProgressSnapshotEntity find(Long taskId, String moduleType) {
if (taskId == null || taskId <= 0 || isBlank(moduleType)) {
return null;
}
return taskProgressSnapshotMapper.selectOne(new LambdaQueryWrapper<TaskProgressSnapshotEntity>()
.eq(TaskProgressSnapshotEntity::getTaskId, taskId)
.eq(TaskProgressSnapshotEntity::getModuleType, moduleType)
.last("limit 1"));
}
private String writeJson(Object value) {
try {
return objectMapper.writeValueAsString(value);
} catch (Exception ex) {
throw new BusinessException("序列化任务进度快照失败");
}
}
private boolean isBlank(String value) {
return value == null || value.isBlank();
}
}

Some files were not shown because too many files have changed in this diff Show More