fix(similarasin): LLM 切换 gemini-3.8-flash + 修复双实例 stale 判死盲区

- 模型切换:glm-5.3-flash / gemini-3.7-flash(类目/合规/图片对比,appearancepatent)→ gemini-3.8-flash
  涉及 SimilarAsinProperties、AppearancePatentProperties、application.yml、TLS 探针与测试断言
- stale 判死盲区:finalizeStaleTasks 去掉 isOwnerCurrent 过滤,改由 job 锁持有实例全局判死
  (任务 27117 线上死因:客户端 done 信号被任务锁拒后断线,owner 过滤导致无人判死,
  被另一实例 job stuck 扫描误判为"文件生成任务运行超时";appearancepatent 同源盲区一并修复)
- 客户端回传重试:3x2s → 8 次,40902 锁忙固定等 15s(app_client/similar_asin.py,随客户端发包生效)
This commit is contained in:
2026-09-07 22:10:20 +08:00
parent 50d0d110a2
commit 83e38d4278
8 changed files with 41 additions and 19 deletions
@@ -18,7 +18,7 @@ public class AppearancePatentProperties {
/** /**
* 外观检测模型(视觉) * 外观检测模型(视觉)
*/ */
private String appearanceModel = "gemini-3.7-flash"; private String appearanceModel = "gemini-3.8-flash";
private int llmMaxTokens = 64000; private int llmMaxTokens = 64000;
private int llmConnectTimeoutMillis = 10000; private int llmConnectTimeoutMillis = 10000;
private int llmReadTimeoutMillis = 180000; private int llmReadTimeoutMillis = 180000;
@@ -150,13 +150,13 @@ public class SimilarAsinProperties {
private String llmApiKey = ""; private String llmApiKey = "";
/** 类目匹配(一级/二级)使用的小模型。 */ /** 类目匹配(一级/二级)使用的小模型。 */
private String llmCategoryModel = "glm-5.3-flash"; private String llmCategoryModel = "gemini-3.8-flash";
/** 合规检查(is_conform/reason/category)使用的小模型。 */ /** 合规检查(is_conform/reason/category)使用的小模型。 */
private String llmConformModel = "glm-5.3-flash"; private String llmConformModel = "gemini-3.8-flash";
/** 图片相似度对比(主图 vs 拼接图)使用的模型。 */ /** 图片相似度对比(主图 vs 拼接图)使用的模型。 */
private String llmImageCompareModel = "glm-5.3-flash"; private String llmImageCompareModel = "gemini-3.8-flash";
private int llmMaxTokens = 64000; private int llmMaxTokens = 64000;
private int llmConnectTimeoutMillis = 10000; private int llmConnectTimeoutMillis = 10000;
@@ -563,14 +563,22 @@ public class AppearancePatentTaskService {
.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(); .atZone(ZoneId.systemDefault()).toInstant().toEpochMilli();
List<FileTaskEntity> tasks = listStaleFinalizeCandidates(); List<FileTaskEntity> tasks = listStaleFinalizeCandidates();
for (FileTaskEntity task : tasks) { for (FileTaskEntity task : tasks) {
if (!isOwnerCurrent(ownerFromTask(task))) { // P1-8(双实例 stale 判死盲区修复,与 similar-asin 同源):本方法由
continue; // DeleteBrandStaleTaskService 的 "delete-brand:stale-check" 分布式 job 锁保护,
} // 单实例执行;历史上 owner 非本实例的任务直接跳过,若任务 owner 是另一实例则无人判死,
// 只能等 job 级 stuck 扫描误判。现改为全局判死:跨实例并发互斥由 task 分布式锁
// acquire 不等待、TTL 5 分钟)保证,owner 实例仍在正常推进的任务心跳不会 stale。
if (!isHeartbeatStale(task, thresholdMillis)) { if (!isHeartbeatStale(task, thresholdMillis)) {
continue; continue;
} }
String owner = ownerFromTask(task);
if (!isOwnerCurrent(owner)) {
log.warn("[appearance-patent] 心跳超时任务归其他实例所有,由本实例(job 锁持有者)接管判死/恢复 taskId={} owner={} current={}",
task.getId(), owner, currentInstanceId());
}
TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(task.getId(), 0L); TaskDistributedLockService.LockHandle lockHandle = acquireTaskLock(task.getId(), 0L);
if (lockHandle == null) { if (lockHandle == null) {
// 锁被 owner 实例/文件任务 worker 占用说明其仍活跃,静默跳过等下一轮
continue; continue;
} }
try (lockHandle) { try (lockHandle) {
@@ -941,19 +941,33 @@ public class SimilarAsinTaskService {
// "等 Python 上传" 分支每 30 分钟自我续命resetStuckJobs 重跑 processResultFileJob // "等 Python 上传" 分支每 30 分钟自我续命resetStuckJobs 重跑 processResultFileJob
// 导致 Python 已断线的 SIMILAR_ASIN 任务永远进不到候选集30 分钟兜底失效 // 导致 Python 已断线的 SIMILAR_ASIN 任务永远进不到候选集30 分钟兜底失效
// Redis heartbeat 仅在 Python 心跳/上传 result 时写入不会被 Java 内部循环刷新 // Redis heartbeat 仅在 Python 心跳/上传 result 时写入不会被 Java 内部循环刷新
// P1-8双实例 stale 判死盲区修复本方法由 DeleteBrandStaleTaskService
// "delete-brand:stale-check" 分布式 job 锁保护任意时刻只有持有该锁的单个实例执行
// 历史上对 owner 非本实例的任务直接 continue若任务 owner 是另一实例owner 实例
// 因拿不到 job 锁不会执行本方法拿到 job 锁的实例又因 owner 过滤跳过该任务
// 导致心跳超时任务无人判死最终被 job stuck 扫描不分 owner误判为
// 文件生成任务运行超时现改为全局判死
// 1) owner 实例内部操作互斥由 task 分布式锁保证acquire 不等待TTL 5 分钟自动续期
// 拿不到锁说明对方正在持锁推进跳过等下轮再判
// 2) owner 实例仍在正常推进的任务其 Redis 心跳不会 stale不会被误伤
// 3) finalizeStaleTask 内部先走 tryRecoverTimedOutPythonTask封口上传并继续 LLM 收尾
// 不会粗暴杀任务跨实例接管是安全的
long thresholdMillis = LocalDateTime.now() long thresholdMillis = LocalDateTime.now()
.minusMinutes(Math.max(1, properties.getStaleTimeoutMinutes())) .minusMinutes(Math.max(1, properties.getStaleTimeoutMinutes()))
.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(); .atZone(ZoneId.systemDefault()).toInstant().toEpochMilli();
List<FileTaskEntity> tasks = listStaleFinalizeCandidates(); List<FileTaskEntity> tasks = listStaleFinalizeCandidates();
for (FileTaskEntity task : tasks) { for (FileTaskEntity task : tasks) {
if (!isOwnerCurrent(ownerFromTask(task))) {
continue;
}
if (!isHeartbeatStale(task, thresholdMillis)) { if (!isHeartbeatStale(task, thresholdMillis)) {
continue; continue;
} }
String owner = ownerFromTask(task);
if (!isOwnerCurrent(owner)) {
log.warn("[similar-asin] 心跳超时任务归其他实例所有,由本实例(job 锁持有者)接管判死/恢复 taskId={} owner={} current={}",
task.getId(), owner, currentInstanceId());
}
TaskDistributedLockService.LockHandle taskLockHandle = acquireTaskLock(task.getId(), 0L); TaskDistributedLockService.LockHandle taskLockHandle = acquireTaskLock(task.getId(), 0L);
if (taskLockHandle == null) { if (taskLockHandle == null) {
// 锁被 owner 实例/文件任务 worker 占用说明其仍活跃静默跳过等下一轮2 分钟后再判
continue; continue;
} }
try (taskLockHandle) { try (taskLockHandle) {
@@ -63,7 +63,7 @@ AIIMAGE_MODULE_CLEANUP_MODULE_TYPES=DEDUPE,SPLIT,CONVERT,DELETE_BRAND
AIIMAGE_APPEARANCE_PATENT_LLM_HOST=https://ai.t8star.org AIIMAGE_APPEARANCE_PATENT_LLM_HOST=https://ai.t8star.org
AIIMAGE_APPEARANCE_PATENT_TITLE_MODEL=deepseek-v4-flash AIIMAGE_APPEARANCE_PATENT_TITLE_MODEL=deepseek-v4-flash
AIIMAGE_APPEARANCE_PATENT_APPEARANCE_MODEL=gemini-3.7-flash AIIMAGE_APPEARANCE_PATENT_APPEARANCE_MODEL=gemini-3.8-flash
AIIMAGE_APPEARANCE_PATENT_LLM_MAX_TOKENS=64000 AIIMAGE_APPEARANCE_PATENT_LLM_MAX_TOKENS=64000
AIIMAGE_APPEARANCE_PATENT_LLM_BATCH_SIZE=10 AIIMAGE_APPEARANCE_PATENT_LLM_BATCH_SIZE=10
AIIMAGE_APPEARANCE_PATENT_LLM_ROW_CONCURRENCY=10 AIIMAGE_APPEARANCE_PATENT_LLM_ROW_CONCURRENCY=10
@@ -242,7 +242,7 @@ aiimage:
appearance-patent: appearance-patent:
llm-host: ${AIIMAGE_APPEARANCE_PATENT_LLM_HOST:https://ai.t8star.org} llm-host: ${AIIMAGE_APPEARANCE_PATENT_LLM_HOST:https://ai.t8star.org}
title-model: ${AIIMAGE_APPEARANCE_PATENT_TITLE_MODEL:deepseek-v4-flash} title-model: ${AIIMAGE_APPEARANCE_PATENT_TITLE_MODEL:deepseek-v4-flash}
appearance-model: ${AIIMAGE_APPEARANCE_PATENT_APPEARANCE_MODEL:gemini-3.7-flash} appearance-model: ${AIIMAGE_APPEARANCE_PATENT_APPEARANCE_MODEL:gemini-3.8-flash}
llm-max-tokens: ${AIIMAGE_APPEARANCE_PATENT_LLM_MAX_TOKENS:64000} llm-max-tokens: ${AIIMAGE_APPEARANCE_PATENT_LLM_MAX_TOKENS:64000}
llm-connect-timeout-millis: ${AIIMAGE_APPEARANCE_PATENT_LLM_CONNECT_TIMEOUT_MILLIS:10000} llm-connect-timeout-millis: ${AIIMAGE_APPEARANCE_PATENT_LLM_CONNECT_TIMEOUT_MILLIS:10000}
llm-read-timeout-millis: ${AIIMAGE_APPEARANCE_PATENT_LLM_READ_TIMEOUT_MILLIS:180000} llm-read-timeout-millis: ${AIIMAGE_APPEARANCE_PATENT_LLM_READ_TIMEOUT_MILLIS:180000}
@@ -272,9 +272,9 @@ aiimage:
direct-llm-enabled: ${AIIMAGE_SIMILAR_ASIN_DIRECT_LLM_ENABLED:true} direct-llm-enabled: ${AIIMAGE_SIMILAR_ASIN_DIRECT_LLM_ENABLED:true}
llm-host: ${AIIMAGE_SIMILAR_ASIN_LLM_HOST:https://ai.t8star.org} llm-host: ${AIIMAGE_SIMILAR_ASIN_LLM_HOST:https://ai.t8star.org}
llm-api-key: ${AIIMAGE_SIMILAR_ASIN_LLM_API_KEY:} llm-api-key: ${AIIMAGE_SIMILAR_ASIN_LLM_API_KEY:}
llm-category-model: ${AIIMAGE_SIMILAR_ASIN_LLM_CATEGORY_MODEL:gemini-3.5-flash-lite} llm-category-model: ${AIIMAGE_SIMILAR_ASIN_LLM_CATEGORY_MODEL:gemini-3.8-flash}
llm-conform-model: ${AIIMAGE_SIMILAR_ASIN_LLM_CONFORM_MODEL:gemini-3.5-flash-lite} llm-conform-model: ${AIIMAGE_SIMILAR_ASIN_LLM_CONFORM_MODEL:gemini-3.8-flash}
llm-image-compare-model: ${AIIMAGE_SIMILAR_ASIN_LLM_IMAGE_COMPARE_MODEL:gemini-3.7-flash} llm-image-compare-model: ${AIIMAGE_SIMILAR_ASIN_LLM_IMAGE_COMPARE_MODEL:gemini-3.8-flash}
llm-max-tokens: ${AIIMAGE_SIMILAR_ASIN_LLM_MAX_TOKENS:64000} llm-max-tokens: ${AIIMAGE_SIMILAR_ASIN_LLM_MAX_TOKENS:64000}
llm-connect-timeout-millis: ${AIIMAGE_SIMILAR_ASIN_LLM_CONNECT_TIMEOUT_MILLIS:10000} llm-connect-timeout-millis: ${AIIMAGE_SIMILAR_ASIN_LLM_CONNECT_TIMEOUT_MILLIS:10000}
llm-read-timeout-millis: ${AIIMAGE_SIMILAR_ASIN_LLM_READ_TIMEOUT_MILLIS:180000} llm-read-timeout-millis: ${AIIMAGE_SIMILAR_ASIN_LLM_READ_TIMEOUT_MILLIS:180000}
@@ -126,7 +126,7 @@ class AppearancePatentLlmClientHttpTest {
assertThat(result.getAppearanceReason()).contains("视觉拆解"); assertThat(result.getAppearanceReason()).contains("视觉拆解");
assertThat(result.getTitleReason()).contains("Apple"); assertThat(result.getTitleReason()).contains("Apple");
assertThat(callCounts.get("deepseek-v4-flash")).hasValue(1); assertThat(callCounts.get("deepseek-v4-flash")).hasValue(1);
assertThat(callCounts.get("gemini-3.7-flash")).hasValue(1); assertThat(callCounts.get("gemini-3.8-flash")).hasValue(1);
} }
@Test @Test
@@ -162,7 +162,7 @@ class AppearancePatentLlmClientHttpTest {
AppearancePatentResultRowDto result = rows.get(0); AppearancePatentResultRowDto result = rows.get(0);
assertThat(result.getAppearanceRisk()).isEqualTo("外观识别异常"); assertThat(result.getAppearanceRisk()).isEqualTo("外观识别异常");
assertThat(callCounts.get("deepseek-v4-flash")).isNull(); assertThat(callCounts.get("deepseek-v4-flash")).isNull();
assertThat(callCounts.get("gemini-3.7-flash")).isNull(); assertThat(callCounts.get("gemini-3.8-flash")).isNull();
} }
@Test @Test
@@ -182,7 +182,7 @@ class AppearancePatentLlmClientHttpTest {
client.inspectRows(List.of(row("6", "B006", "普通数据线", "", "https://img.example.com/6.jpg")), null, "test-key"); client.inspectRows(List.of(row("6", "B006", "普通数据线", "", "https://img.example.com/6.jpg")), null, "test-key");
String appearanceBody = capturedBodies.stream() String appearanceBody = capturedBodies.stream()
.filter(b -> b.contains("gemini-3.7-flash")) .filter(b -> b.contains("gemini-3.8-flash"))
.findFirst() .findFirst()
.orElseThrow(); .orElseThrow();
assertThat(appearanceBody).contains("https://img.example.com/6.jpg"); assertThat(appearanceBody).contains("https://img.example.com/6.jpg");
@@ -61,7 +61,7 @@ public class LlmGatewayTlsProbe {
} }
HttpClient client = builder.build(); HttpClient client = builder.build();
try { try {
String body = "{\"model\":\"glm-5.3-flash\",\"messages\":[{\"role\":\"user\",\"content\":\"ping\"}],\"max_tokens\":10}"; String body = "{\"model\":\"gemini-3.8-flash\",\"messages\":[{\"role\":\"user\",\"content\":\"ping\"}],\"max_tokens\":10}";
HttpRequest request = HttpRequest.newBuilder() HttpRequest request = HttpRequest.newBuilder()
.uri(URI.create("https://" + host + "/v1/chat/completions")) .uri(URI.create("https://" + host + "/v1/chat/completions"))
.timeout(Duration.ofSeconds(30)) .timeout(Duration.ofSeconds(30))