提交货源流程优化

This commit is contained in:
super
2026-05-25 22:16:50 +08:00
parent cbee1d4e23
commit 705ccf70a5
14 changed files with 1490 additions and 85 deletions
@@ -0,0 +1,29 @@
package com.nanri.aiimage.modules.task.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskImageCacheEntity;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Select;
import org.apache.ibatis.annotations.Update;
/**
* P2-12:图片缩略图缓存 mapper。提供 LRU 命中刷新、按 url_hash 直读字节两个轻量入口,
* 其余 CRUD 走 {@link BaseMapper} 默认实现。
*/
@Mapper
public interface TaskImageCacheMapper extends BaseMapper<TaskImageCacheEntity> {
/**
* 命中时同步 bumping last_used_at,便于后续按 LRU 清理 7 天前未用记录。
*/
@Update("UPDATE biz_task_image_cache SET last_used_at = NOW(3) WHERE url_hash = #{urlHash}")
int touchLastUsed(@Param("urlHash") String urlHash);
/**
* 按 url_hash 直读缩略图字节,命中时返回 BLOB;未命中返回 null。
* 选择只 select image_bytes 一列,避免把整行 entity(含 url 字符串)拉回 JVM。
*/
@Select("SELECT image_bytes FROM biz_task_image_cache WHERE url_hash = #{urlHash} LIMIT 1")
byte[] selectBytesByUrlHash(@Param("urlHash") String urlHash);
}
@@ -0,0 +1,36 @@
package com.nanri.aiimage.modules.task.model.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.time.LocalDateTime;
/**
* P2-12:图片缩略图缓存。配合 {@code SimilarAsinImagePrefetchService} 跨任务复用 Coze
* 回包中的 main_url / puzzle_img 图片,避免 assemble 阶段每次都重新下载。
*
* <p>对应表 {@code biz_task_image_cache}(migration V53)。
* <ul>
* <li>{@link #urlHash}:sha256 lowercase hex,作为唯一键避免 1024 长 URL 命中索引长度限制;</li>
* <li>{@link #imageBytes}:已 resize 后的 JPEG 缩略图字节,最大约 300KB(保持与
* {@code SimilarAsinImageEmbedder.MAX_THUMB_SIZE_BYTES} 对齐);</li>
* <li>{@link #lastUsedAt}:用于后续 LRU 清理(>7 天未用)。</li>
* </ul>
*/
@Data
@TableName("biz_task_image_cache")
public class TaskImageCacheEntity {
@TableId(type = IdType.AUTO)
private Long id;
private String urlHash;
private String url;
private byte[] imageBytes;
private Integer byteSize;
private Integer width;
private Integer height;
private LocalDateTime createdAt;
private LocalDateTime lastUsedAt;
}
@@ -1,11 +1,16 @@
package com.nanri.aiimage.modules.task.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.InstanceMetadata;
import com.nanri.aiimage.config.StorageProperties;
import com.nanri.aiimage.config.TransientStorageProperties;
import com.nanri.aiimage.modules.file.service.object.RustfsObjectStorageService;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
@@ -16,9 +21,13 @@ import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.util.ArrayList;
import java.util.Base64;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Objects;
import java.util.Set;
import java.util.UUID;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
@@ -34,12 +43,43 @@ public class TransientPayloadStorageService {
private static final String OSS_POINTER_PREFIX = "oss:";
private static final String LOCAL_PAYLOAD_DIR = "transient-payload";
/**
* P0-3:记录"最近一次 store 是否因 RustFS 失败回落到 local"。
* 调用方(如 SimilarAsinTaskService.submitResult)拿到 storedPayload 后可以查询
* 此 ThreadLocal 决定是否把 task 绑定到 owner instance、是否更新 state_json
* 的 localFallback 标记,从而保证后续 assemble 走对实例。
*/
private static final ThreadLocal<Boolean> LAST_STORE_LOCAL_FALLBACK = ThreadLocal.withInitial(() -> Boolean.FALSE);
private final TransientStorageProperties properties;
private final StorageProperties storageProperties;
private final RustfsObjectStorageService rustfsObjectStorageService;
private final OssStorageService ossStorageService;
private final ObjectMapper objectMapper;
private final InstanceMetadata instanceMetadata;
/**
* P2-9:全局引用计数依赖的两个 mapper。
* 删除 transient payload 之前要反查 biz_task_chunk / biz_task_scope_state,
* 任何一张表还有其它行引用同一个 pointer/value 就跳过删除,避免 deterministic key
* 残留场景下"赢家写、输家删"造成的下游悬空。
*/
private final TaskChunkMapper taskChunkMapper;
private final TaskScopeStateMapper taskScopeStateMapper;
/**
* P0-3:返回上一次 store 调用是否走了 RustFS → local 兜底。
* 注意:跨线程不传递;调用方在同一线程内 store 完后立即读取。
*/
public boolean wasLastStoreLocalFallback() {
return Boolean.TRUE.equals(LAST_STORE_LOCAL_FALLBACK.get());
}
/**
* P0-3:返回当前实例 ID,供调用方写入 ownerInstance 锁定。
*/
public String currentInstanceId() {
return instanceMetadata.getInstanceId();
}
public boolean isWriteEnabled() {
return properties.isEnabled()
@@ -124,6 +164,13 @@ public class TransientPayloadStorageService {
if (pointer == null) {
return;
}
// P2-9:全局引用计数兜底——deterministic key 残留场景下,
// 同一个 pointer 可能被多个 biz_task_chunk / biz_task_scope_state 行共享。
// 删之前先反查一次,确认没有别的行还在引用,再真正落地物理删除。
if (isStillReferenced(pointer, value)) {
log.info("[transient-payload] skip delete, still referenced pointer={}", pointer);
return;
}
if (pointer.startsWith(LOCAL_POINTER_PREFIX)) {
String localKey = pointer.substring(LOCAL_POINTER_PREFIX.length());
String ownerInstance = extractLocalInstanceId(localKey);
@@ -145,6 +192,67 @@ public class TransientPayloadStorageService {
}
}
/**
* P2-9:判断给定 pointer 是否仍被 biz_task_chunk / biz_task_scope_state 中的其它行引用。
*
* <p>背景:历史 deterministic key({@code storeChunkPayload} 等)在并发回传时多个调用方会
* 拿到同一个 pointer,DB 里也会出现多条不同的 task_chunk 行共享同一个 payload 字段值。
* 老路径 caller 把自己那行覆盖/删除后立即调用 {@link #deletePayloadIfPresent},
* 此时若不做反查就会把"赢家"对象一并物理删掉,下游读 chunk 直接变成
* {@code read chunk payload failed}。
*
* <p>判断口径:
* <ul>
* <li>{@code biz_task_chunk.payload_json} 命中 &gt; 1 行(&gt; 1 表示除了 caller 视角下
* 自己即将释放的那一行之外,至少还有别的 chunk 行也指向同一对象)→ 视为仍被引用。</li>
* <li>{@code biz_task_scope_state.parsed_payload_json}/{@code state_json} 命中 &gt; 0 行 →
* 视为仍被引用(这两个字段不是 caller 自身行的常见持有者,命中即非自我引用)。</li>
* </ul>
*
* <p>查询时同时把 caller 传入的 raw value 与 {@code pointer}、{@code "pointer"}(JSON 编码后
* 的字符串形式)都纳入候选,覆盖以下两类常见 DB 字段写入形式:
* <ul>
* <li>{@code objectMapper.writeValueAsString(pointer)} 写入的 JSON 字符串包裹形式;</li>
* <li>少数老路径直接写入裸 pointer 的形式(如 {@link #deleteReplacedPayloadIfNeeded})。</li>
* </ul>
*
* <p>查询出现异常时保守返回 {@code true}(不删),由后续清理任务兜底。
*/
private boolean isStillReferenced(String pointer, String originalValue) {
try {
Set<String> candidates = new LinkedHashSet<>();
if (originalValue != null && !originalValue.isBlank()) {
candidates.add(originalValue);
}
if (pointer != null && !pointer.isBlank()) {
candidates.add(pointer);
try {
candidates.add(objectMapper.writeValueAsString(pointer));
} catch (Exception ignored) {
// 编码失败时仅依赖其它候选
}
}
if (candidates.isEmpty()) {
return false;
}
List<String> values = new ArrayList<>(candidates);
Long chunkCount = taskChunkMapper.selectCount(new LambdaQueryWrapper<TaskChunkEntity>()
.in(TaskChunkEntity::getPayloadJson, values));
if (chunkCount != null && chunkCount > 1L) {
return true;
}
Long scopeStateCount = taskScopeStateMapper.selectCount(new LambdaQueryWrapper<TaskScopeStateEntity>()
.and(w -> w.in(TaskScopeStateEntity::getParsedPayloadJson, values)
.or()
.in(TaskScopeStateEntity::getStateJson, values)));
return scopeStateCount != null && scopeStateCount > 0L;
} catch (Exception ex) {
log.warn("[transient-payload] reference check failed pointer={} err={}", pointer, ex.getMessage());
// 查询出错时保守:不删,等周期性清理兜底
return true;
}
}
public void deleteReplacedPayloadIfNeeded(String oldValue, String newValue) {
String oldPointer = extractPointer(oldValue);
String newPointer = extractPointer(newValue);
@@ -206,18 +314,22 @@ public class TransientPayloadStorageService {
String objectKey = buildObjectKey(category, moduleType, taskId, scopeHash, entryKey);
String storedContent = encodeStoredPayload(content);
String pointer = null;
boolean rustfsFallbackToLocal = false;
if (rustfsObjectStorageService.isConfigured()) {
try {
pointer = RUSTFS_POINTER_PREFIX + rustfsObjectStorageService.uploadText(objectKey, storedContent, verifyAfterUpload);
} catch (Exception ex) {
rustfsFallbackToLocal = true;
// 升级为 ERROR:rustfs 失败后只能落到本地,多实例下其他节点读不到,必须能告警。
log.error("[transient-payload] rustfs upload failed, fallback to local store instanceId={} objectKey={} err={}",
instanceMetadata.getInstanceId(), objectKey, ex.getMessage());
log.error("[transient-payload] rustfs upload failed, fallback to local store instanceId={} category={} taskId={} objectKey={} err={}",
instanceMetadata.getInstanceId(), category, taskId, objectKey, ex.getMessage());
}
}
if (pointer == null) {
pointer = storeLocal(objectKey, storedContent);
}
// P0-3:记录本次 store 是否走 local 兜底,供调用方在拿到 pointer 后立即查询。
LAST_STORE_LOCAL_FALLBACK.set(rustfsFallbackToLocal && pointer != null && pointer.startsWith(LOCAL_POINTER_PREFIX));
if (pointer == null) {
return content;
}