提交货源流程优化
This commit is contained in:
+29
@@ -0,0 +1,29 @@
|
||||
package com.nanri.aiimage.modules.task.mapper;
|
||||
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskImageCacheEntity;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
import org.apache.ibatis.annotations.Select;
|
||||
import org.apache.ibatis.annotations.Update;
|
||||
|
||||
/**
|
||||
* P2-12:图片缩略图缓存 mapper。提供 LRU 命中刷新、按 url_hash 直读字节两个轻量入口,
|
||||
* 其余 CRUD 走 {@link BaseMapper} 默认实现。
|
||||
*/
|
||||
@Mapper
|
||||
public interface TaskImageCacheMapper extends BaseMapper<TaskImageCacheEntity> {
|
||||
|
||||
/**
|
||||
* 命中时同步 bumping last_used_at,便于后续按 LRU 清理 7 天前未用记录。
|
||||
*/
|
||||
@Update("UPDATE biz_task_image_cache SET last_used_at = NOW(3) WHERE url_hash = #{urlHash}")
|
||||
int touchLastUsed(@Param("urlHash") String urlHash);
|
||||
|
||||
/**
|
||||
* 按 url_hash 直读缩略图字节,命中时返回 BLOB;未命中返回 null。
|
||||
* 选择只 select image_bytes 一列,避免把整行 entity(含 url 字符串)拉回 JVM。
|
||||
*/
|
||||
@Select("SELECT image_bytes FROM biz_task_image_cache WHERE url_hash = #{urlHash} LIMIT 1")
|
||||
byte[] selectBytesByUrlHash(@Param("urlHash") String urlHash);
|
||||
}
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
package com.nanri.aiimage.modules.task.model.entity;
|
||||
|
||||
import com.baomidou.mybatisplus.annotation.IdType;
|
||||
import com.baomidou.mybatisplus.annotation.TableId;
|
||||
import com.baomidou.mybatisplus.annotation.TableName;
|
||||
import lombok.Data;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
/**
|
||||
* P2-12:图片缩略图缓存。配合 {@code SimilarAsinImagePrefetchService} 跨任务复用 Coze
|
||||
* 回包中的 main_url / puzzle_img 图片,避免 assemble 阶段每次都重新下载。
|
||||
*
|
||||
* <p>对应表 {@code biz_task_image_cache}(migration V53)。
|
||||
* <ul>
|
||||
* <li>{@link #urlHash}:sha256 lowercase hex,作为唯一键避免 1024 长 URL 命中索引长度限制;</li>
|
||||
* <li>{@link #imageBytes}:已 resize 后的 JPEG 缩略图字节,最大约 300KB(保持与
|
||||
* {@code SimilarAsinImageEmbedder.MAX_THUMB_SIZE_BYTES} 对齐);</li>
|
||||
* <li>{@link #lastUsedAt}:用于后续 LRU 清理(>7 天未用)。</li>
|
||||
* </ul>
|
||||
*/
|
||||
@Data
|
||||
@TableName("biz_task_image_cache")
|
||||
public class TaskImageCacheEntity {
|
||||
|
||||
@TableId(type = IdType.AUTO)
|
||||
private Long id;
|
||||
private String urlHash;
|
||||
private String url;
|
||||
private byte[] imageBytes;
|
||||
private Integer byteSize;
|
||||
private Integer width;
|
||||
private Integer height;
|
||||
private LocalDateTime createdAt;
|
||||
private LocalDateTime lastUsedAt;
|
||||
}
|
||||
+114
-2
@@ -1,11 +1,16 @@
|
||||
package com.nanri.aiimage.modules.task.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.config.InstanceMetadata;
|
||||
import com.nanri.aiimage.config.StorageProperties;
|
||||
import com.nanri.aiimage.config.TransientStorageProperties;
|
||||
import com.nanri.aiimage.modules.file.service.object.RustfsObjectStorageService;
|
||||
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
@@ -16,9 +21,13 @@ import java.nio.file.Path;
|
||||
import java.nio.file.StandardOpenOption;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Base64;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
import java.util.zip.GZIPInputStream;
|
||||
import java.util.zip.GZIPOutputStream;
|
||||
@@ -34,12 +43,43 @@ public class TransientPayloadStorageService {
|
||||
private static final String OSS_POINTER_PREFIX = "oss:";
|
||||
private static final String LOCAL_PAYLOAD_DIR = "transient-payload";
|
||||
|
||||
/**
|
||||
* P0-3:记录"最近一次 store 是否因 RustFS 失败回落到 local"。
|
||||
* 调用方(如 SimilarAsinTaskService.submitResult)拿到 storedPayload 后可以查询
|
||||
* 此 ThreadLocal 决定是否把 task 绑定到 owner instance、是否更新 state_json
|
||||
* 的 localFallback 标记,从而保证后续 assemble 走对实例。
|
||||
*/
|
||||
private static final ThreadLocal<Boolean> LAST_STORE_LOCAL_FALLBACK = ThreadLocal.withInitial(() -> Boolean.FALSE);
|
||||
|
||||
private final TransientStorageProperties properties;
|
||||
private final StorageProperties storageProperties;
|
||||
private final RustfsObjectStorageService rustfsObjectStorageService;
|
||||
private final OssStorageService ossStorageService;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final InstanceMetadata instanceMetadata;
|
||||
/**
|
||||
* P2-9:全局引用计数依赖的两个 mapper。
|
||||
* 删除 transient payload 之前要反查 biz_task_chunk / biz_task_scope_state,
|
||||
* 任何一张表还有其它行引用同一个 pointer/value 就跳过删除,避免 deterministic key
|
||||
* 残留场景下"赢家写、输家删"造成的下游悬空。
|
||||
*/
|
||||
private final TaskChunkMapper taskChunkMapper;
|
||||
private final TaskScopeStateMapper taskScopeStateMapper;
|
||||
|
||||
/**
|
||||
* P0-3:返回上一次 store 调用是否走了 RustFS → local 兜底。
|
||||
* 注意:跨线程不传递;调用方在同一线程内 store 完后立即读取。
|
||||
*/
|
||||
public boolean wasLastStoreLocalFallback() {
|
||||
return Boolean.TRUE.equals(LAST_STORE_LOCAL_FALLBACK.get());
|
||||
}
|
||||
|
||||
/**
|
||||
* P0-3:返回当前实例 ID,供调用方写入 ownerInstance 锁定。
|
||||
*/
|
||||
public String currentInstanceId() {
|
||||
return instanceMetadata.getInstanceId();
|
||||
}
|
||||
|
||||
public boolean isWriteEnabled() {
|
||||
return properties.isEnabled()
|
||||
@@ -124,6 +164,13 @@ public class TransientPayloadStorageService {
|
||||
if (pointer == null) {
|
||||
return;
|
||||
}
|
||||
// P2-9:全局引用计数兜底——deterministic key 残留场景下,
|
||||
// 同一个 pointer 可能被多个 biz_task_chunk / biz_task_scope_state 行共享。
|
||||
// 删之前先反查一次,确认没有别的行还在引用,再真正落地物理删除。
|
||||
if (isStillReferenced(pointer, value)) {
|
||||
log.info("[transient-payload] skip delete, still referenced pointer={}", pointer);
|
||||
return;
|
||||
}
|
||||
if (pointer.startsWith(LOCAL_POINTER_PREFIX)) {
|
||||
String localKey = pointer.substring(LOCAL_POINTER_PREFIX.length());
|
||||
String ownerInstance = extractLocalInstanceId(localKey);
|
||||
@@ -145,6 +192,67 @@ public class TransientPayloadStorageService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* P2-9:判断给定 pointer 是否仍被 biz_task_chunk / biz_task_scope_state 中的其它行引用。
|
||||
*
|
||||
* <p>背景:历史 deterministic key({@code storeChunkPayload} 等)在并发回传时多个调用方会
|
||||
* 拿到同一个 pointer,DB 里也会出现多条不同的 task_chunk 行共享同一个 payload 字段值。
|
||||
* 老路径 caller 把自己那行覆盖/删除后立即调用 {@link #deletePayloadIfPresent},
|
||||
* 此时若不做反查就会把"赢家"对象一并物理删掉,下游读 chunk 直接变成
|
||||
* {@code read chunk payload failed}。
|
||||
*
|
||||
* <p>判断口径:
|
||||
* <ul>
|
||||
* <li>{@code biz_task_chunk.payload_json} 命中 > 1 行(> 1 表示除了 caller 视角下
|
||||
* 自己即将释放的那一行之外,至少还有别的 chunk 行也指向同一对象)→ 视为仍被引用。</li>
|
||||
* <li>{@code biz_task_scope_state.parsed_payload_json}/{@code state_json} 命中 > 0 行 →
|
||||
* 视为仍被引用(这两个字段不是 caller 自身行的常见持有者,命中即非自我引用)。</li>
|
||||
* </ul>
|
||||
*
|
||||
* <p>查询时同时把 caller 传入的 raw value 与 {@code pointer}、{@code "pointer"}(JSON 编码后
|
||||
* 的字符串形式)都纳入候选,覆盖以下两类常见 DB 字段写入形式:
|
||||
* <ul>
|
||||
* <li>{@code objectMapper.writeValueAsString(pointer)} 写入的 JSON 字符串包裹形式;</li>
|
||||
* <li>少数老路径直接写入裸 pointer 的形式(如 {@link #deleteReplacedPayloadIfNeeded})。</li>
|
||||
* </ul>
|
||||
*
|
||||
* <p>查询出现异常时保守返回 {@code true}(不删),由后续清理任务兜底。
|
||||
*/
|
||||
private boolean isStillReferenced(String pointer, String originalValue) {
|
||||
try {
|
||||
Set<String> candidates = new LinkedHashSet<>();
|
||||
if (originalValue != null && !originalValue.isBlank()) {
|
||||
candidates.add(originalValue);
|
||||
}
|
||||
if (pointer != null && !pointer.isBlank()) {
|
||||
candidates.add(pointer);
|
||||
try {
|
||||
candidates.add(objectMapper.writeValueAsString(pointer));
|
||||
} catch (Exception ignored) {
|
||||
// 编码失败时仅依赖其它候选
|
||||
}
|
||||
}
|
||||
if (candidates.isEmpty()) {
|
||||
return false;
|
||||
}
|
||||
List<String> values = new ArrayList<>(candidates);
|
||||
Long chunkCount = taskChunkMapper.selectCount(new LambdaQueryWrapper<TaskChunkEntity>()
|
||||
.in(TaskChunkEntity::getPayloadJson, values));
|
||||
if (chunkCount != null && chunkCount > 1L) {
|
||||
return true;
|
||||
}
|
||||
Long scopeStateCount = taskScopeStateMapper.selectCount(new LambdaQueryWrapper<TaskScopeStateEntity>()
|
||||
.and(w -> w.in(TaskScopeStateEntity::getParsedPayloadJson, values)
|
||||
.or()
|
||||
.in(TaskScopeStateEntity::getStateJson, values)));
|
||||
return scopeStateCount != null && scopeStateCount > 0L;
|
||||
} catch (Exception ex) {
|
||||
log.warn("[transient-payload] reference check failed pointer={} err={}", pointer, ex.getMessage());
|
||||
// 查询出错时保守:不删,等周期性清理兜底
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
public void deleteReplacedPayloadIfNeeded(String oldValue, String newValue) {
|
||||
String oldPointer = extractPointer(oldValue);
|
||||
String newPointer = extractPointer(newValue);
|
||||
@@ -206,18 +314,22 @@ public class TransientPayloadStorageService {
|
||||
String objectKey = buildObjectKey(category, moduleType, taskId, scopeHash, entryKey);
|
||||
String storedContent = encodeStoredPayload(content);
|
||||
String pointer = null;
|
||||
boolean rustfsFallbackToLocal = false;
|
||||
if (rustfsObjectStorageService.isConfigured()) {
|
||||
try {
|
||||
pointer = RUSTFS_POINTER_PREFIX + rustfsObjectStorageService.uploadText(objectKey, storedContent, verifyAfterUpload);
|
||||
} catch (Exception ex) {
|
||||
rustfsFallbackToLocal = true;
|
||||
// 升级为 ERROR:rustfs 失败后只能落到本地,多实例下其他节点读不到,必须能告警。
|
||||
log.error("[transient-payload] rustfs upload failed, fallback to local store instanceId={} objectKey={} err={}",
|
||||
instanceMetadata.getInstanceId(), objectKey, ex.getMessage());
|
||||
log.error("[transient-payload] rustfs upload failed, fallback to local store instanceId={} category={} taskId={} objectKey={} err={}",
|
||||
instanceMetadata.getInstanceId(), category, taskId, objectKey, ex.getMessage());
|
||||
}
|
||||
}
|
||||
if (pointer == null) {
|
||||
pointer = storeLocal(objectKey, storedContent);
|
||||
}
|
||||
// P0-3:记录本次 store 是否走 local 兜底,供调用方在拿到 pointer 后立即查询。
|
||||
LAST_STORE_LOCAL_FALLBACK.set(rustfsFallbackToLocal && pointer != null && pointer.startsWith(LOCAL_POINTER_PREFIX));
|
||||
if (pointer == null) {
|
||||
return content;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user