fix(临时载荷): 收口删除守卫 + chunk 读兜底,修「对象被误删→任务永久失败」

线上任务 28459(外观专利)三个分片的载荷对象被删、DB 行仍指向已删的确定性
key,组装读不到 → 整单 FAILED。数据其实还在同槽位的版本化对象里,是行指错了。

三条收口:
- 上传补偿删除加准入:只有末段带 UUID 的版本化 key(本次写入独占)才允许入队。
  确定性 key 会被重传复用,删它就删掉了行仍在引用的对象;而删除队列
  (deleteObjectFromRetry -> removeObject)本身不做引用反查,准入必须卡在入队处。
- 引用守卫阈值 >1 收紧为 >0:原来把「恰好 1 行引用」当作调用方自己那行而放行,
  但调用方无法证明归属。物理删除本就约定在 DB 行删除之后执行,正常路径引用数
  必为 0;宁可留孤儿(有保留期清理兜底),也不删可能仍被引用的对象。
- chunk 读兜底:指针对象确已缺失(NoSuchKey)时回退同槽位版本化对象 chunk-N-*。
  仅限 chunk 槽位 + 确为缺失两个条件,避免误配无关对象或掩盖真实故障。

测试:新增 31 个用例(补偿删除准入 / 引用守卫 / 兄弟对象兜底),均先确认 RED
再实现;另更新 3 个断言旧行为的既有用例。全量 mvn test 3082 个 0 失败。
This commit is contained in:
2026-09-17 10:42:53 +08:00
parent 1fe3368c5a
commit 3137299bfe
7 changed files with 812 additions and 15 deletions
@@ -5,11 +5,14 @@ import io.micrometer.core.instrument.DistributionSummary;
import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.Timer; import io.micrometer.core.instrument.Timer;
import io.minio.GetObjectArgs; import io.minio.GetObjectArgs;
import io.minio.ListObjectsArgs;
import io.minio.MinioClient; import io.minio.MinioClient;
import io.minio.PutObjectArgs; import io.minio.PutObjectArgs;
import io.minio.RemoveObjectArgs; import io.minio.RemoveObjectArgs;
import io.minio.Result;
import io.minio.StatObjectArgs; import io.minio.StatObjectArgs;
import io.minio.errors.ErrorResponseException; import io.minio.errors.ErrorResponseException;
import io.minio.messages.Item;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import okhttp3.ConnectionPool; import okhttp3.ConnectionPool;
import okhttp3.Dispatcher; import okhttp3.Dispatcher;
@@ -21,8 +24,11 @@ import org.springframework.stereotype.Service;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream; import java.io.ByteArrayOutputStream;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects; import java.util.Objects;
import java.util.Set; import java.util.Set;
import java.util.UUID;
import java.util.concurrent.Semaphore; import java.util.concurrent.Semaphore;
import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
@@ -45,6 +51,12 @@ public class RustfsObjectStorageService {
"NoSuchKey", "NoSuchBucket", "NoSuchVersion", "NoSuchKey", "NoSuchBucket", "NoSuchVersion",
"AccessDenied", "InvalidAccessKeyId", "SignatureDoesNotMatch", "InvalidBucketName"); "AccessDenied", "InvalidAccessKeyId", "SignatureDoesNotMatch", "InvalidBucketName");
/** 标准 UUID 字符串长度(8-4-4-4-12),用于识别版本化对象 key。 */
private static final int UUID_STRING_LENGTH = 36;
/** 兄弟对象兜底列出的上限:只为找回同槽位的版本化对象,不需要列全。 */
private static final int MAX_SIBLING_LIST_KEYS = 50;
private final TransientStorageProperties properties; private final TransientStorageProperties properties;
private final ObjectProvider<MeterRegistry> meterRegistryProvider; private final ObjectProvider<MeterRegistry> meterRegistryProvider;
private final ObjectProvider<RustfsDeleteRetryService> deleteRetryServiceProvider; private final ObjectProvider<RustfsDeleteRetryService> deleteRetryServiceProvider;
@@ -106,7 +118,22 @@ public class RustfsObjectStorageService {
return uploadBytes(objectKey, bytes, verifyAfterUpload); return uploadBytes(objectKey, bytes, verifyAfterUpload);
} }
/**
* 三参重载:是否做「上传失败补偿删除」按对象 key 形态自动判定。
*
* <p>只有版本化 key(末段以 UUID 结尾)是本次写入独占的;确定性 key 会被重传重写复用,
* 删它就可能删掉别的 DB 行仍在引用的对象(2026-09-17 线上任务 28459 的载荷对象就是这么丢的)。
*/
public String uploadBytes(String objectKey, byte[] content, boolean verifyAfterUpload) { public String uploadBytes(String objectKey, byte[] content, boolean verifyAfterUpload) {
return uploadBytes(objectKey, content, verifyAfterUpload, isVersionedObjectKey(objectKey));
}
/**
* @param compensateDeleteOnFailure put 已完成、但后续可见性校验失败时,是否把该对象排进删除补偿队列。
* 仅当调用方能确认「该对象不会被其它写入复用时」才可传 true。
*/
public String uploadBytes(String objectKey, byte[] content, boolean verifyAfterUpload,
boolean compensateDeleteOnFailure) {
long deadlineNanos = operationDeadlineNanos(); long deadlineNanos = operationDeadlineNanos();
if (!isConfigured()) { if (!isConfigured()) {
throw new IllegalStateException("transient storage is not configured"); throw new IllegalStateException("transient storage is not configured");
@@ -136,13 +163,45 @@ public class RustfsObjectStorageService {
} }
return uploadedObjectKey; return uploadedObjectKey;
} catch (RuntimeException ex) { } catch (RuntimeException ex) {
if (putCompleted.get()) { if (putCompleted.get() && compensateDeleteOnFailure) {
enqueueDeleteRetry(objectKey, ex); enqueueDeleteRetry(objectKey, ex);
} else if (putCompleted.get()) {
// 共享 key 会被重传重写:此处删除可能删掉别的行正在引用的对象,交给保留期清理兜底。
// 线上任务 28459 的 chunk-462/473/484 就是被这条无条件删除队列删掉的。
log.warn("[rustfs] 跳过上传失败补偿删除(对象非本次独占,可能被复用)objectKey={} err={}",
objectKey, ex.getMessage());
} }
throw ex; throw ex;
} }
} }
/**
* 对象 key 是否为「本次写入独占」的版本化 key:末段(去掉 {@code .json} 后缀)以 UUID 结尾。
*
* <p>只看末段——UUID 出现在中间段(如 scopeHash)不代表该对象被独占;解析失败一律按共享处理
* (保守:宁可留下孤儿对象,也不删掉可能仍被引用的对象)。
*/
static boolean isVersionedObjectKey(String objectKey) {
if (objectKey == null) {
return false;
}
String key = objectKey.trim();
if (key.endsWith(".json")) {
key = key.substring(0, key.length() - ".json".length());
}
int slash = key.lastIndexOf('/');
String lastSegment = slash < 0 ? key : key.substring(slash + 1);
if (lastSegment.length() < UUID_STRING_LENGTH) {
return false;
}
try {
UUID.fromString(lastSegment.substring(lastSegment.length() - UUID_STRING_LENGTH));
return true;
} catch (IllegalArgumentException ex) {
return false;
}
}
public String readObjectAsString(String objectKey) { public String readObjectAsString(String objectKey) {
byte[] bytes = readObjectBytes(objectKey); byte[] bytes = readObjectBytes(objectKey);
return new String(bytes, StandardCharsets.UTF_8); return new String(bytes, StandardCharsets.UTF_8);
@@ -182,6 +241,43 @@ public class RustfsObjectStorageService {
}); });
} }
/**
* 列出前缀下的对象 key,按最后修改时间倒序(最新在前)。
*
* <p>只服务于「指针指向的对象已不存在、需要找回同槽位的版本化兄弟对象」这一兜底路径,
* 因此刻意不做重试、不参与失败窗口记账:列出失败直接抛错,由调用方按原错误语义处理。
*/
public List<String> listObjectKeysNewestFirst(String prefix, int limit) {
if (!isConfigured()) {
throw new IllegalStateException("transient storage is not configured");
}
int safeLimit = Math.max(1, Math.min(limit, MAX_SIBLING_LIST_KEYS));
long deadlineNanos = operationDeadlineNanos();
List<String[]> entries = new ArrayList<>();
try {
Iterable<Result<Item>> results = buildClient(deadlineNanos).listObjects(ListObjectsArgs.builder()
.bucket(properties.getBucket())
.prefix(prefix == null ? "" : prefix)
.recursive(true)
.maxKeys(safeLimit)
.build());
for (Result<Item> result : results) {
Item item = result.get();
entries.add(new String[]{item.objectName(),
item.lastModified() == null ? "" : item.lastModified().toString()});
}
} catch (Exception ex) {
throw new IllegalStateException("transient storage list failed prefix=" + prefix
+ " err=" + ex.getMessage(), ex);
}
entries.sort((left, right) -> right[1].compareTo(left[1]));
List<String> keys = new ArrayList<>(entries.size());
for (String[] entry : entries) {
keys.add(entry[0]);
}
return keys;
}
public void deleteObject(String objectKey) { public void deleteObject(String objectKey) {
deleteObject(objectKey, true, operationDeadlineNanos()); deleteObject(objectKey, true, operationDeadlineNanos());
} }
@@ -30,6 +30,7 @@ import java.util.Locale;
import java.util.Objects; import java.util.Objects;
import java.util.Set; import java.util.Set;
import java.util.UUID; import java.util.UUID;
import java.util.regex.Pattern;
import java.util.zip.GZIPInputStream; import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream; import java.util.zip.GZIPOutputStream;
@@ -150,8 +151,20 @@ public class TransientPayloadStorageService {
return decodeStoredPayloadBytes(readLocalPayloadBytes(stripLocalInstanceId(localKey))); return decodeStoredPayloadBytes(readLocalPayloadBytes(stripLocalInstanceId(localKey)));
} }
if (pointer.startsWith(RUSTFS_POINTER_PREFIX)) { if (pointer.startsWith(RUSTFS_POINTER_PREFIX)) {
return decodeStoredPayloadBytes( String objectKey = pointer.substring(RUSTFS_POINTER_PREFIX.length());
rustfsObjectStorageService.readObjectBytes(pointer.substring(RUSTFS_POINTER_PREFIX.length()))); try {
return decodeStoredPayloadBytes(rustfsObjectStorageService.readObjectBytes(objectKey));
} catch (RuntimeException readException) {
String sibling = findVersionedChunkSibling(objectKey, readException);
if (sibling == null) {
throw readException;
}
// 2026-09-17 线上任务 28459:行指向的普通 key 被误删,但同一分片槽位的版本化对象还在。
// 读出它即可让任务按已有数据出结果,不必整单失败。
log.warn("[transient-payload] chunk 载荷对象不存在,回退同槽位版本化对象 pointer={} sibling={}",
pointer, sibling);
return decodeStoredPayloadBytes(rustfsObjectStorageService.readObjectBytes(sibling));
}
} }
if (pointer.startsWith(OSS_POINTER_PREFIX)) { if (pointer.startsWith(OSS_POINTER_PREFIX)) {
return decodeStoredPayloadBytes( return decodeStoredPayloadBytes(
@@ -164,6 +177,53 @@ public class TransientPayloadStorageService {
return value; return value;
} }
/** chunk 载荷槽位的 entryKey 形态:{@code chunk-<index>}(版本化写入则形如 {@code chunk-<index>-<uuid>})。 */
private static final Pattern CHUNK_ENTRY_KEY_PATTERN = Pattern.compile("chunk-\\d+");
/** 兄弟对象查找上限:只为找回同槽位对象,不需要列全。 */
private static final int MAX_SIBLING_LOOKUP_KEYS = 50;
/**
* 指针对象已不存在时,尝试找回同一分片槽位的版本化兄弟对象。
*
* <p>两个条件同时满足才兜底,避免读到无关对象或掩盖真实故障:
* <ol>
* <li>末段 entryKey 是 {@code chunk-<index>} 形态——只有这种槽位才有「版本化兄弟」语义;</li>
* <li>失败原因是对象确实不存在(NoSuchKey)——权限/网络类失败照旧上抛以便重试。</li>
* </ol>
*/
private String findVersionedChunkSibling(String objectKey, Throwable cause) {
if (!isObjectMissing(cause)) {
return null;
}
int slash = objectKey.lastIndexOf('/');
String directory = slash < 0 ? "" : objectKey.substring(0, slash + 1);
String fileName = slash < 0 ? objectKey : objectKey.substring(slash + 1);
if (!fileName.endsWith(".json")) {
return null;
}
String entryKey = fileName.substring(0, fileName.length() - ".json".length());
if (!CHUNK_ENTRY_KEY_PATTERN.matcher(entryKey).matches()) {
return null;
}
List<String> candidates = rustfsObjectStorageService.listObjectKeysNewestFirst(
directory + entryKey + "-", MAX_SIBLING_LOOKUP_KEYS);
return candidates.isEmpty() ? null : candidates.getFirst();
}
/** 对象确已不存在:RustFS 返回 NoSuchKeymessage 为 "The specified key does not exist."。 */
private static boolean isObjectMissing(Throwable error) {
Throwable cursor = error;
while (cursor != null) {
String message = cursor.getMessage();
if (message != null && message.contains("does not exist")) {
return true;
}
cursor = cursor.getCause();
}
return false;
}
public void deletePayloadIfPresent(String value) { public void deletePayloadIfPresent(String value) {
String pointer = extractPointer(value); String pointer = extractPointer(value);
if (pointer == null) { if (pointer == null) {
@@ -209,8 +269,10 @@ public class TransientPayloadStorageService {
* *
* <p>判断口径: * <p>判断口径:
* <ul> * <ul>
* <li>{@code biz_task_chunk.payload_json} 命中 &gt; 1 行(&gt; 1 表示除了 caller 视角下 * <li>{@code biz_task_chunk.payload_json} 命中任意行(&ge; 1)→ 视为仍被引用。
* 自己即将释放的那一行之外,至少还有别的 chunk 行也指向同一对象)→ 视为仍被引用。</li> * 曾用 {@code > 1} 作判据,等于放行「恰好还有 1 行引用」的情况,会把对方仍在用的对象
* 删掉(2026-09-17 线上任务 28459:合并成功后指针未落库 + 旧对象被删 → 该分片永久读不到)。
* 物理删除本就约定在 DB 行删除之后执行,故调用方正常路径下引用数必然为 0。</li>
* <li>{@code biz_task_scope_state.parsed_payload_json}/{@code state_json} 命中 &gt; 0 行 → * <li>{@code biz_task_scope_state.parsed_payload_json}/{@code state_json} 命中 &gt; 0 行 →
* 视为仍被引用(这两个字段不是 caller 自身行的常见持有者,命中即非自我引用)。</li> * 视为仍被引用(这两个字段不是 caller 自身行的常见持有者,命中即非自我引用)。</li>
* </ul> * </ul>
@@ -248,7 +310,7 @@ public class TransientPayloadStorageService {
// 解析不出 taskId 时按原口径全局查,行为与改造前一致。 // 解析不出 taskId 时按原口径全局查,行为与改造前一致。
Long pointerTaskId = extractTaskId(pointer); Long pointerTaskId = extractTaskId(pointer);
Long chunkCount = referencedChunkCount(pointerTaskId, values); Long chunkCount = referencedChunkCount(pointerTaskId, values);
if (chunkCount != null && chunkCount > 1L) { if (chunkCount != null && chunkCount > 0L) {
return true; return true;
} }
Long scopeStateCount = referencedScopeStateCount(pointerTaskId, values); Long scopeStateCount = referencedScopeStateCount(pointerTaskId, values);
@@ -297,16 +297,19 @@ class RustfsObjectStorageServiceTest {
RustfsObjectStorageService service = new RustfsObjectStorageService( RustfsObjectStorageService service = new RustfsObjectStorageService(
properties, emptyProvider(), provider(retryService), () -> client); properties, emptyProvider(), provider(retryService), () -> client);
// 补偿删除只对版本化 key(本次写入独占)生效,故这里用 versioned key 覆盖该路径
String firstKey = "task/a-11111111-2222-3333-4444-555555555555.json";
String secondKey = "task/b-66666666-7777-8888-9999-000000000000.json";
IllegalStateException firstFailure = assertThrows(IllegalStateException.class, IllegalStateException firstFailure = assertThrows(IllegalStateException.class,
() -> service.uploadText("task/a.json", "{}", true)); () -> service.uploadText(firstKey, "{}", true));
IllegalStateException secondFailure = assertThrows(IllegalStateException.class, IllegalStateException secondFailure = assertThrows(IllegalStateException.class,
() -> service.uploadText("task/b.json", "{}", true)); () -> service.uploadText(secondKey, "{}", true));
assertTrue(firstFailure.getMessage().contains("not visible")); assertTrue(firstFailure.getMessage().contains("not visible"));
assertTrue(secondFailure.getMessage().contains("not visible")); assertTrue(secondFailure.getMessage().contains("not visible"));
verify(client, times(5)).statObject(any(StatObjectArgs.class)); verify(client, times(5)).statObject(any(StatObjectArgs.class));
verify(retryService).enqueue(eq("task/a.json"), same(firstFailure)); verify(retryService).enqueue(eq(firstKey), same(firstFailure));
verify(retryService).enqueue(eq("task/b.json"), same(secondFailure)); verify(retryService).enqueue(eq(secondKey), same(secondFailure));
IllegalStateException rejected = assertThrows(IllegalStateException.class, IllegalStateException rejected = assertThrows(IllegalStateException.class,
() -> service.uploadText("task/c.json", "{}", false)); () -> service.uploadText("task/c.json", "{}", false));
assertTrue(rejected.getMessage().contains("cooldown active")); assertTrue(rejected.getMessage().contains("cooldown active"));
@@ -360,6 +363,9 @@ class RustfsObjectStorageServiceTest {
@Test @Test
void completedUploadIsEnqueuedWhenDeadlineExpiresAfterPut() throws Exception { void completedUploadIsEnqueuedWhenDeadlineExpiresAfterPut() throws Exception {
// 只有版本化 key(本次写入独占)才允许补偿删除;共享 key 的守卫见
// RustfsUploadCompensationGuardTest#deterministicKeyDerivedFromShapeDoesNotEnqueueCompensation
String versionedKey = "task/chunk-1-0ffc254b-afad-4279-aab2-e85e3ff955e9.json";
TransientStorageProperties properties = configuredProperties(); TransientStorageProperties properties = configuredProperties();
properties.setOperationTimeoutSeconds(1); properties.setOperationTimeoutSeconds(1);
RustfsDeleteRetryService retryService = mock(RustfsDeleteRetryService.class); RustfsDeleteRetryService retryService = mock(RustfsDeleteRetryService.class);
@@ -372,11 +378,11 @@ class RustfsObjectStorageServiceTest {
properties, emptyProvider(), provider(retryService), () -> client); properties, emptyProvider(), provider(retryService), () -> client);
IllegalStateException failure = assertThrows(IllegalStateException.class, IllegalStateException failure = assertThrows(IllegalStateException.class,
() -> service.uploadText("task/a.json", "{}", false)); () -> service.uploadText(versionedKey, "{}", false));
assertTrue(failure.getMessage().contains("operation timeout")); assertTrue(failure.getMessage().contains("operation timeout"));
verify(client).putObject(any(PutObjectArgs.class)); verify(client).putObject(any(PutObjectArgs.class));
verify(retryService).enqueue(eq("task/a.json"), same(failure)); verify(retryService).enqueue(eq(versionedKey), same(failure));
} }
private static TransientStorageProperties configuredProperties() { private static TransientStorageProperties configuredProperties() {
@@ -0,0 +1,217 @@
package com.nanri.aiimage.modules.file.service.object;
import com.nanri.aiimage.config.TransientStorageProperties;
import io.minio.MinioClient;
import io.minio.PutObjectArgs;
import io.minio.StatObjectArgs;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.ObjectProvider;
import java.io.IOException;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* 上传失败补偿删除的准入守卫(2026-09-17 线上任务 28459 事故)。
*
* <p>事故形态:客户端重传复用**确定性 key**{@code chunk-462.json})。第一次上传 put 已成功、
* 但随后的 {@code verifyObjectVisible} 抖动 → 旧代码无条件
* {@code enqueueDeleteRetry(objectKey)};而删除重试队列对该 key 不再做任何引用校验
* {@code RustfsDeleteRetryService} → {@code deleteObjectFromRetry} → {@code removeObject}),
* 于是把重传成功后 DB 行仍指向的对象删掉,下游读 chunk 直接 404,整个任务组装失败。
*
* <p>口径:只有**本次写入独占的对象**(版本化 key,末段以 UUID 结尾)才允许补偿删除;
* 确定性 key 会被重写复用,删它就可能删掉别的行正在引用的对象。
*/
class RustfsUploadCompensationGuardTest {
// ===== 正常路径 =====
/** 唯一 key(版本化)+ 验证失败 + 显式允许 → 补偿删除入队。 */
@Test
void uniqueKeyVerifyFailureEnqueuesCompensation() throws Exception {
RustfsDeleteRetryService retryService = mock(RustfsDeleteRetryService.class);
RustfsObjectStorageService service = new RustfsObjectStorageService(
configuredProperties(), emptyProvider(), provider(retryService), RustfsUploadCompensationGuardTest::putOkButStatFails);
assertThrows(IllegalStateException.class,
() -> service.uploadBytes("task/chunk-462-0ffc254b-afad-4279-aab2-e85e3ff955e9.json",
"{}".getBytes(), true, true));
verify(retryService).enqueue(eq("task/chunk-462-0ffc254b-afad-4279-aab2-e85e3ff955e9.json"),
any(Throwable.class));
}
/** 三参重载按 key 形态判定:版本化 key 自动允许补偿删除。 */
@Test
void versionedKeyDerivedFromShapeEnqueuesCompensation() throws Exception {
RustfsDeleteRetryService retryService = mock(RustfsDeleteRetryService.class);
RustfsObjectStorageService service = new RustfsObjectStorageService(
configuredProperties(), emptyProvider(), provider(retryService), RustfsUploadCompensationGuardTest::putOkButStatFails);
assertThrows(IllegalStateException.class,
() -> service.uploadBytes("task/chunk-77-d03bf3b0-abbb-469d-9bb8-a7c01c309477.json",
"{}".getBytes(), true));
verify(retryService).enqueue(anyString(), any(Throwable.class));
}
// ===== 事故根因:共享 key 必须不删 =====
/** 确定性(共享)key + 显式禁止 → 不入队(异常仍照旧抛出,失败归属不变)。 */
@Test
void sharedKeyVerifyFailureDoesNotEnqueueCompensation() throws Exception {
RustfsDeleteRetryService retryService = mock(RustfsDeleteRetryService.class);
RustfsObjectStorageService service = new RustfsObjectStorageService(
configuredProperties(), emptyProvider(), provider(retryService), RustfsUploadCompensationGuardTest::putOkButStatFails);
assertThrows(IllegalStateException.class,
() -> service.uploadBytes("task/chunk-462.json", "{}".getBytes(), true, false));
verify(retryService, never()).enqueue(anyString(), any(Throwable.class));
}
/** 三参重载按 key 形态判定:确定性 key 自动禁止补偿删除(本次事故的直接修复点)。 */
@Test
void deterministicKeyDerivedFromShapeDoesNotEnqueueCompensation() throws Exception {
RustfsDeleteRetryService retryService = mock(RustfsDeleteRetryService.class);
RustfsObjectStorageService service = new RustfsObjectStorageService(
configuredProperties(), emptyProvider(), provider(retryService), RustfsUploadCompensationGuardTest::putOkButStatFails);
assertThrows(IllegalStateException.class,
() -> service.uploadBytes("task/chunk-462.json", "{}".getBytes(), true));
verify(retryService, never()).enqueue(anyString(), any(Throwable.class));
}
/** verify=false 上传即成功,不得产生任何补偿删除。 */
@Test
void verifyDisabledNeverEnqueuesCompensation() throws Exception {
RustfsDeleteRetryService retryService = mock(RustfsDeleteRetryService.class);
RustfsObjectStorageService service = new RustfsObjectStorageService(
configuredProperties(), emptyProvider(), provider(retryService), RustfsUploadCompensationGuardTest::putOkButStatFails);
String key = service.uploadBytes("task/chunk-462-uuid-x.json", "{}".getBytes(), false, true);
assertEquals("task/chunk-462-uuid-x.json", key);
verify(retryService, never()).enqueue(anyString(), any(Throwable.class));
}
// ===== 异常路径 =====
/** put 从未成功过:对象本就不存在,不该产生删除动作(与开关无关)。 */
@Test
void putNeverCompletedDoesNotEnqueueCompensation() throws Exception {
RustfsDeleteRetryService retryService = mock(RustfsDeleteRetryService.class);
MinioClient client = mock(MinioClient.class);
when(client.putObject(any(PutObjectArgs.class))).thenThrow(new IOException("put failed"));
TransientStorageProperties properties = configuredProperties();
properties.setUploadMaxRetries(1);
RustfsObjectStorageService service = new RustfsObjectStorageService(
properties, emptyProvider(), provider(retryService), () -> client);
assertThrows(RuntimeException.class,
() -> service.uploadBytes("task/chunk-462-uuid-x.json", "{}".getBytes(), true, true));
verify(retryService, never()).enqueue(anyString(), any(Throwable.class));
}
/** 补偿删除入队本身失败:不得把上传失败改写成入队异常,原异常必须继续上抛。 */
@Test
void enqueueFailureDoesNotMaskUploadFailure() throws Exception {
RustfsDeleteRetryService retryService = mock(RustfsDeleteRetryService.class);
org.mockito.Mockito.doThrow(new IllegalStateException("queue down"))
.when(retryService).enqueue(anyString(), any(Throwable.class));
RustfsObjectStorageService service = new RustfsObjectStorageService(
configuredProperties(), emptyProvider(), provider(retryService), RustfsUploadCompensationGuardTest::putOkButStatFails);
IllegalStateException ex = assertThrows(IllegalStateException.class,
() -> service.uploadBytes("task/chunk-9-11111111-2222-3333-4444-555555555555.json",
"{}".getBytes(), true, true));
assertFalse(ex.getMessage() == null || ex.getMessage().contains("queue down"),
"入队异常不应成为对外错误;实际=" + ex.getMessage());
}
// ===== key 形态判定的边界 =====
@Test
void versionedKeyShapeIsRecognized() {
assertTrue(RustfsObjectStorageService.isVersionedObjectKey(
"task-chunk/appearance_patent/28459/hash/chunk-462-0ffc254b-afad-4279-aab2-e85e3ff955e9.json"));
assertTrue(RustfsObjectStorageService.isVersionedObjectKey(
"task-chunk/x/1/hash/0ffc254b-afad-4279-aab2-e85e3ff955e9.json"),
"整个 entryKey 就是 UUID 时也应识别为唯一 key");
assertTrue(RustfsObjectStorageService.isVersionedObjectKey(
"task-chunk/x/1/hash/chunk-1-0ffc254b-afad-4279-aab2-e85e3ff955e9"),
"无 .json 后缀但末段是 UUID,同样视为唯一");
}
@Test
void deterministicKeyShapeIsNotRecognized() {
assertFalse(RustfsObjectStorageService.isVersionedObjectKey("task-chunk/x/1/hash/chunk-462.json"));
assertFalse(RustfsObjectStorageService.isVersionedObjectKey("task-parsed/x/1/hash/latest.json"));
assertFalse(RustfsObjectStorageService.isVersionedObjectKey("task-result-payload/x/1/hash/28459.json"),
"提交号这类 key 会被重写复用,不能当成唯一 key");
assertFalse(RustfsObjectStorageService.isVersionedObjectKey(
"task-chunk/x/1/0ffc254b-afad-4279-aab2-e85e3ff955e9/chunk-462.json"),
"UUID 出现在中间段不算唯一——重写复用的判据只看末段");
}
@Test
void malformedKeyShapeDoesNotThrowAndIsTreatedAsShared() {
assertFalse(RustfsObjectStorageService.isVersionedObjectKey(null));
assertFalse(RustfsObjectStorageService.isVersionedObjectKey(""));
assertFalse(RustfsObjectStorageService.isVersionedObjectKey(" "));
assertFalse(RustfsObjectStorageService.isVersionedObjectKey("chunk-462.json"));
assertFalse(RustfsObjectStorageService.isVersionedObjectKey("-afad-4279-aab2-e85e3ff955e9.json".substring(0, 20)));
}
// ===== 辅助 =====
private static MinioClient putOkButStatFails() {
MinioClient client = mock(MinioClient.class);
try {
when(client.putObject(any(PutObjectArgs.class))).thenReturn(null);
when(client.statObject(any(StatObjectArgs.class))).thenThrow(new IOException("stat 抖动"));
} catch (Exception ex) {
throw new IllegalStateException(ex);
}
return client;
}
private static TransientStorageProperties configuredProperties() {
TransientStorageProperties properties = new TransientStorageProperties();
properties.setEndpoint("http://127.0.0.1:9000");
properties.setBucket("bucket");
properties.setAccessKeyId("ak");
properties.setAccessKeySecret("sk");
properties.setBaseRetryDelayMillis(0);
properties.setRetryJitterMillis(0);
return properties;
}
private static <T> ObjectProvider<T> provider(T value) {
@SuppressWarnings("unchecked")
ObjectProvider<T> provider = mock(ObjectProvider.class);
when(provider.getIfAvailable()).thenReturn(value);
return provider;
}
private static <T> ObjectProvider<T> emptyProvider() {
@SuppressWarnings("unchecked")
ObjectProvider<T> provider = mock(ObjectProvider.class);
when(provider.getIfAvailable()).thenReturn(null);
return provider;
}
}
@@ -33,8 +33,13 @@ import static org.mockito.Mockito.when;
/** /**
* task-150:清理前引用检查契约(plan 09)。 * task-150:清理前引用检查契约(plan 09)。
* payload 被 biz_task_chunk / biz_task_scope_state 引用则不清理; * payload 被 biz_task_chunk / biz_task_scope_state 引用则不清理;
* chunk 仅剩自身一行(count=1)不算共享引用;查询异常保守保留; * chunk 命中任意行(含仅剩 1 行)都不算「可安全清理」;查询异常保守保留;
* 检查只读无副作用;候选值批量反查。 * 检查只读无副作用;候选值批量反查。
*
* <p>2026-09-17 线上任务 28459 之后收紧:原契约把 count==1 当作「调用方自己那行」放行删除,
* 但调用方无法证明那一行就是自己(如合并已把自己那行指向新对象、另一行仍指向旧对象时 count 恰为 1),
* 删除即造成该分片永久读不到 → 整单失败。物理删除本就约定在 DB 行删除之后执行,正常路径引用数必然为 0;
* 宁可留下孤儿对象(有保留期清理兜底),也不删掉可能仍被引用的对象。
*/ */
@ExtendWith(MockitoExtension.class) @ExtendWith(MockitoExtension.class)
class TaskPayloadReferenceCheckTest { class TaskPayloadReferenceCheckTest {
@@ -100,13 +105,14 @@ class TaskPayloadReferenceCheckTest {
verify(rustfsObjectStorageService, never()).deleteObject(any()); verify(rustfsObjectStorageService, never()).deleteObject(any());
} }
/** count==1 无法证明那一行就是调用方自己(可能正是别的行仍在用)→ 保守不删。 */
@Test @Test
void singleChunkRowIsOwnRowNotSharedReference() { void singleChunkRowBlocksDeleteBecauseOwnershipCannotBeProven() {
when(taskChunkMapper.selectCount(any())).thenReturn(1L); when(taskChunkMapper.selectCount(any())).thenReturn(1L);
service.deletePayloadIfPresent(RUSTFS_VALUE); service.deletePayloadIfPresent(RUSTFS_VALUE);
verify(rustfsObjectStorageService).deleteObject("payload-key"); verify(rustfsObjectStorageService, never()).deleteObject(any());
} }
@Test @Test
@@ -0,0 +1,230 @@
package com.nanri.aiimage.modules.task.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.InstanceMetadata;
import com.nanri.aiimage.config.StorageProperties;
import com.nanri.aiimage.config.TransientStorageProperties;
import com.nanri.aiimage.modules.file.service.object.RustfsObjectStorageService;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import org.junit.jupiter.api.Test;
import java.nio.charset.StandardCharsets;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* chunk 载荷对象缺失时的「同槽位版本化兄弟对象」读兜底(2026-09-17 线上任务 28459)。
*
* <p>事故后 DB 行指向的普通 key{@code chunk-462.json})已被误删,但同槽位的版本化对象
* {@code chunk-462-<uuid>.json})还在。兜底让读路径自动改用兄弟对象,任务即可按已有数据出结果,
* 而不是整单 FAILED——这是防止「删错对象 → 任务永久失败」的最后一道网。
*/
class TransientPayloadChunkSiblingFallbackTest {
private static final String DIR = "task-chunk/appearance_patent/28459/"
+ "2248d39710545b47b7c7035fc60c4e33924a16174abf65dd1c5a230918e6f61e/";
private static final String CHUNK_KEY = DIR + "chunk-462.json";
private static final String CHUNK_POINTER = "rustfs:" + CHUNK_KEY;
private static final String SIBLING_KEY = DIR + "chunk-462-0ffc254b-afad-4279-aab2-e85e3ff955e9.json";
// ===== 正常路径 =====
/** 对象在:直接读,不得触发列对象(正常读路径不能因为兜底变慢)。 */
@Test
void presentObjectIsReadWithoutListing() {
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
when(rustfs.readObjectBytes(CHUNK_KEY)).thenReturn(bytes("[\"原始\"]"));
String payload = service(rustfs).resolvePayload(CHUNK_POINTER, "read chunk failed");
assertEquals("[\"原始\"]", payload);
verify(rustfs, never()).listObjectKeysNewestFirst(anyString(), anyInt());
}
/** 对象缺失 + 存在兄弟对象 → 读兄弟对象(本次事故的自动恢复路径)。 */
@Test
void missingChunkObjectFallsBackToVersionedSibling() {
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
when(rustfs.readObjectBytes(CHUNK_KEY)).thenThrow(missingKey());
when(rustfs.listObjectKeysNewestFirst(DIR + "chunk-462-", 50)).thenReturn(List.of(SIBLING_KEY));
when(rustfs.readObjectBytes(SIBLING_KEY)).thenReturn(bytes("[\"合并后\"]"));
String payload = service(rustfs).resolvePayload(CHUNK_POINTER, "read chunk failed");
assertEquals("[\"合并后\"]", payload);
verify(rustfs).listObjectKeysNewestFirst(DIR + "chunk-462-", 50);
}
/** 多个兄弟对象:取列表首个(实现约定按最后修改时间倒序,最新在前)。 */
@Test
void newestSiblingIsPreferred() {
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
String newest = DIR + "chunk-462-ffffffff-ffff-ffff-ffff-ffffffffffff.json";
when(rustfs.readObjectBytes(CHUNK_KEY)).thenThrow(missingKey());
when(rustfs.listObjectKeysNewestFirst(anyString(), anyInt())).thenReturn(List.of(newest, SIBLING_KEY));
when(rustfs.readObjectBytes(newest)).thenReturn(bytes("[\"最新\"]"));
assertEquals("[\"最新\"]", service(rustfs).resolvePayload(CHUNK_POINTER, "read chunk failed"));
verify(rustfs).readObjectBytes(newest);
verify(rustfs, never()).readObjectBytes(SIBLING_KEY);
}
/** 指针是 JSON 编码形式(DB 里的常见写法)时兜底同样生效。 */
@Test
void jsonQuotedPointerStillFallsBack() {
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
when(rustfs.readObjectBytes(CHUNK_KEY)).thenThrow(missingKey());
when(rustfs.listObjectKeysNewestFirst(anyString(), anyInt())).thenReturn(List.of(SIBLING_KEY));
when(rustfs.readObjectBytes(SIBLING_KEY)).thenReturn(bytes("[\"带引号\"]"));
assertEquals("[\"带引号\"]",
service(rustfs).resolvePayload('"' + CHUNK_POINTER + '"', "read chunk failed"));
}
/** 非指针内容(直接就是载荷 JSON)原样返回,不触碰对象存储。 */
@Test
void rawContentIsReturnedAsIs() {
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
assertEquals("[1,2,3]", service(rustfs).resolvePayload("[1,2,3]", "read chunk failed"));
verify(rustfs, never()).readObjectBytes(anyString());
}
// ===== 边界:不该兜底的场景 =====
/** 非 chunk 槽位(如 row-1.json)缺失 → 不兜底,照旧抛错(避免误配无关对象)。 */
@Test
void nonChunkSlotDoesNotFallBack() {
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
String key = "task-result-item/similar-asin/9/hash/row-1.json";
when(rustfs.readObjectBytes(key)).thenThrow(missingKey());
assertThrows(IllegalStateException.class,
() -> service(rustfs).resolvePayload("rustfs:" + key, "read item failed"));
verify(rustfs, never()).listObjectKeysNewestFirst(anyString(), anyInt());
}
/** parsed 槽位(latest.json)缺失 → 不兜底(该槽位没有版本化兄弟语义)。 */
@Test
void parsedSlotDoesNotFallBack() {
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
String key = "task-parsed/appearance_patent/28459/hash/latest.json";
when(rustfs.readObjectBytes(key)).thenThrow(missingKey());
assertThrows(IllegalStateException.class,
() -> service(rustfs).resolvePayload("rustfs:" + key, "read parsed failed"));
verify(rustfs, never()).listObjectKeysNewestFirst(anyString(), anyInt());
}
/** 非缺失类失败(权限/网络)→ 不兜底,照旧上抛以便重试。 */
@Test
void nonMissingFailureDoesNotFallBack() {
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
when(rustfs.readObjectBytes(CHUNK_KEY))
.thenThrow(new IllegalStateException("Access Denied"));
assertThrows(IllegalStateException.class,
() -> service(rustfs).resolvePayload(CHUNK_POINTER, "read chunk failed"));
verify(rustfs, never()).listObjectKeysNewestFirst(anyString(), anyInt());
}
// ===== 异常路径 =====
/** 没有兄弟对象 → 抛错,且错误信息必须保留原 errorMessage 与原指针(便于定位)。 */
@Test
void noSiblingRethrowsWithOriginalPointer() {
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
when(rustfs.readObjectBytes(CHUNK_KEY)).thenThrow(missingKey());
when(rustfs.listObjectKeysNewestFirst(anyString(), anyInt())).thenReturn(List.of());
IllegalStateException ex = assertThrows(IllegalStateException.class,
() -> service(rustfs).resolvePayload(CHUNK_POINTER, "read chunk failed"));
assertTrue(ex.getMessage().contains("read chunk failed"), "错误信息=" + ex.getMessage());
assertTrue(ex.getMessage().contains("chunk-462.json"), "错误信息=" + ex.getMessage());
}
/** 兄弟对象读取也失败 → 异常上抛,不得静默返回空内容。 */
@Test
void siblingReadFailurePropagates() {
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
when(rustfs.readObjectBytes(CHUNK_KEY)).thenThrow(missingKey());
when(rustfs.listObjectKeysNewestFirst(anyString(), anyInt())).thenReturn(List.of(SIBLING_KEY));
when(rustfs.readObjectBytes(SIBLING_KEY)).thenThrow(new IllegalStateException("read timeout"));
IllegalStateException ex = assertThrows(IllegalStateException.class,
() -> service(rustfs).resolvePayload(CHUNK_POINTER, "read chunk failed"));
assertTrue(ex.getMessage().contains("read chunk failed"), "错误信息=" + ex.getMessage());
}
/** 列对象本身失败(桶不可用)→ 仍按原错误抛出,错误信息不含兄弟查找细节。 */
@Test
void listFailureDoesNotMaskOriginalError() {
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
when(rustfs.readObjectBytes(CHUNK_KEY)).thenThrow(missingKey());
when(rustfs.listObjectKeysNewestFirst(anyString(), anyInt()))
.thenThrow(new IllegalStateException("bucket unavailable"));
IllegalStateException ex = assertThrows(IllegalStateException.class,
() -> service(rustfs).resolvePayload(CHUNK_POINTER, "read chunk failed"));
assertTrue(ex.getMessage().contains("read chunk failed"), "错误信息=" + ex.getMessage());
}
/** local 指针不走 rustfs 兄弟兜底。 */
@Test
void localPointerDoesNotUseChunkSiblingFallback() {
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
assertThrows(IllegalStateException.class,
() -> service(rustfs).resolvePayload("local:i/server-121/" + CHUNK_KEY, "read chunk failed"));
verify(rustfs, never()).listObjectKeysNewestFirst(anyString(), anyInt());
}
// ===== 辅助 =====
private static IllegalStateException missingKey() {
return new IllegalStateException("The specified key does not exist.");
}
private static byte[] bytes(String value) {
return value.getBytes(StandardCharsets.UTF_8);
}
private TransientPayloadStorageService service(RustfsObjectStorageService rustfs) {
TransientStorageProperties transientProperties = new TransientStorageProperties();
transientProperties.setEnabled(true);
transientProperties.setEndpoint("http://127.0.0.1:9000");
transientProperties.setBucket("json-server");
transientProperties.setAccessKeyId("ak");
transientProperties.setAccessKeySecret("sk");
StorageProperties storageProperties = new StorageProperties();
storageProperties.setLocalTempDir(System.getProperty("java.io.tmpdir"));
return new TransientPayloadStorageService(
transientProperties,
storageProperties,
rustfs,
mock(OssStorageService.class),
new ObjectMapper(),
new InstanceMetadata("server-121"),
mock(TaskChunkMapper.class),
mock(TaskScopeStateMapper.class));
}
}
@@ -0,0 +1,180 @@
package com.nanri.aiimage.modules.task.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.InstanceMetadata;
import com.nanri.aiimage.config.StorageProperties;
import com.nanri.aiimage.config.TransientStorageProperties;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.file.service.object.RustfsObjectStorageService;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* 临时载荷删除前的引用守卫(2026-09-17 线上任务 28459 事故的第二条同族缺陷)。
*
* <p>守卫本意是「还有别的行引用同一 pointer 就不物理删」。原实现用
* {@code chunkCount > 1} 作判据,等价于「恰好还有 1 行引用时可以删」——
* 这在 caller 已经把自己那行挪走、但**另一行**仍指向旧对象的场景下,会把
* 对方仍在用的对象删掉。docs 里 CollectDataDeleteConsistencyTest 已明确口径是
* 「跨任务共享指针仍引用 → 跳过删除」,故判据收紧为「命中任意一行即不删」。
*/
class TransientPayloadDeleteReferenceGuardTest {
private static final String RUSTFS_CHUNK_POINTER =
"rustfs:task-chunk/appearance_patent/28459/2248d39710545b47b7c7035fc60c4e33924a16174abf65dd1c5a230918e6f61e/chunk-462.json";
// ===== 正常路径 =====
/** 没有任何行引用 → 允许物理删除。 */
@Test
void zeroReferencesAllowsPhysicalDelete() {
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
TaskChunkMapper chunkMapper = mock(TaskChunkMapper.class);
TaskScopeStateMapper scopeMapper = mock(TaskScopeStateMapper.class);
when(chunkMapper.selectCount(any())).thenReturn(0L);
when(scopeMapper.selectCount(any())).thenReturn(0L);
service(rustfs, chunkMapper, scopeMapper).deletePayloadIfPresent(RUSTFS_CHUNK_POINTER);
verify(rustfs).deleteObject(eq("task-chunk/appearance_patent/28459/2248d39710545b47b7c7035fc60c4e33924a16174abf65dd1c5a230918e6f61e/chunk-462.json"));
}
/** 多条行引用 → 不删(原有行为必须保持)。 */
@Test
void multipleReferencesBlockDelete() {
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
TaskChunkMapper chunkMapper = mock(TaskChunkMapper.class);
when(chunkMapper.selectCount(any())).thenReturn(3L);
service(rustfs, chunkMapper, mock(TaskScopeStateMapper.class)).deletePayloadIfPresent(RUSTFS_CHUNK_POINTER);
verify(rustfs, never()).deleteObject(anyString());
}
// ===== 本次修复点 =====
/** 恰好 1 行仍引用 → 必须不删(原实现会删掉这唯一一行仍在用的对象)。 */
@Test
void singleReferencingChunkRowBlocksDelete() {
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
TaskChunkMapper chunkMapper = mock(TaskChunkMapper.class);
when(chunkMapper.selectCount(any())).thenReturn(1L);
service(rustfs, chunkMapper, mock(TaskScopeStateMapper.class)).deletePayloadIfPresent(RUSTFS_CHUNK_POINTER);
verify(rustfs, never()).deleteObject(anyString());
}
/** scope_state 单行引用同样阻断删除(该字段命中即非自我引用)。 */
@Test
void singleScopeStateReferenceBlocksDelete() {
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
TaskChunkMapper chunkMapper = mock(TaskChunkMapper.class);
TaskScopeStateMapper scopeMapper = mock(TaskScopeStateMapper.class);
when(chunkMapper.selectCount(any())).thenReturn(0L);
when(scopeMapper.selectCount(any())).thenReturn(1L);
service(rustfs, chunkMapper, scopeMapper).deletePayloadIfPresent(RUSTFS_CHUNK_POINTER);
verify(rustfs, never()).deleteObject(anyString());
}
// ===== 边界 =====
/** 传入 JSON 编码(带引号)的指针——DB 里常见写法,必须能解析并同样受守卫约束。 */
@Test
void jsonQuotedPointerIsParsedAndStillGuarded() {
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
TaskChunkMapper chunkMapper = mock(TaskChunkMapper.class);
when(chunkMapper.selectCount(any())).thenReturn(1L);
service(rustfs, chunkMapper, mock(TaskScopeStateMapper.class))
.deletePayloadIfPresent('"' + RUSTFS_CHUNK_POINTER + '"');
verify(rustfs, never()).deleteObject(anyString());
}
/** 非指针值原样传入 → 不解析、不查库、不删。 */
@Test
void nonPointerValueIsIgnoredWithoutLookup() {
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
TaskChunkMapper chunkMapper = mock(TaskChunkMapper.class);
TransientPayloadStorageService service = service(rustfs, chunkMapper, mock(TaskScopeStateMapper.class));
assertDoesNotThrow(() -> service.deletePayloadIfPresent("{\"rows\":[1,2,3]}"));
verify(chunkMapper, never()).selectCount(any());
verify(rustfs, never()).deleteObject(anyString());
}
/** 空值不得抛异常、不得触发任何删除。 */
@Test
void nullOrBlankValueIsIgnored() {
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
TaskChunkMapper chunkMapper = mock(TaskChunkMapper.class);
TransientPayloadStorageService service = service(rustfs, chunkMapper, mock(TaskScopeStateMapper.class));
assertDoesNotThrow(() -> service.deletePayloadIfPresent(null));
assertDoesNotThrow(() -> service.deletePayloadIfPresent(" "));
verify(chunkMapper, never()).selectCount(any());
verify(rustfs, never()).deleteObject(anyString());
}
// ===== 异常路径 =====
/** 反查引用的查询抛异常 → 保守不删(原实现即如此,必须保持)。 */
@Test
void referenceQueryFailureIsConservativeAndSkipsDelete() {
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
TaskChunkMapper chunkMapper = mock(TaskChunkMapper.class);
when(chunkMapper.selectCount(any())).thenThrow(new IllegalStateException("db down"));
assertDoesNotThrow(() ->
service(rustfs, chunkMapper, mock(TaskScopeStateMapper.class))
.deletePayloadIfPresent(RUSTFS_CHUNK_POINTER));
verify(rustfs, never()).deleteObject(anyString());
}
/** 跨实例的 local 指针:不物理删(留给持有实例的清理任务)。 */
@Test
void crossInstanceLocalPointerIsNotDeleted() {
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
TaskChunkMapper chunkMapper = mock(TaskChunkMapper.class);
when(chunkMapper.selectCount(any())).thenReturn(0L);
service(rustfs, chunkMapper, mock(TaskScopeStateMapper.class))
.deletePayloadIfPresent("local:i/server-110/task-chunk/x/1/hash/chunk-1.json");
verify(rustfs, never()).deleteObject(anyString());
}
private TransientPayloadStorageService service(RustfsObjectStorageService rustfs,
TaskChunkMapper chunkMapper,
TaskScopeStateMapper scopeMapper) {
TransientStorageProperties transientProperties = new TransientStorageProperties();
transientProperties.setEnabled(true);
StorageProperties storageProperties = new StorageProperties();
storageProperties.setLocalTempDir(System.getProperty("java.io.tmpdir"));
return new TransientPayloadStorageService(
transientProperties,
storageProperties,
rustfs,
mock(OssStorageService.class),
new ObjectMapper(),
new InstanceMetadata("server-121"),
chunkMapper,
scopeMapper);
}
}