fix(临时载荷): 收口删除守卫 + chunk 读兜底,修「对象被误删→任务永久失败」

线上任务 28459(外观专利)三个分片的载荷对象被删、DB 行仍指向已删的确定性
key,组装读不到 → 整单 FAILED。数据其实还在同槽位的版本化对象里,是行指错了。

三条收口:
- 上传补偿删除加准入:只有末段带 UUID 的版本化 key(本次写入独占)才允许入队。
  确定性 key 会被重传复用,删它就删掉了行仍在引用的对象;而删除队列
  (deleteObjectFromRetry -> removeObject)本身不做引用反查,准入必须卡在入队处。
- 引用守卫阈值 >1 收紧为 >0:原来把「恰好 1 行引用」当作调用方自己那行而放行,
  但调用方无法证明归属。物理删除本就约定在 DB 行删除之后执行,正常路径引用数
  必为 0;宁可留孤儿(有保留期清理兜底),也不删可能仍被引用的对象。
- chunk 读兜底:指针对象确已缺失(NoSuchKey)时回退同槽位版本化对象 chunk-N-*。
  仅限 chunk 槽位 + 确为缺失两个条件,避免误配无关对象或掩盖真实故障。

测试:新增 31 个用例(补偿删除准入 / 引用守卫 / 兄弟对象兜底),均先确认 RED
再实现;另更新 3 个断言旧行为的既有用例。全量 mvn test 3082 个 0 失败。
This commit is contained in:
2026-09-17 10:42:53 +08:00
parent a75b843937
commit c9be8f87d4
7 changed files with 812 additions and 15 deletions
@@ -33,8 +33,13 @@ import static org.mockito.Mockito.when;
/**
* task-150:清理前引用检查契约(plan 09)。
* payload 被 biz_task_chunk / biz_task_scope_state 引用则不清理;
* chunk 仅剩自身一行(count=1)不算共享引用;查询异常保守保留;
* chunk 命中任意行(含仅剩 1 行)都不算「可安全清理」;查询异常保守保留;
* 检查只读无副作用;候选值批量反查。
*
* <p>2026-09-17 线上任务 28459 之后收紧:原契约把 count==1 当作「调用方自己那行」放行删除,
* 但调用方无法证明那一行就是自己(如合并已把自己那行指向新对象、另一行仍指向旧对象时 count 恰为 1),
* 删除即造成该分片永久读不到 → 整单失败。物理删除本就约定在 DB 行删除之后执行,正常路径引用数必然为 0;
* 宁可留下孤儿对象(有保留期清理兜底),也不删掉可能仍被引用的对象。
*/
@ExtendWith(MockitoExtension.class)
class TaskPayloadReferenceCheckTest {
@@ -100,13 +105,14 @@ class TaskPayloadReferenceCheckTest {
verify(rustfsObjectStorageService, never()).deleteObject(any());
}
/** count==1 无法证明那一行就是调用方自己(可能正是别的行仍在用)→ 保守不删。 */
@Test
void singleChunkRowIsOwnRowNotSharedReference() {
void singleChunkRowBlocksDeleteBecauseOwnershipCannotBeProven() {
when(taskChunkMapper.selectCount(any())).thenReturn(1L);
service.deletePayloadIfPresent(RUSTFS_VALUE);
verify(rustfsObjectStorageService).deleteObject("payload-key");
verify(rustfsObjectStorageService, never()).deleteObject(any());
}
@Test
@@ -0,0 +1,230 @@
package com.nanri.aiimage.modules.task.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.InstanceMetadata;
import com.nanri.aiimage.config.StorageProperties;
import com.nanri.aiimage.config.TransientStorageProperties;
import com.nanri.aiimage.modules.file.service.object.RustfsObjectStorageService;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import org.junit.jupiter.api.Test;
import java.nio.charset.StandardCharsets;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* chunk 载荷对象缺失时的「同槽位版本化兄弟对象」读兜底(2026-09-17 线上任务 28459)。
*
* <p>事故后 DB 行指向的普通 key({@code chunk-462.json})已被误删,但同槽位的版本化对象
* ({@code chunk-462-<uuid>.json})还在。兜底让读路径自动改用兄弟对象,任务即可按已有数据出结果,
* 而不是整单 FAILED——这是防止「删错对象 → 任务永久失败」的最后一道网。
*/
class TransientPayloadChunkSiblingFallbackTest {
private static final String DIR = "task-chunk/appearance_patent/28459/"
+ "2248d39710545b47b7c7035fc60c4e33924a16174abf65dd1c5a230918e6f61e/";
private static final String CHUNK_KEY = DIR + "chunk-462.json";
private static final String CHUNK_POINTER = "rustfs:" + CHUNK_KEY;
private static final String SIBLING_KEY = DIR + "chunk-462-0ffc254b-afad-4279-aab2-e85e3ff955e9.json";
// ===== 正常路径 =====
/** 对象在:直接读,不得触发列对象(正常读路径不能因为兜底变慢)。 */
@Test
void presentObjectIsReadWithoutListing() {
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
when(rustfs.readObjectBytes(CHUNK_KEY)).thenReturn(bytes("[\"原始\"]"));
String payload = service(rustfs).resolvePayload(CHUNK_POINTER, "read chunk failed");
assertEquals("[\"原始\"]", payload);
verify(rustfs, never()).listObjectKeysNewestFirst(anyString(), anyInt());
}
/** 对象缺失 + 存在兄弟对象 → 读兄弟对象(本次事故的自动恢复路径)。 */
@Test
void missingChunkObjectFallsBackToVersionedSibling() {
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
when(rustfs.readObjectBytes(CHUNK_KEY)).thenThrow(missingKey());
when(rustfs.listObjectKeysNewestFirst(DIR + "chunk-462-", 50)).thenReturn(List.of(SIBLING_KEY));
when(rustfs.readObjectBytes(SIBLING_KEY)).thenReturn(bytes("[\"合并后\"]"));
String payload = service(rustfs).resolvePayload(CHUNK_POINTER, "read chunk failed");
assertEquals("[\"合并后\"]", payload);
verify(rustfs).listObjectKeysNewestFirst(DIR + "chunk-462-", 50);
}
/** 多个兄弟对象:取列表首个(实现约定按最后修改时间倒序,最新在前)。 */
@Test
void newestSiblingIsPreferred() {
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
String newest = DIR + "chunk-462-ffffffff-ffff-ffff-ffff-ffffffffffff.json";
when(rustfs.readObjectBytes(CHUNK_KEY)).thenThrow(missingKey());
when(rustfs.listObjectKeysNewestFirst(anyString(), anyInt())).thenReturn(List.of(newest, SIBLING_KEY));
when(rustfs.readObjectBytes(newest)).thenReturn(bytes("[\"最新\"]"));
assertEquals("[\"最新\"]", service(rustfs).resolvePayload(CHUNK_POINTER, "read chunk failed"));
verify(rustfs).readObjectBytes(newest);
verify(rustfs, never()).readObjectBytes(SIBLING_KEY);
}
/** 指针是 JSON 编码形式(DB 里的常见写法)时兜底同样生效。 */
@Test
void jsonQuotedPointerStillFallsBack() {
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
when(rustfs.readObjectBytes(CHUNK_KEY)).thenThrow(missingKey());
when(rustfs.listObjectKeysNewestFirst(anyString(), anyInt())).thenReturn(List.of(SIBLING_KEY));
when(rustfs.readObjectBytes(SIBLING_KEY)).thenReturn(bytes("[\"带引号\"]"));
assertEquals("[\"带引号\"]",
service(rustfs).resolvePayload('"' + CHUNK_POINTER + '"', "read chunk failed"));
}
/** 非指针内容(直接就是载荷 JSON)原样返回,不触碰对象存储。 */
@Test
void rawContentIsReturnedAsIs() {
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
assertEquals("[1,2,3]", service(rustfs).resolvePayload("[1,2,3]", "read chunk failed"));
verify(rustfs, never()).readObjectBytes(anyString());
}
// ===== 边界:不该兜底的场景 =====
/** 非 chunk 槽位(如 row-1.json)缺失 → 不兜底,照旧抛错(避免误配无关对象)。 */
@Test
void nonChunkSlotDoesNotFallBack() {
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
String key = "task-result-item/similar-asin/9/hash/row-1.json";
when(rustfs.readObjectBytes(key)).thenThrow(missingKey());
assertThrows(IllegalStateException.class,
() -> service(rustfs).resolvePayload("rustfs:" + key, "read item failed"));
verify(rustfs, never()).listObjectKeysNewestFirst(anyString(), anyInt());
}
/** parsed 槽位(latest.json)缺失 → 不兜底(该槽位没有版本化兄弟语义)。 */
@Test
void parsedSlotDoesNotFallBack() {
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
String key = "task-parsed/appearance_patent/28459/hash/latest.json";
when(rustfs.readObjectBytes(key)).thenThrow(missingKey());
assertThrows(IllegalStateException.class,
() -> service(rustfs).resolvePayload("rustfs:" + key, "read parsed failed"));
verify(rustfs, never()).listObjectKeysNewestFirst(anyString(), anyInt());
}
/** 非缺失类失败(权限/网络)→ 不兜底,照旧上抛以便重试。 */
@Test
void nonMissingFailureDoesNotFallBack() {
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
when(rustfs.readObjectBytes(CHUNK_KEY))
.thenThrow(new IllegalStateException("Access Denied"));
assertThrows(IllegalStateException.class,
() -> service(rustfs).resolvePayload(CHUNK_POINTER, "read chunk failed"));
verify(rustfs, never()).listObjectKeysNewestFirst(anyString(), anyInt());
}
// ===== 异常路径 =====
/** 没有兄弟对象 → 抛错,且错误信息必须保留原 errorMessage 与原指针(便于定位)。 */
@Test
void noSiblingRethrowsWithOriginalPointer() {
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
when(rustfs.readObjectBytes(CHUNK_KEY)).thenThrow(missingKey());
when(rustfs.listObjectKeysNewestFirst(anyString(), anyInt())).thenReturn(List.of());
IllegalStateException ex = assertThrows(IllegalStateException.class,
() -> service(rustfs).resolvePayload(CHUNK_POINTER, "read chunk failed"));
assertTrue(ex.getMessage().contains("read chunk failed"), "错误信息=" + ex.getMessage());
assertTrue(ex.getMessage().contains("chunk-462.json"), "错误信息=" + ex.getMessage());
}
/** 兄弟对象读取也失败 → 异常上抛,不得静默返回空内容。 */
@Test
void siblingReadFailurePropagates() {
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
when(rustfs.readObjectBytes(CHUNK_KEY)).thenThrow(missingKey());
when(rustfs.listObjectKeysNewestFirst(anyString(), anyInt())).thenReturn(List.of(SIBLING_KEY));
when(rustfs.readObjectBytes(SIBLING_KEY)).thenThrow(new IllegalStateException("read timeout"));
IllegalStateException ex = assertThrows(IllegalStateException.class,
() -> service(rustfs).resolvePayload(CHUNK_POINTER, "read chunk failed"));
assertTrue(ex.getMessage().contains("read chunk failed"), "错误信息=" + ex.getMessage());
}
/** 列对象本身失败(桶不可用)→ 仍按原错误抛出,错误信息不含兄弟查找细节。 */
@Test
void listFailureDoesNotMaskOriginalError() {
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
when(rustfs.readObjectBytes(CHUNK_KEY)).thenThrow(missingKey());
when(rustfs.listObjectKeysNewestFirst(anyString(), anyInt()))
.thenThrow(new IllegalStateException("bucket unavailable"));
IllegalStateException ex = assertThrows(IllegalStateException.class,
() -> service(rustfs).resolvePayload(CHUNK_POINTER, "read chunk failed"));
assertTrue(ex.getMessage().contains("read chunk failed"), "错误信息=" + ex.getMessage());
}
/** local 指针不走 rustfs 兄弟兜底。 */
@Test
void localPointerDoesNotUseChunkSiblingFallback() {
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
assertThrows(IllegalStateException.class,
() -> service(rustfs).resolvePayload("local:i/server-121/" + CHUNK_KEY, "read chunk failed"));
verify(rustfs, never()).listObjectKeysNewestFirst(anyString(), anyInt());
}
// ===== 辅助 =====
private static IllegalStateException missingKey() {
return new IllegalStateException("The specified key does not exist.");
}
private static byte[] bytes(String value) {
return value.getBytes(StandardCharsets.UTF_8);
}
private TransientPayloadStorageService service(RustfsObjectStorageService rustfs) {
TransientStorageProperties transientProperties = new TransientStorageProperties();
transientProperties.setEnabled(true);
transientProperties.setEndpoint("http://127.0.0.1:9000");
transientProperties.setBucket("json-server");
transientProperties.setAccessKeyId("ak");
transientProperties.setAccessKeySecret("sk");
StorageProperties storageProperties = new StorageProperties();
storageProperties.setLocalTempDir(System.getProperty("java.io.tmpdir"));
return new TransientPayloadStorageService(
transientProperties,
storageProperties,
rustfs,
mock(OssStorageService.class),
new ObjectMapper(),
new InstanceMetadata("server-121"),
mock(TaskChunkMapper.class),
mock(TaskScopeStateMapper.class));
}
}
@@ -0,0 +1,180 @@
package com.nanri.aiimage.modules.task.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.config.InstanceMetadata;
import com.nanri.aiimage.config.StorageProperties;
import com.nanri.aiimage.config.TransientStorageProperties;
import com.nanri.aiimage.modules.file.service.oss.OssStorageService;
import com.nanri.aiimage.modules.file.service.object.RustfsObjectStorageService;
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* 临时载荷删除前的引用守卫(2026-09-17 线上任务 28459 事故的第二条同族缺陷)。
*
* <p>守卫本意是「还有别的行引用同一 pointer 就不物理删」。原实现用
* {@code chunkCount > 1} 作判据,等价于「恰好还有 1 行引用时可以删」——
* 这在 caller 已经把自己那行挪走、但**另一行**仍指向旧对象的场景下,会把
* 对方仍在用的对象删掉。docs 里 CollectDataDeleteConsistencyTest 已明确口径是
* 「跨任务共享指针仍引用 → 跳过删除」,故判据收紧为「命中任意一行即不删」。
*/
class TransientPayloadDeleteReferenceGuardTest {
private static final String RUSTFS_CHUNK_POINTER =
"rustfs:task-chunk/appearance_patent/28459/2248d39710545b47b7c7035fc60c4e33924a16174abf65dd1c5a230918e6f61e/chunk-462.json";
// ===== 正常路径 =====
/** 没有任何行引用 → 允许物理删除。 */
@Test
void zeroReferencesAllowsPhysicalDelete() {
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
TaskChunkMapper chunkMapper = mock(TaskChunkMapper.class);
TaskScopeStateMapper scopeMapper = mock(TaskScopeStateMapper.class);
when(chunkMapper.selectCount(any())).thenReturn(0L);
when(scopeMapper.selectCount(any())).thenReturn(0L);
service(rustfs, chunkMapper, scopeMapper).deletePayloadIfPresent(RUSTFS_CHUNK_POINTER);
verify(rustfs).deleteObject(eq("task-chunk/appearance_patent/28459/2248d39710545b47b7c7035fc60c4e33924a16174abf65dd1c5a230918e6f61e/chunk-462.json"));
}
/** 多条行引用 → 不删(原有行为必须保持)。 */
@Test
void multipleReferencesBlockDelete() {
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
TaskChunkMapper chunkMapper = mock(TaskChunkMapper.class);
when(chunkMapper.selectCount(any())).thenReturn(3L);
service(rustfs, chunkMapper, mock(TaskScopeStateMapper.class)).deletePayloadIfPresent(RUSTFS_CHUNK_POINTER);
verify(rustfs, never()).deleteObject(anyString());
}
// ===== 本次修复点 =====
/** 恰好 1 行仍引用 → 必须不删(原实现会删掉这唯一一行仍在用的对象)。 */
@Test
void singleReferencingChunkRowBlocksDelete() {
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
TaskChunkMapper chunkMapper = mock(TaskChunkMapper.class);
when(chunkMapper.selectCount(any())).thenReturn(1L);
service(rustfs, chunkMapper, mock(TaskScopeStateMapper.class)).deletePayloadIfPresent(RUSTFS_CHUNK_POINTER);
verify(rustfs, never()).deleteObject(anyString());
}
/** scope_state 单行引用同样阻断删除(该字段命中即非自我引用)。 */
@Test
void singleScopeStateReferenceBlocksDelete() {
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
TaskChunkMapper chunkMapper = mock(TaskChunkMapper.class);
TaskScopeStateMapper scopeMapper = mock(TaskScopeStateMapper.class);
when(chunkMapper.selectCount(any())).thenReturn(0L);
when(scopeMapper.selectCount(any())).thenReturn(1L);
service(rustfs, chunkMapper, scopeMapper).deletePayloadIfPresent(RUSTFS_CHUNK_POINTER);
verify(rustfs, never()).deleteObject(anyString());
}
// ===== 边界 =====
/** 传入 JSON 编码(带引号)的指针——DB 里常见写法,必须能解析并同样受守卫约束。 */
@Test
void jsonQuotedPointerIsParsedAndStillGuarded() {
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
TaskChunkMapper chunkMapper = mock(TaskChunkMapper.class);
when(chunkMapper.selectCount(any())).thenReturn(1L);
service(rustfs, chunkMapper, mock(TaskScopeStateMapper.class))
.deletePayloadIfPresent('"' + RUSTFS_CHUNK_POINTER + '"');
verify(rustfs, never()).deleteObject(anyString());
}
/** 非指针值原样传入 → 不解析、不查库、不删。 */
@Test
void nonPointerValueIsIgnoredWithoutLookup() {
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
TaskChunkMapper chunkMapper = mock(TaskChunkMapper.class);
TransientPayloadStorageService service = service(rustfs, chunkMapper, mock(TaskScopeStateMapper.class));
assertDoesNotThrow(() -> service.deletePayloadIfPresent("{\"rows\":[1,2,3]}"));
verify(chunkMapper, never()).selectCount(any());
verify(rustfs, never()).deleteObject(anyString());
}
/** 空值不得抛异常、不得触发任何删除。 */
@Test
void nullOrBlankValueIsIgnored() {
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
TaskChunkMapper chunkMapper = mock(TaskChunkMapper.class);
TransientPayloadStorageService service = service(rustfs, chunkMapper, mock(TaskScopeStateMapper.class));
assertDoesNotThrow(() -> service.deletePayloadIfPresent(null));
assertDoesNotThrow(() -> service.deletePayloadIfPresent(" "));
verify(chunkMapper, never()).selectCount(any());
verify(rustfs, never()).deleteObject(anyString());
}
// ===== 异常路径 =====
/** 反查引用的查询抛异常 → 保守不删(原实现即如此,必须保持)。 */
@Test
void referenceQueryFailureIsConservativeAndSkipsDelete() {
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
TaskChunkMapper chunkMapper = mock(TaskChunkMapper.class);
when(chunkMapper.selectCount(any())).thenThrow(new IllegalStateException("db down"));
assertDoesNotThrow(() ->
service(rustfs, chunkMapper, mock(TaskScopeStateMapper.class))
.deletePayloadIfPresent(RUSTFS_CHUNK_POINTER));
verify(rustfs, never()).deleteObject(anyString());
}
/** 跨实例的 local 指针:不物理删(留给持有实例的清理任务)。 */
@Test
void crossInstanceLocalPointerIsNotDeleted() {
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
TaskChunkMapper chunkMapper = mock(TaskChunkMapper.class);
when(chunkMapper.selectCount(any())).thenReturn(0L);
service(rustfs, chunkMapper, mock(TaskScopeStateMapper.class))
.deletePayloadIfPresent("local:i/server-110/task-chunk/x/1/hash/chunk-1.json");
verify(rustfs, never()).deleteObject(anyString());
}
private TransientPayloadStorageService service(RustfsObjectStorageService rustfs,
TaskChunkMapper chunkMapper,
TaskScopeStateMapper scopeMapper) {
TransientStorageProperties transientProperties = new TransientStorageProperties();
transientProperties.setEnabled(true);
StorageProperties storageProperties = new StorageProperties();
storageProperties.setLocalTempDir(System.getProperty("java.io.tmpdir"));
return new TransientPayloadStorageService(
transientProperties,
storageProperties,
rustfs,
mock(OssStorageService.class),
new ObjectMapper(),
new InstanceMetadata("server-121"),
chunkMapper,
scopeMapper);
}
}