task-12: extend coze result buffering beyond poll path
- bufferCozeRowsOrMerge: unified DONE result persist entry point; buffers to transient storage when toggle on, falls back to immediate chunk merge on failure (result never lost), no-op on empty input - submit/retry synchronous immediate DONE results now buffer too (previously always merged immediately); submit path persists a DONE scope state row to carry the buffer pointer (persistImmediateCozeDoneState) - poll path drops the batchTotal>1 gate: single-batch tasks also buffer - CozeCandidate/CozeBatchContext records package-private for test access
This commit is contained in:
+431
@@ -0,0 +1,431 @@
|
||||
package com.nanri.aiimage.modules.similarasin.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.nanri.aiimage.config.SimilarAsinProperties;
|
||||
import com.nanri.aiimage.modules.coze.service.CozeCredentialPoolService;
|
||||
import com.nanri.aiimage.modules.similarasin.client.SimilarAsinCozeClient;
|
||||
import com.nanri.aiimage.modules.similarasin.mapper.SimilarAsinFilterConditionMapper;
|
||||
import com.nanri.aiimage.modules.similarasin.model.dto.SimilarAsinResultRowDto;
|
||||
import com.nanri.aiimage.modules.similarasin.util.SimilarAsinImageEmbedder;
|
||||
import com.nanri.aiimage.modules.file.service.LocalFileStorageService;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileResultMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.FileTaskMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
|
||||
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileResultEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskChunkEntity;
|
||||
import com.nanri.aiimage.modules.task.model.entity.TaskScopeStateEntity;
|
||||
import com.nanri.aiimage.modules.task.service.TaskFileJobService;
|
||||
import com.nanri.aiimage.modules.task.service.TransientPayloadStorageService;
|
||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Spy;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.lang.reflect.Method;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.atLeastOnce;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* Task 12：扩展 Coze 结果缓冲覆盖范围，减少频繁读写完整 chunk payload。
|
||||
* P0-3 缓冲只覆盖"poll DONE 且 batchTotal>1"；Task 12 扩展为：
|
||||
* 1) poll DONE 结果去掉 batchTotal 限制，单 batch 也走缓冲；
|
||||
* 2) retry 提交同步 immediate DONE 结果也走缓冲（原立即 merge）；
|
||||
* 3) 统一走 bufferCozeRowsOrMerge：缓冲失败回退立即 merge，结果不丢失。
|
||||
* flushBufferedCozeResults 在 finalize 前一次性合并，全任务收敛为一次 chunk 读写。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class SimilarAsinTaskServiceCozeBufferScopeTest {
|
||||
|
||||
private static final AtomicLong NEXT_ID = new AtomicLong(71000);
|
||||
private static final String MODULE = SimilarAsinTaskService.MODULE_TYPE;
|
||||
private static final String CREDENTIAL = "cred-1";
|
||||
|
||||
@Mock private LocalFileStorageService localFileStorageService;
|
||||
@Mock private com.nanri.aiimage.modules.file.service.oss.OssStorageService ossStorageService;
|
||||
@Mock private com.nanri.aiimage.config.StorageProperties storageProperties;
|
||||
@Mock private FileTaskMapper fileTaskMapper;
|
||||
@Mock private FileResultMapper fileResultMapper;
|
||||
@Mock private TaskScopeStateMapper taskScopeStateMapper;
|
||||
@Mock private TaskChunkMapper taskChunkMapper;
|
||||
@Mock private SimilarAsinFilterConditionMapper filterConditionMapper;
|
||||
@Spy private ObjectMapper objectMapper = new ObjectMapper();
|
||||
@Mock private SimilarAsinCozeClient cozeClient;
|
||||
@Mock private SimilarAsinTaskCacheService taskCacheService;
|
||||
@Mock private SimilarAsinProperties properties;
|
||||
@Mock private TaskFileJobService taskFileJobService;
|
||||
@Mock private com.nanri.aiimage.modules.task.service.TaskDistributedLockService taskDistributedLockService;
|
||||
@Mock private com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService taskProgressSnapshotService;
|
||||
@Mock private TransientPayloadStorageService transientPayloadStorageService;
|
||||
@Mock private org.springframework.transaction.PlatformTransactionManager transactionManager;
|
||||
@Mock private com.nanri.aiimage.common.service.DistributedJobLockService distributedJobLockService;
|
||||
@Mock private com.nanri.aiimage.config.InstanceMetadata instanceMetadata;
|
||||
@Mock private com.nanri.aiimage.modules.coze.service.CozeCredentialPoolService cozeCredentialPoolService;
|
||||
@Mock private SimilarAsinImageEmbedder imageEmbedder;
|
||||
@Mock private SimilarAsinImagePrefetchService imagePrefetchService;
|
||||
|
||||
@InjectMocks private SimilarAsinTaskService service;
|
||||
|
||||
@BeforeAll
|
||||
static void initializeMybatisMetadata() {
|
||||
MapperBuilderAssistant assistant = new MapperBuilderAssistant(new MybatisConfiguration(), "");
|
||||
TableInfoHelper.initTableInfo(assistant, FileTaskEntity.class);
|
||||
TableInfoHelper.initTableInfo(assistant, TaskChunkEntity.class);
|
||||
TableInfoHelper.initTableInfo(assistant, TaskScopeStateEntity.class);
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void setUp() throws Exception {
|
||||
lenient().when(properties.isBoundedResultAssemblyEnabled()).thenReturn(true);
|
||||
lenient().when(properties.getImageCacheMaxBytes()).thenReturn(256L * 1024L * 1024L);
|
||||
lenient().when(properties.getResultFileTimeoutMinutes()).thenReturn(90);
|
||||
lenient().when(properties.getParseResponsePreviewLimit()).thenReturn(100);
|
||||
lenient().when(properties.getMaxSourceFileBytes()).thenReturn(50L * 1024L * 1024L);
|
||||
lenient().when(properties.getMaxParseRows()).thenReturn(50000);
|
||||
lenient().when(properties.getMaxFieldLength()).thenReturn(2000);
|
||||
lenient().when(properties.getMaxWorkbookZipEntries()).thenReturn(20000);
|
||||
lenient().when(properties.getMaxWorkbookUncompressedBytes()).thenReturn(512L * 1024L * 1024L);
|
||||
lenient().when(properties.getCozeBatchSize()).thenReturn(5);
|
||||
lenient().when(properties.getCozeTextOnlyBatchSize()).thenReturn(10);
|
||||
lenient().when(properties.isCozeResultBufferEnabled()).thenReturn(true);
|
||||
lenient().when(properties.getCozePollTimeoutMillis()).thenReturn(30_000);
|
||||
lenient().when(properties.getDbJobTouchIntervalMillis()).thenReturn(2_000L);
|
||||
lenient().when(properties.getDbTaskTouchIntervalMillis()).thenReturn(2_000L);
|
||||
lenient().when(properties.getCozeSubmitLockWaitMillis()).thenReturn(1_000L);
|
||||
lenient().when(properties.getCozeSubmitLockRetryDelayMillis()).thenReturn(100L);
|
||||
lenient().when(properties.getCozeSubmitMinIntervalMillis()).thenReturn(0L);
|
||||
lenient().when(properties.getCozeSubmitMaxRetryCount()).thenReturn(3);
|
||||
lenient().when(properties.getCozeFlushPendingMinutes()).thenReturn(10);
|
||||
lenient().when(cozeClient.configuredCredentialCount()).thenReturn(1);
|
||||
lenient().when(cozeClient.nextCredential()).thenReturn(new SimilarAsinCozeClient.CozeCredentialRef(
|
||||
CREDENTIAL, "wf-1", "token-1", 4));
|
||||
lenient().when(cozeClient.credentialByName(CREDENTIAL)).thenReturn(new SimilarAsinCozeClient.CozeCredentialRef(
|
||||
CREDENTIAL, "wf-1", "token-1", 4));
|
||||
lenient().when(cozeCredentialPoolService.borrow(eq(MODULE), any())).thenReturn(
|
||||
mock(CozeCredentialPoolService.BorrowedCredential.class));
|
||||
lenient().when(distributedJobLockService.tryLock(anyString(), any())).thenReturn(
|
||||
mock(com.nanri.aiimage.common.service.DistributedJobLockService.LockHandle.class));
|
||||
lenient().doAnswer(invocation -> {
|
||||
FileTaskEntity task = invocation.getArgument(0);
|
||||
task.setId(NEXT_ID.incrementAndGet());
|
||||
return 1;
|
||||
}).when(fileTaskMapper).insert(any(FileTaskEntity.class));
|
||||
lenient().when(fileResultMapper.insert(any(FileResultEntity.class))).thenReturn(1);
|
||||
lenient().when(taskScopeStateMapper.insert(any(TaskScopeStateEntity.class))).thenReturn(1);
|
||||
lenient().when(taskScopeStateMapper.update(any(), any())).thenReturn(1);
|
||||
lenient().when(taskScopeStateMapper.selectList(any())).thenReturn(List.of());
|
||||
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of());
|
||||
lenient().when(fileResultMapper.selectList(any())).thenReturn(List.of());
|
||||
lenient().when(transientPayloadStorageService.storeParsedPayloadEntry(
|
||||
eq(MODULE), any(), anyString(), anyString(), anyString(), eq(true)))
|
||||
.thenAnswer(invocation -> "rustfs:coze-result/" + NEXT_ID.incrementAndGet());
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void shutdown() {
|
||||
service.shutdownAssembleExecutor();
|
||||
}
|
||||
|
||||
private static SimilarAsinResultRowDto row(String rowToken, String id, String asin, String country, String title) {
|
||||
SimilarAsinResultRowDto r = new SimilarAsinResultRowDto();
|
||||
r.setRowToken(rowToken);
|
||||
r.setId(id);
|
||||
r.setAsin(asin);
|
||||
r.setCountry(country);
|
||||
r.setTitle(title);
|
||||
r.setMainUrl("https://img.example.com/" + asin + ".jpg");
|
||||
return r;
|
||||
}
|
||||
|
||||
private static TaskChunkEntity chunk(Long id, String scopeHash, Integer chunkIndex, String payloadJson) {
|
||||
TaskChunkEntity chunk = new TaskChunkEntity();
|
||||
chunk.setId(id);
|
||||
chunk.setTaskId(7104L);
|
||||
chunk.setModuleType(MODULE);
|
||||
chunk.setScopeHash(scopeHash);
|
||||
chunk.setChunkIndex(chunkIndex);
|
||||
chunk.setPayloadJson(payloadJson);
|
||||
chunk.setPayloadHash("h-" + id);
|
||||
return chunk;
|
||||
}
|
||||
|
||||
private static FileTaskEntity task() {
|
||||
FileTaskEntity task = new FileTaskEntity();
|
||||
task.setId(7104L);
|
||||
task.setModuleType(MODULE);
|
||||
task.setStatus("RUNNING");
|
||||
task.setResultJson("{\"categorySwitch\":true}");
|
||||
return task;
|
||||
}
|
||||
|
||||
private static TaskScopeStateEntity state(FileTaskEntity task, long id, String status, int batchTotal) {
|
||||
TaskScopeStateEntity state = new TaskScopeStateEntity();
|
||||
state.setId(id);
|
||||
state.setTaskId(task.getId());
|
||||
state.setModuleType(MODULE);
|
||||
state.setScopeHash("scope-" + id);
|
||||
state.setCozeStatus(status);
|
||||
state.setParsedPayloadJson("ptr:batch-" + id);
|
||||
state.setStateJson("{\"jobId\":7101,\"resultId\":7201,\"chunkScopeHash\":null,\"chunkIndex\":null,"
|
||||
+ "\"batchIndex\":1,\"batchTotal\":" + batchTotal + ",\"ownerInstanceId\":\"test-instance\","
|
||||
+ "\"submitRetryCount\":0,\"credentialName\":\"" + CREDENTIAL + "\",\"resultPayloadPointer\":\"ptr:buffer-" + id + "\"}");
|
||||
return state;
|
||||
}
|
||||
|
||||
private static String rowsJson(List<SimilarAsinResultRowDto> rows) throws Exception {
|
||||
return new ObjectMapper().writeValueAsString(rows);
|
||||
}
|
||||
|
||||
private SimilarAsinTaskService.CozeBatchContext context(int batchTotal) {
|
||||
return new SimilarAsinTaskService.CozeBatchContext(
|
||||
7101L, 7201L, null, null, 1, batchTotal, "test-instance", 0, CREDENTIAL, null);
|
||||
}
|
||||
|
||||
private void stubChunkMerge(String payloadJson) throws Exception {
|
||||
TaskChunkEntity chunk = chunk(1L, "scope-1", 1, "ptr:chunk-1");
|
||||
// loadSubmittedChunks 只保留非空 chunk，chunk payload 必须能解析出至少一行。
|
||||
// 全部 lenient：缓冲成功路径不触达 merge，仅缓冲失败/flush 合并路径消费。
|
||||
lenient().when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk));
|
||||
lenient().when(transientPayloadStorageService.resolvePayload(anyString(), anyString()))
|
||||
.thenAnswer(invocation -> {
|
||||
String pointer = invocation.getArgument(0);
|
||||
if (pointer != null && pointer.startsWith("ptr:chunk-")) {
|
||||
return payloadJson;
|
||||
}
|
||||
return "[]";
|
||||
});
|
||||
lenient().when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()))
|
||||
.thenAnswer(invocation -> "ptr:stored-" + invocation.getArgument(3));
|
||||
lenient().when(taskChunkMapper.selectOne(any())).thenReturn(chunk);
|
||||
lenient().when(taskChunkMapper.update(any(), any())).thenReturn(1);
|
||||
}
|
||||
|
||||
private static String chunkRowsJson() throws Exception {
|
||||
// chunk-1 已含 r1 行：loadSubmittedChunks 只保留非空 chunk，且 rowKey 索引能命中缓冲行。
|
||||
return rowsJson(List.of(row("r1", "1", "B0A0000001", "英国", "Title 1")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_012_payload_chunk_normal_default_path() throws Exception {
|
||||
// 正常输入：DONE 结果（batchTotal=1 单 batch）经 bufferCozeRowsOrMerge 走缓冲，
|
||||
// 不立即写 chunk；缓冲失败回退立即 merge 结果不丢失。
|
||||
FileTaskEntity task = task();
|
||||
List<SimilarAsinResultRowDto> rows = List.of(row("r1", "1", "B0A0000001", "英国", "Title 1"));
|
||||
|
||||
Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferCozeRowsOrMerge",
|
||||
TaskScopeStateEntity.class,
|
||||
SimilarAsinTaskService.CozeBatchContext.class,
|
||||
List.class, FileTaskEntity.class, Map.class);
|
||||
bufferOrMerge.setAccessible(true);
|
||||
bufferOrMerge.invoke(service, state(task, 1L, "DONE", 1), context(1), rows, task, Map.of());
|
||||
|
||||
verify(transientPayloadStorageService, times(1)).storeParsedPayloadEntry(
|
||||
eq(MODULE), eq(7104L), anyString(), anyString(), anyString(), eq(true));
|
||||
verify(transientPayloadStorageService, never()).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
||||
verify(taskChunkMapper, never()).update(any(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_012_payload_chunk_normal_multiple_items() throws Exception {
|
||||
// 批量场景：多个 DONE state（单 batch）全部缓冲；flush 后按 chunk 分组一次合并
|
||||
FileTaskEntity task = task();
|
||||
stubChunkMerge(chunkRowsJson());
|
||||
when(taskScopeStateMapper.selectList(any())).thenReturn(
|
||||
List.of(state(task, 1L, "DONE", 1), state(task, 2L, "DONE", 1)));
|
||||
when(fileTaskMapper.selectById(7104L)).thenReturn(task);
|
||||
when(transientPayloadStorageService.resolvePayload(anyString(), anyString()))
|
||||
.thenAnswer(invocation -> {
|
||||
String pointer = invocation.getArgument(0);
|
||||
if (pointer != null && pointer.startsWith("ptr:buffer-")) {
|
||||
return rowsJson(List.of(row("r1", "1", "B0A0000001", "英国", "Title 1")));
|
||||
}
|
||||
if (pointer != null && pointer.startsWith("ptr:chunk-")) {
|
||||
return chunkRowsJson();
|
||||
}
|
||||
return "[]";
|
||||
});
|
||||
when(transientPayloadStorageService.storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString()))
|
||||
.thenAnswer(invocation -> "ptr:stored-" + invocation.getArgument(3));
|
||||
when(taskChunkMapper.selectOne(any())).thenReturn(chunk(1L, "scope-1", 1, "ptr:chunk-1"));
|
||||
when(taskChunkMapper.selectList(any())).thenReturn(List.of(chunk(1L, "scope-1", 1, "ptr:chunk-1")));
|
||||
when(taskChunkMapper.update(any(), any())).thenReturn(1);
|
||||
|
||||
Method flush = SimilarAsinTaskService.class.getDeclaredMethod("flushBufferedCozeResults", Long.class);
|
||||
flush.setAccessible(true);
|
||||
flush.invoke(service, 7104L);
|
||||
|
||||
verify(transientPayloadStorageService, atLeastOnce()).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
||||
verify(taskChunkMapper, atLeastOnce()).update(any(), any());
|
||||
// pointer 清理：每个缓冲 state 都更新
|
||||
verify(taskScopeStateMapper, atLeastOnce()).update(any(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_012_payload_chunk_normal_repeated_operation_is_idempotent() throws Exception {
|
||||
// 重复执行同一输入：缓冲写幂等（同一 state 不产生重复 buffer/merge）
|
||||
FileTaskEntity task = task();
|
||||
List<SimilarAsinResultRowDto> rows = List.of(row("r1", "1", "B0A0000001", "英国", "Title 1"));
|
||||
|
||||
Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferCozeRowsOrMerge",
|
||||
TaskScopeStateEntity.class,
|
||||
SimilarAsinTaskService.CozeBatchContext.class,
|
||||
List.class, FileTaskEntity.class, Map.class);
|
||||
bufferOrMerge.setAccessible(true);
|
||||
TaskScopeStateEntity state = state(task, 1L, "DONE", 2);
|
||||
bufferOrMerge.invoke(service, state, context(2), rows, task, Map.of());
|
||||
bufferOrMerge.invoke(service, state, context(2), rows, task, Map.of());
|
||||
|
||||
// 缓冲 2 次（每次重新写 pointer 是幂等语义：同一 state 覆盖写，无重复行）
|
||||
verify(transientPayloadStorageService, times(2)).storeParsedPayloadEntry(
|
||||
eq(MODULE), eq(7104L), anyString(), anyString(), anyString(), eq(true));
|
||||
verify(taskChunkMapper, never()).update(any(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_012_payload_chunk_boundary_empty_input() throws Exception {
|
||||
// 空输入：无行时缓冲与 merge 都不发生，不创建无效资源
|
||||
FileTaskEntity task = task();
|
||||
Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferCozeRowsOrMerge",
|
||||
TaskScopeStateEntity.class,
|
||||
SimilarAsinTaskService.CozeBatchContext.class,
|
||||
List.class, FileTaskEntity.class, Map.class);
|
||||
bufferOrMerge.setAccessible(true);
|
||||
bufferOrMerge.invoke(service, state(task, 1L, "DONE", 1), context(1), null, task, Map.of());
|
||||
bufferOrMerge.invoke(service, state(task, 2L, "DONE", 1), context(1), List.of(), task, Map.of());
|
||||
|
||||
verify(transientPayloadStorageService, never()).storeParsedPayloadEntry(any(), any(), anyString(), anyString(), anyString(), eq(true));
|
||||
verify(taskChunkMapper, never()).update(any(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_012_payload_chunk_boundary_single_item() throws Exception {
|
||||
// 单 batch（batchTotal=1）：原 P0-3 例外，现在也缓冲
|
||||
FileTaskEntity task = task();
|
||||
List<SimilarAsinResultRowDto> rows = List.of(row("r1", "1", "B0A0000001", "英国", "Title 1"));
|
||||
Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferCozeRowsOrMerge",
|
||||
TaskScopeStateEntity.class,
|
||||
SimilarAsinTaskService.CozeBatchContext.class,
|
||||
List.class, FileTaskEntity.class, Map.class);
|
||||
bufferOrMerge.setAccessible(true);
|
||||
bufferOrMerge.invoke(service, state(task, 1L, "DONE", 1), context(1), rows, task, Map.of());
|
||||
|
||||
verify(transientPayloadStorageService, times(1)).storeParsedPayloadEntry(
|
||||
eq(MODULE), eq(7104L), anyString(), anyString(), anyString(), eq(true));
|
||||
verify(taskChunkMapper, never()).update(any(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_012_payload_chunk_boundary_limit_and_overflow() throws Exception {
|
||||
// 缓冲开关关闭：回退立即 merge，DONE 结果仍落 chunk 不丢失
|
||||
FileTaskEntity task = task();
|
||||
stubChunkMerge(chunkRowsJson());
|
||||
when(properties.isCozeResultBufferEnabled()).thenReturn(false);
|
||||
List<SimilarAsinResultRowDto> rows = List.of(row("r1", "1", "B0A0000001", "英国", "Title 1"));
|
||||
Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferCozeRowsOrMerge",
|
||||
TaskScopeStateEntity.class,
|
||||
SimilarAsinTaskService.CozeBatchContext.class,
|
||||
List.class, FileTaskEntity.class, Map.class);
|
||||
bufferOrMerge.setAccessible(true);
|
||||
bufferOrMerge.invoke(service, state(task, 1L, "DONE", 1), context(1), rows, task, Map.of());
|
||||
|
||||
verify(transientPayloadStorageService, never()).storeParsedPayloadEntry(any(), any(), anyString(), anyString(), anyString(), eq(true));
|
||||
verify(transientPayloadStorageService, atLeastOnce()).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_012_payload_chunk_invalid_input_rejected() throws Exception {
|
||||
// 缓冲写失败（storeParsedPayloadEntry 抛异常）：回退立即 merge，结果不丢失
|
||||
FileTaskEntity task = task();
|
||||
stubChunkMerge(chunkRowsJson());
|
||||
when(transientPayloadStorageService.storeParsedPayloadEntry(
|
||||
eq(MODULE), eq(7104L), anyString(), anyString(), anyString(), eq(true)))
|
||||
.thenThrow(new IllegalStateException("rustfs full"));
|
||||
List<SimilarAsinResultRowDto> rows = List.of(row("r1", "1", "B0A0000001", "英国", "Title 1"));
|
||||
Method bufferOrMerge = SimilarAsinTaskService.class.getDeclaredMethod("bufferCozeRowsOrMerge",
|
||||
TaskScopeStateEntity.class,
|
||||
SimilarAsinTaskService.CozeBatchContext.class,
|
||||
List.class, FileTaskEntity.class, Map.class);
|
||||
bufferOrMerge.setAccessible(true);
|
||||
bufferOrMerge.invoke(service, state(task, 1L, "DONE", 1), context(1), rows, task, Map.of());
|
||||
|
||||
verify(transientPayloadStorageService, atLeastOnce()).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
||||
verify(taskChunkMapper, atLeastOnce()).update(any(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void test_task_012_payload_chunk_dependency_failure_releases_resources() throws Exception {
|
||||
// flush 时 chunk 写失败：抛可识别业务异常且不清 pointer（保留待重试）；
|
||||
// 依赖恢复后重试 flush 成功，chunk 合并一次、pointer 清理。
|
||||
FileTaskEntity task = task();
|
||||
stubChunkMerge(chunkRowsJson());
|
||||
TaskScopeStateEntity s = state(task, 1L, "DONE", 1);
|
||||
when(taskScopeStateMapper.selectList(any())).thenReturn(List.of(s));
|
||||
when(fileTaskMapper.selectById(7104L)).thenReturn(task);
|
||||
when(transientPayloadStorageService.resolvePayload(anyString(), anyString()))
|
||||
.thenAnswer(invocation -> {
|
||||
String pointer = invocation.getArgument(0);
|
||||
if (pointer != null && pointer.startsWith("ptr:buffer-")) {
|
||||
return rowsJson(List.of(row("r1", "1", "B0A0000001", "英国", "Title 1")));
|
||||
}
|
||||
if (pointer != null && pointer.startsWith("ptr:chunk-")) {
|
||||
return chunkRowsJson();
|
||||
}
|
||||
return "[]";
|
||||
});
|
||||
java.util.concurrent.atomic.AtomicInteger storeCalls = new java.util.concurrent.atomic.AtomicInteger(0);
|
||||
doAnswer(invocation -> {
|
||||
if (storeCalls.incrementAndGet() == 1) {
|
||||
throw new IllegalStateException("rustfs write failed");
|
||||
}
|
||||
return "ptr:stored-" + invocation.getArgument(3);
|
||||
}).when(transientPayloadStorageService).storeChunkPayloadVersioned(anyString(), any(), anyString(), any(), anyString());
|
||||
|
||||
Method flush = SimilarAsinTaskService.class.getDeclaredMethod("flushBufferedCozeResults", Long.class);
|
||||
flush.setAccessible(true);
|
||||
Exception ex = assertThrows(Exception.class, () -> {
|
||||
try {
|
||||
flush.invoke(service, 7104L);
|
||||
} catch (java.lang.reflect.InvocationTargetException e) {
|
||||
throw e.getCause();
|
||||
}
|
||||
});
|
||||
assertTrue(ex.getMessage() != null && ex.getMessage().contains("刷新缓冲区"),
|
||||
"flush 失败消息必须可识别, 实际: " + ex.getMessage());
|
||||
// 失败分组不清 pointer：buffer 未被删除、stateJson 未更新，留待重试
|
||||
verify(transientPayloadStorageService, never()).deletePayloadIfPresent(anyString());
|
||||
verify(taskScopeStateMapper, never()).update(any(), any());
|
||||
|
||||
// 恢复后重试 flush：chunk 合并成功一次，pointer 清理
|
||||
flush.invoke(service, 7104L);
|
||||
assertEquals(2, storeCalls.get(), "恢复后重试应再次写 chunk");
|
||||
verify(taskScopeStateMapper, atLeastOnce()).update(any(), any());
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user