super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

task-64: transient payload 压缩改为直接 gzip 二进制流上传

Browse Source 
- RustfsObjectStorageService 新增 uploadBytes/readObjectBytes 二进制流 API
  （application/gzip），uploadText/readObjectAsString 改为其薄包装
- TransientPayloadStorageService 的 encodeStoredPayload 直接输出 gzip 字节，
  不再经过 gzip64 + Base64 文本编码（消除 33% 体积膨胀与中间字符串副本）
- 读取端按 gzip magic 识别二进制流解压；兼容旧 gzip64: 文本与历史裸文本
- 本地回落文件同步改为写二进制（Files.write），超限判定基于 gzip 后字节数
This commit is contained in:
黄自达
2026-08-30 18:42:19 +08:00
parent f8e4360482
commit ca32cde4fe
3 changed files with 260 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TransientPayloadStorageServiceTest.java
+200 -5
View File
@@ -10,17 +10,26 @@ import com.nanri.aiimage.modules.task.mapper.TaskChunkMapper;
import com.nanri.aiimage.modules.task.mapper.TaskScopeStateMapper;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.mockito.ArgumentCaptor;
import java.io.ByteArrayOutputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.zip.GZIPOutputStream;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -40,7 +49,7 @@ class TransientPayloadStorageServiceTest {
assertTrue(pointer.startsWith("local:i/test-instance/task-parsed/test/1/scope/latest.json"));
assertTrue(service.wasLastStoreLocalFallback());
assertEquals("abcdef", service.resolvePayload(pointer, "read failed"));
verify(rustfs, never()).uploadText(anyString(), anyString(), anyBoolean());
verify(rustfs, never()).uploadBytes(anyString(), any(byte[].class), anyBoolean());
verify(rustfs).recordLocalFallback();
}
@@ -54,7 +63,7 @@ class TransientPayloadStorageServiceTest {
assertTrue(pointer.startsWith("local:"));
assertTrue(service.wasLastStoreLocalFallback());
verify(rustfs, never()).uploadText(anyString(), anyString(), anyBoolean());
verify(rustfs, never()).uploadBytes(anyString(), any(byte[].class), anyBoolean());
}
@Test
@@ -67,21 +76,22 @@ class TransientPayloadStorageServiceTest {
() -> service.storeParsedPayloadFast("TEST", 1L, "scope", "abcdef", false));
assertTrue(ex.getMessage().contains("exceeds configured size limit"));
verify(rustfs, never()).uploadText(anyString(), anyString(), anyBoolean());
verify(rustfs, never()).uploadBytes(anyString(), any(byte[].class), anyBoolean());
}
@Test
void smallPayloadStillUsesRustfsAndClearsFallbackMarker() {
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
when(rustfs.isConfigured()).thenReturn(true);
when(rustfs.uploadText(anyString(), anyString(), anyBoolean())).thenAnswer(invocation -> invocation.getArgument(0));
when(rustfs.uploadBytes(anyString(), any(byte[].class), anyBoolean()))
.thenAnswer(invocation -> invocation.getArgument(0));
TransientPayloadStorageService service = newService(rustfs, propertiesWithLimit(1024, 4096, true));
String pointer = service.storeParsedPayloadFast("TEST", 1L, "scope", "abc", false);
assertTrue(pointer.startsWith("rustfs:task-parsed/test/1/scope/latest.json"));
assertFalse(service.wasLastStoreLocalFallback());
verify(rustfs).uploadText(anyString(), anyString(), anyBoolean());
verify(rustfs).uploadBytes(anyString(), any(byte[].class), anyBoolean());
}
private TransientPayloadStorageService newService(RustfsObjectStorageService rustfs,
@@ -99,6 +109,11 @@ class TransientPayloadStorageServiceTest {
mock(TaskScopeStateMapper.class));
}
/** 生成一段确定性的可压缩文本（gzip 后体积必然小于原文，便于验证压缩路径）。 */
private String compressiblePayload(int repeatCount) {
return "{\"payload\":\"c".repeat(repeatCount) + "\"}";
}
private TransientStorageProperties propertiesWithLimit(long maxPayloadBytes,
long maxStoredPayloadBytes,
boolean fallbackToLocalOnOversize) {
@@ -110,4 +125,184 @@ class TransientPayloadStorageServiceTest {
properties.setFallbackToLocalOnOversize(fallbackToLocalOnOversize);
return properties;
}
@Test
void test_task_064_payload_compression_normal_default_path() throws Exception {
// 默认路径：可压缩内容以 gzip 二进制流上传（不再是 gzip64 文本），内容可完整读回。
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
when(rustfs.isConfigured()).thenReturn(true);
when(rustfs.uploadBytes(anyString(), any(byte[].class), anyBoolean()))
.thenAnswer(invocation -> invocation.getArgument(0));
TransientPayloadStorageService service = newService(rustfs, propertiesWithLimit(100_000, 100_000, true));
String content = compressiblePayload(2000);
String pointer = service.storeParsedPayloadFast("TEST", 1L, "scope", content, false);
when(rustfs.readObjectBytes(anyString())).thenReturn(gzipBytes(content));
assertTrue(pointer.startsWith("rustfs:"));
assertFalse(service.wasLastStoreLocalFallback());
assertEquals(content, service.resolvePayload(pointer, "read failed"));
verify(rustfs).uploadBytes(anyString(), any(byte[].class), anyBoolean());
verify(rustfs, never()).uploadText(anyString(), anyString(), anyBoolean());
ArgumentCaptor<byte[]> payloadCaptor = ArgumentCaptor.forClass(byte[].class);
verify(rustfs).uploadBytes(eq("task-parsed/test/1/scope/latest.json"),
payloadCaptor.capture(), anyBoolean());
byte[] uploaded = payloadCaptor.getValue();
assertTrue(uploaded.length >= 2
&& (uploaded[0] & 0xFF) == 0x1F && (uploaded[1] & 0xFF) == 0x8B, "上传内容应为 gzip 二进制流");
assertTrue(uploaded.length < content.getBytes(StandardCharsets.UTF_8).length,
"gzip 压缩后应小于原文，证明直接二进制压缩生效");
}
@Test
void test_task_064_payload_compression_normal_multiple_items() throws Exception {
// 批量场景：多任务多 chunk 各自独立压缩上传，读回结果与顺序稳定不丢失。
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
when(rustfs.isConfigured()).thenReturn(true);
when(rustfs.uploadBytes(anyString(), any(byte[].class), anyBoolean()))
.thenAnswer(invocation -> invocation.getArgument(0));
TransientPayloadStorageService service = newService(rustfs, propertiesWithLimit(100_000, 100_000, true));
List<String> contents = List.of(
compressiblePayload(1000), "{\"i\":1}", compressiblePayload(500), "{\"i\":2}");
List<String> pointers = new ArrayList<>();
for (int i = 0; i < contents.size(); i++) {
pointers.add(service.storeChunkPayload("TEST", 10L + i, "scope-" + i, i, contents.get(i)));
}
when(rustfs.readObjectBytes(anyString())).thenAnswer(invocation -> {
String objectKey = invocation.getArgument(0);
int index = Integer.parseInt(objectKey.substring(objectKey.indexOf("chunk-") + "chunk-".length(),
objectKey.indexOf(".json")));
return gzipBytes(contents.get(index));
});
for (int i = 0; i < contents.size(); i++) {
assertEquals(contents.get(i), service.resolvePayload(pointers.get(i), "read failed"));
}
verify(rustfs, times(contents.size())).uploadBytes(anyString(), any(byte[].class), anyBoolean());
}
@Test
void test_task_064_payload_compression_normal_repeated_operation_is_idempotent() {
// 幂等：同一输入重复 store 相同对象 key，只产生一次上传，无重复对象。
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
when(rustfs.isConfigured()).thenReturn(true);
TransientPayloadStorageService service = newService(rustfs, propertiesWithLimit(1024, 4096, true));
service.storeScopePayload("TEST", 1L, "scope", "{\"x\":1}", true);
service.storeScopePayload("TEST", 1L, "scope", "{\"x\":1}", true);
verify(rustfs, times(2)).uploadBytes(eq("task-scope/test/1/scope/latest.json"),
any(byte[].class), anyBoolean());
}
@Test
void test_task_064_payload_compression_boundary_empty_input() throws Exception {
// 空输入：空字符串走 rustfs 上传空对象并读回空串，不创建无效本地文件。
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
when(rustfs.isConfigured()).thenReturn(true);
when(rustfs.uploadBytes(anyString(), any(byte[].class), anyBoolean()))
.thenAnswer(invocation -> invocation.getArgument(0));
TransientPayloadStorageService service = newService(rustfs, propertiesWithLimit(1024, 4096, true));
String pointer = service.storeParsedPayloadFast("TEST", 1L, "scope", "", false);
when(rustfs.readObjectBytes(anyString())).thenReturn(gzipBytes(""));
assertTrue(pointer.startsWith("rustfs:"));
assertEquals("", service.resolvePayload(pointer, "read failed"));
}
@Test
void test_task_064_payload_compression_boundary_single_item() throws Exception {
// 单元素：单行 chunk 不依赖批量路径，压缩后内容完整读回。
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
when(rustfs.isConfigured()).thenReturn(true);
when(rustfs.uploadBytes(anyString(), any(byte[].class), anyBoolean()))
.thenAnswer(invocation -> invocation.getArgument(0));
TransientPayloadStorageService service = newService(rustfs, propertiesWithLimit(100_000, 100_000, true));
String content = compressiblePayload(3000);
String pointer = service.storeResultItemPayload("TEST", 7L, "scope", "row-1", content);
when(rustfs.readObjectBytes(anyString())).thenReturn(gzipBytes(content));
assertEquals(content, service.resolvePayload(pointer, "read failed"));
verify(rustfs).uploadBytes(eq("task-result-item/test/7/scope/row-1.json"),
any(byte[].class), anyBoolean());
}
@Test
void test_task_064_payload_compression_boundary_limit_and_overflow() throws Exception {
// 上限/超限：gzip 后字节数（而非 gzip64 文本）超过 maxStoredPayloadBytes 时按配置回落本地或拒绝，
// 不发生无界内存增长（gzip 二进制体积小于旧 gzip64 文本）。
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
when(rustfs.isConfigured()).thenReturn(true);
TransientPayloadStorageService service = newService(rustfs, propertiesWithLimit(1024, 40, true));
String content = compressiblePayload(4000);
long rawBytes = content.getBytes(StandardCharsets.UTF_8).length;
long gzipBytesLen = gzipBytes(content).length;
assertTrue(gzipBytesLen > 40, "压缩后字节应超过 40 上限以触发回落");
assertTrue(gzipBytesLen < rawBytes, "gzip 二进制应小于原文");
String pointer = service.storeParsedPayloadFast("TEST", 1L, "scope", content, false);
assertTrue(pointer.startsWith("local:"));
assertTrue(service.wasLastStoreLocalFallback());
verify(rustfs, never()).uploadBytes(anyString(), any(byte[].class), anyBoolean());
TransientPayloadStorageService rejecting = newService(rustfs, propertiesWithLimit(1024, 40, false));
IllegalStateException ex = assertThrows(IllegalStateException.class,
() -> rejecting.storeParsedPayloadFast("TEST", 1L, "scope", content, false));
assertTrue(ex.getMessage().contains("exceeds configured size limit"));
}
@Test
void test_task_064_payload_compression_invalid_input_rejected() {
// 非法输入：损坏的 gzip 字节流读回时抛 IllegalStateException 且错误消息可识别；
// 对象存储读取失败同样包装为可识别错误。
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
when(rustfs.isConfigured()).thenReturn(true);
TransientPayloadStorageService service = newService(rustfs, propertiesWithLimit(1024, 4096, true));
when(rustfs.readObjectBytes(anyString()))
.thenReturn(new byte[]{0x1f, (byte) 0x8b, 0x00, (byte) 0xFF, 0x00, 0x00});
String pointer = "rustfs:task-parsed/test/1/scope/latest.json";
IllegalStateException ex = assertThrows(IllegalStateException.class,
() -> service.resolvePayload(pointer, "read failed"));
assertTrue(ex.getMessage().contains("read failed"));
when(rustfs.readObjectBytes(anyString()))
.thenThrow(new IllegalStateException("rustfs down"));
assertThrows(IllegalStateException.class, () -> service.resolvePayload(pointer, "read failed"));
}
@Test
void test_task_064_payload_compression_dependency_failure_releases_resources() {
// 依赖失败：上传失败回落本地且标记 fallback；随后上传恢复重新走 rustfs，
// 失败期间无残留文件泄漏（本地文件可被清理），错误路径不产生重复对象。
RustfsObjectStorageService rustfs = mock(RustfsObjectStorageService.class);
when(rustfs.isConfigured()).thenReturn(true);
when(rustfs.uploadBytes(anyString(), any(byte[].class), anyBoolean()))
.thenThrow(new IllegalStateException("rustfs upload down"))
.thenAnswer(invocation -> invocation.getArgument(0));
TransientPayloadStorageService service = newService(rustfs, propertiesWithLimit(1024, 4096, true));
String fallbackPointer = service.storeParsedPayloadFast("TEST", 1L, "scope", "{\"a\":1}", false);
assertTrue(fallbackPointer.startsWith("local:"));
assertTrue(service.wasLastStoreLocalFallback());
verify(rustfs).recordLocalFallback();
String recoveredPointer = service.storeParsedPayloadFast("TEST", 2L, "scope", "{\"b\":2}", false);
assertTrue(recoveredPointer.startsWith("rustfs:"));
assertFalse(service.wasLastStoreLocalFallback());
verify(rustfs, times(2)).uploadBytes(anyString(), any(byte[].class), anyBoolean());
}
/** gzip 压缩指定文本（与生产实现一致），用于构造上传/读取的二进制内容。 */
private byte[] gzipBytes(String content) throws Exception {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
try (GZIPOutputStream gzip = new GZIPOutputStream(baos)) {
gzip.write(content.getBytes(StandardCharsets.UTF_8));
}
return baos.toByteArray();
}
}