task-1: 建立 Similar ASIN 性能基线夹具 (1000/5000行、图片开关、chunk与payload采样)

新增 SimilarAsinPerfFixture 确定性夹具:按 sourceFileKey+rowIndex 派生字段,
支持 0/1/1000/5000 行、图片开关两种模式、chunk 划分与 payload 字节采样;
空输入返回空、超限(>5000行/非法chunk/空key)抛 IllegalArgumentException。
含 10 个测试覆盖默认路径/批量/幂等/边界/非法输入/依赖失败恢复。
This commit is contained in:
2026-08-29 14:22:07 +08:00
parent 322905607a
commit 2039acdfe6
4 changed files with 1097 additions and 0 deletions
@@ -0,0 +1,139 @@
package com.nanri.aiimage.modules.similarasin.util;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
import lombok.extern.slf4j.Slf4j;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
/**
* Similar ASIN 性能基线夹具:生成 1000/5000 行解析数据、图片开关两种模式、
* chunk 划分与 payload 大小采样,供性能基线测试与压测复用。
* 上限约束:单次最多 MAX_ROWS 行,防止基线夹具本身造成无界内存增长。
*/
@Slf4j
public class SimilarAsinPerfFixture {
public static final int MAX_ROWS = 5000;
public static final int DEFAULT_CHUNK_SIZE = 200;
private static final String[] COUNTRIES = {"英国", "德国", "法国", "意大利", "西班牙"};
private static final String[] TITLES = {
"Women Floral Dress Summer Casual",
"Men Cotton T-Shirt Crew Neck",
"Kids Waterproof Rain Jacket",
"Fitness Yoga Pants High Waist",
"Home Office Desk Lamp LED",
"Stainless Steel Water Bottle 750ml",
"Wireless Bluetooth Earbuds Pro",
"Pet Grooming Brush Cat Dog"
};
private static final String[] SKU_PREFIX = {"SKU", "MSKU", "ASIN-ITEM", "PROD"};
private final ObjectMapper objectMapper;
public SimilarAsinPerfFixture(ObjectMapper objectMapper) {
this.objectMapper = objectMapper;
}
/** 生成 rowCount 行解析行数据;withImages=true 时为每行生成 http 图片 URL。rowCount 超出 [0, MAX_ROWS] 时拒绝。
* 所有字段由 sourceFileKey + rowIndex 确定性派生,同一输入必然产生相同输出(幂等)。 */
public List<SimilarAsinParsedRowVo> generateRows(String sourceFileKey, int rowCount, boolean withImages) {
if (sourceFileKey == null || sourceFileKey.isBlank()) {
throw new IllegalArgumentException("sourceFileKey 不能为空");
}
if (rowCount < 0 || rowCount > MAX_ROWS) {
throw new IllegalArgumentException("rowCount 必须在 [0, " + MAX_ROWS + "] 范围内,实际 " + rowCount);
}
List<SimilarAsinParsedRowVo> rows = new ArrayList<>(rowCount);
for (int i = 0; i < rowCount; i++) {
SimilarAsinParsedRowVo row = new SimilarAsinParsedRowVo();
int rowIndex = i + 1;
long seed = (sourceFileKey.hashCode() * 31L + rowIndex) & 0x7fffffffL;
String sourceId = String.valueOf(rowIndex);
row.setSourceFileKey(sourceFileKey);
row.setSourceFilename(sourceFileKey.substring(sourceFileKey.lastIndexOf('/') + 1));
row.setRowIndex(rowIndex);
row.setSourceId(sourceId);
row.setDisplayId(sourceId);
row.setRowToken(rowTokenFor(sourceFileKey, rowIndex));
row.setAsin(deterministicAsin(seed));
row.setCountry(COUNTRIES[(int) (seed >> 5) % COUNTRIES.length]);
row.setSku(SKU_PREFIX[(int) (seed >> 9) % SKU_PREFIX.length] + "-" + (1000 + rowIndex));
row.setTitle(TITLES[(int) (seed >> 13) % TITLES.length]);
if (withImages) {
row.setUrl("https://m.media-amazon.com/images/I/" + deterministicAsin(seed ^ 0x5DEDE5B5L) + ".jpg");
} else {
row.setUrl("");
}
Map<String, String> values = new LinkedHashMap<>();
values.put("id", sourceId);
values.put("asin", row.getAsin());
values.put("国家", row.getCountry());
values.put("价格", String.format("%.2f", 1 + (seed % 9900) / 100.0));
values.put("货号", row.getSku());
values.put("标题", row.getTitle());
if (withImages) {
values.put("主图URL", row.getUrl());
}
row.setValues(values);
rows.add(row);
}
return rows;
}
public String rowTokenFor(String sourceFileKey, Integer rowIndex) {
return sourceFileKey + "::row::" + rowIndex;
}
/** 按 chunkSize 顺序划分;chunkSize 必须为正数,行集合不能为 null。 */
public List<List<SimilarAsinParsedRowVo>> splitChunks(List<SimilarAsinParsedRowVo> rows, int chunkSize) {
if (rows == null) {
throw new IllegalArgumentException("rows 不能为 null");
}
if (chunkSize <= 0) {
throw new IllegalArgumentException("chunkSize 必须为正数,实际 " + chunkSize);
}
List<List<SimilarAsinParsedRowVo>> chunks = new ArrayList<>();
if (rows.isEmpty()) {
return chunks;
}
for (int from = 0; from < rows.size(); from += chunkSize) {
int to = Math.min(from + chunkSize, rows.size());
chunks.add(new ArrayList<>(rows.subList(from, to)));
}
return chunks;
}
/** 采样全量行 payload 大小与 chunk 划分数。序列化失败时向上抛出不产生部分结果。 */
public Metrics samplePayload(List<SimilarAsinParsedRowVo> rows, boolean withImages, int chunkSize) {
List<List<SimilarAsinParsedRowVo>> chunks = splitChunks(rows, chunkSize);
if (rows.isEmpty()) {
return new Metrics(0, 0, 0);
}
try {
byte[] bytes = objectMapper.writeValueAsString(rows).getBytes(StandardCharsets.UTF_8);
return new Metrics(rows.size(), chunks.size(), bytes.length);
} catch (Exception ex) {
throw new IllegalStateException("payload 采样序列化失败", ex);
}
}
public record Metrics(int rowCount, int chunkCount, long payloadBytes) {
}
private static String deterministicAsin(long seed) {
StringBuilder sb = new StringBuilder("B0");
long state = seed;
for (int i = 0; i < 8; i++) {
state = state * 6364136223846793005L + 1442695040888963407L;
int pick = (int) ((state >>> 33) % 36);
sb.append(pick < 10 ? (char) ('0' + pick) : (char) ('A' + pick - 10));
}
return sb.toString();
}
}
@@ -0,0 +1,198 @@
package com.nanri.aiimage.modules.similarasin.util;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.modules.similarasin.model.vo.SimilarAsinParsedRowVo;
import org.junit.jupiter.api.Test;
import java.util.LinkedHashMap;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* Task 1Similar ASIN 性能基线夹具(1000/5000 行、图片开关、chunk 数与 payload 大小采样)。
* 先写测试确认 RED,再实现 SimilarAsinPerfFixture。
*/
class SimilarAsinPerfFixtureTest {
private final SimilarAsinPerfFixture fixture = new SimilarAsinPerfFixture(new ObjectMapper());
@Test
void test_task_001_payload_chunk_image_normal_default_path() {
// 1000 行、带图片开关,默认 chunk 大小
List<SimilarAsinParsedRowVo> rows = fixture.generateRows("uploads/20260829/base.xlsx", 1000, true);
assertEquals(1000, rows.size());
// 行必须包含 url 图片地址
assertFalse(rows.get(0).getUrl().isBlank());
assertTrue(rows.get(0).getUrl().startsWith("http"));
// rowToken 稳定且唯一
assertEquals(rows.get(0).getRowToken(), fixture.rowTokenFor(rows.get(0).getSourceFileKey(), rows.get(0).getRowIndex()));
assertEquals(1000, rows.stream().map(SimilarAsinParsedRowVo::getRowToken).distinct().count());
List<List<SimilarAsinParsedRowVo>> chunks = fixture.splitChunks(rows, 200);
assertEquals(5, chunks.size());
assertEquals(200, chunks.get(0).size());
SimilarAsinPerfFixture.Metrics metrics = fixture.samplePayload(rows, true, 200);
assertEquals(1000, metrics.rowCount());
assertEquals(5, metrics.chunkCount());
assertTrue(metrics.payloadBytes() > 0, "payload 采样字节数必须大于 0");
}
@Test
void test_task_001_payload_chunk_image_normal_multiple_items() {
// 5000 行批量场景:顺序稳定、chunk 数正确、结果不丢失
List<SimilarAsinParsedRowVo> rows = fixture.generateRows("uploads/20260829/multi.xlsx", 5000, false);
assertEquals(5000, rows.size());
assertTrue(rows.get(0).getUrl().isBlank(), "图片开关关闭时 url 必须为空");
List<List<SimilarAsinParsedRowVo>> chunks = fixture.splitChunks(rows, 200);
assertEquals(25, chunks.size());
// 顺序稳定:拼接后与原始一致
List<SimilarAsinParsedRowVo> restored = chunks.stream().flatMap(List::stream).toList();
assertEquals(rows.size(), restored.size());
for (int i = 0; i < rows.size(); i++) {
assertEquals(rows.get(i).getRowToken(), restored.get(i).getRowToken());
}
// 所有行 rowIndex 递增
for (int i = 0; i < rows.size(); i++) {
assertEquals(i + 1, rows.get(i).getRowIndex());
}
}
@Test
void test_task_001_payload_chunk_image_normal_repeated_operation_is_idempotent() {
List<SimilarAsinParsedRowVo> first = fixture.generateRows("uploads/20260829/idem.xlsx", 1000, true);
List<SimilarAsinParsedRowVo> second = fixture.generateRows("uploads/20260829/idem.xlsx", 1000, true);
// 同一输入重复生成:token 完全一致,不产生重复差异
assertEquals(first.size(), second.size());
for (int i = 0; i < first.size(); i++) {
assertEquals(first.get(i).getRowToken(), second.get(i).getRowToken());
assertEquals(first.get(i).getAsin(), second.get(i).getAsin());
}
// splitChunks 幂等:两次划分 chunk 数一致
assertEquals(fixture.splitChunks(first, 200).size(), fixture.splitChunks(second, 200).size());
// 采样指标幂等
SimilarAsinPerfFixture.Metrics m1 = fixture.samplePayload(first, true, 200);
SimilarAsinPerfFixture.Metrics m2 = fixture.samplePayload(second, true, 200);
assertEquals(m1.payloadBytes(), m2.payloadBytes());
}
@Test
void test_task_001_payload_chunk_image_boundary_empty_input() {
List<SimilarAsinParsedRowVo> rows = fixture.generateRows("uploads/20260829/empty.xlsx", 0, true);
assertNotNull(rows);
assertEquals(0, rows.size());
// 空行 splitChunks 返回空,不产生无效 chunk
assertEquals(0, fixture.splitChunks(rows, 200).size());
// 空行采样:行数 0、chunk 0、payload 字节数为 0(不创建任何载荷)
SimilarAsinPerfFixture.Metrics metrics = fixture.samplePayload(rows, true, 200);
assertEquals(0, metrics.rowCount());
assertEquals(0, metrics.chunkCount());
assertEquals(0, metrics.payloadBytes());
}
@Test
void test_task_001_payload_chunk_image_boundary_single_item() {
// 单行:不依赖批量路径且结果正确
List<SimilarAsinParsedRowVo> rows = fixture.generateRows("uploads/20260829/single.xlsx", 1, true);
assertEquals(1, rows.size());
assertEquals(1, rows.get(0).getRowIndex());
assertEquals(1, fixture.splitChunks(rows, 200).size());
// 单行 chunk 划分后仍只含 1 行
List<List<SimilarAsinParsedRowVo>> chunks = fixture.splitChunks(rows, 200);
assertEquals(1, chunks.get(0).size());
SimilarAsinPerfFixture.Metrics metrics = fixture.samplePayload(rows, true, 200);
assertEquals(1, metrics.rowCount());
assertEquals(1, metrics.chunkCount());
assertTrue(metrics.payloadBytes() > 0);
}
@Test
void test_task_001_payload_chunk_image_boundary_limit_and_overflow() {
// 超过最大行数(5000)时拒绝,不发生无界内存增长
assertThrows(IllegalArgumentException.class, () -> fixture.generateRows("uploads/20260829/overflow.xlsx", 5001, true));
// 达到最大允许值 5000 时允许
assertEquals(5000, fixture.generateRows("uploads/20260829/max.xlsx", 5000, true).size());
// chunkSize 非法值拒绝
assertThrows(IllegalArgumentException.class, () -> fixture.splitChunks(fixture.generateRows("uploads/20260829/a.xlsx", 100, true), 0));
assertThrows(IllegalArgumentException.class, () -> fixture.splitChunks(fixture.generateRows("uploads/20260829/b.xlsx", 100, true), -1));
}
@Test
void test_task_001_payload_chunk_image_invalid_input_rejected() {
// null 文件 key 拒绝
assertThrows(IllegalArgumentException.class, () -> fixture.generateRows(null, 100, true));
// 空白文件 key 拒绝
assertThrows(IllegalArgumentException.class, () -> fixture.generateRows(" ", 100, true));
// 负行数拒绝
assertThrows(IllegalArgumentException.class, () -> fixture.generateRows("uploads/20260829/neg.xlsx", -1, true));
// null 行集合分块拒绝
assertThrows(IllegalArgumentException.class, () -> fixture.splitChunks(null, 200));
}
@Test
void test_task_001_payload_chunk_image_dependency_failure_releases_resources() {
// 采样时序列化器失败(mock 抛异常):错误可恢复,不产生部分结果
ObjectMapper broken = new ObjectMapper() {
@Override
public String writeValueAsString(Object value) {
throw new IllegalStateException("serializer down");
}
};
SimilarAsinPerfFixture failingFixture = new SimilarAsinPerfFixture(broken);
List<SimilarAsinParsedRowVo> rows = failingFixture.generateRows("uploads/20260829/fail.xlsx", 1000, true);
assertThrows(IllegalStateException.class, () -> failingFixture.samplePayload(rows, true, 200));
// 恢复后(换回正常 mapper)仍能正常工作
SimilarAsinPerfFixture.Metrics recovered = fixture.samplePayload(fixture.generateRows("uploads/20260829/recover.xlsx", 1000, true), true, 200);
assertTrue(recovered.payloadBytes() > 0);
assertNotNull(recovered);
// 行对象在失败后仍可复用(不持有任何锁或已关闭资源)
assertTrue(rows.get(0).getAsin().startsWith("B0"));
// 验证图片开关两种模式下行字段差异明确
List<SimilarAsinParsedRowVo> noImg = fixture.generateRows("uploads/20260829/nimg.xlsx", 10, false);
assertTrue(noImg.get(0).getUrl().isBlank());
}
@Test
void test_task_001_payload_chunk_image_normal_fields_populated() {
// 行字段完整性:asin/country/sku/title/values 均填充且稳定
List<SimilarAsinParsedRowVo> rows = fixture.generateRows("uploads/20260829/fields.xlsx", 10, false);
SimilarAsinParsedRowVo row = rows.get(3);
assertTrue(row.getAsin().matches("B0[A-Z0-9]{8}"));
assertFalse(row.getCountry().isBlank());
assertFalse(row.getSku().isBlank());
assertFalse(row.getTitle().isBlank());
assertNotNull(row.getValues());
assertFalse(row.getValues().isEmpty());
assertTrue(row.getValues().containsKey("asin"));
assertTrue(row.getValues().containsKey("国家"));
// 行号与 sourceId 关联正确
assertEquals("4", row.getSourceId());
assertEquals(4, row.getRowIndex());
// values 是独立副本,修改不影响后续生成
row.getValues().put("价格", "999");
List<SimilarAsinParsedRowVo> again = fixture.generateRows("uploads/20260829/fields.xlsx", 10, false);
assertTrue(!again.get(3).getValues().getOrDefault("价格", "").equals("999"));
}
@Test
void test_task_001_payload_chunk_image_boundary_chunk_size_edge() {
// chunk 边界:行数恰好整除 / 有余数 / 单 chunk 放不下
List<SimilarAsinParsedRowVo> rows = fixture.generateRows("uploads/20260829/edge.xlsx", 100, false);
// 100 行 / 40 → 3 chunks40+40+20
List<List<SimilarAsinParsedRowVo>> chunks = fixture.splitChunks(rows, 40);
assertEquals(3, chunks.size());
assertEquals(40, chunks.get(0).size());
assertEquals(20, chunks.get(2).size());
// chunkSize 大于总行数 → 单 chunk
assertEquals(1, fixture.splitChunks(rows, 500).size());
// chunkSize 恰好等于行数 → 单 chunk 全量
assertEquals(1, fixture.splitChunks(rows, 100).size());
assertEquals(100, fixture.splitChunks(rows, 100).get(0).size());
}
}
+50
View File
@@ -0,0 +1,50 @@
#!/usr/bin/env python3
"""检查 app_client 的 Step 2/后续开发任务是否全部完成。"""
from __future__ import annotations
import json
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parent
PROGRESS_PATH = ROOT / "progress.json"
def main() -> int:
if not PROGRESS_PATH.is_file():
print("progress.json not found", file=sys.stderr)
return 2
try:
data = json.loads(PROGRESS_PATH.read_text(encoding="utf-8"))
except Exception as exc:
print(f"invalid progress.json: {exc}", file=sys.stderr)
return 2
total = data.get("total_tasks")
tasks = data.get("tasks")
if not isinstance(total, int) or total <= 0 or not isinstance(tasks, list):
print("invalid progress shape", file=sys.stderr)
return 2
if len(tasks) != total:
print(f"task count mismatch: total_tasks={total}, tasks={len(tasks)}", file=sys.stderr)
return 2
ids = [task.get("id") for task in tasks if isinstance(task, dict)]
if ids != list(range(1, total + 1)):
print("task ids are not contiguous from 1", file=sys.stderr)
return 2
pending = [task for task in tasks if task.get("status") != "done"]
if pending:
print(f"pending tasks: {len(pending)}/{total}")
print("all tasks done: false")
return 1
print(f"all tasks done: true ({total})")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+710
View File
@@ -0,0 +1,710 @@
{
"version": 1,
"total_tasks": 100,
"completed": 1,
"rounds": 1,
"started_at": "2026-08-29T14:10:48+08:00",
"updated_at": "2026-08-29T14:20:00+08:00",
"tasks": [
{
"id": 1,
"title": "建立 Similar ASIN 性能基线夹具:1000/5000 行、图片开关、chunk 数与 payload 大小采样",
"module": "similarasin",
"dependency": "无",
"status": "done"
},
{
"id": 2,
"title": "将解析载荷改为单一规范行集合,消除 items/groups/allItems 重复数据结构",
"module": "similarasin",
"dependency": "1",
"status": "pending"
},
{
"id": 3,
"title": "保留旧 payload 读取兼容逻辑,并验证新旧结构均可恢复全量行",
"module": "similarasin",
"dependency": "2",
"status": "pending"
},
{
"id": 4,
"title": "解析接口改为只返回固定数量预览行,完整行仅保存在后端任务载荷",
"module": "similarasin",
"dependency": "3",
"status": "pending"
},
{
"id": 5,
"title": "为预览行数量增加配置边界、空文件和超限输入校验",
"module": "similarasin",
"dependency": "4",
"status": "pending"
},
{
"id": 6,
"title": "将分组数据改为索引/范围引用,避免 groups 嵌套复制完整行对象",
"module": "similarasin",
"dependency": "5",
"status": "pending"
},
{
"id": 7,
"title": "限制单文件大小、最大行数和最大字段长度,防止解析任务无界增长",
"module": "similarasin",
"dependency": "6",
"status": "pending"
},
{
"id": 8,
"title": "将 WorkbookFactory 输入解析改为受控读取,并验证超大 Excel 的失败提示",
"module": "similarasin",
"dependency": "7",
"status": "pending"
},
{
"id": 9,
"title": "将 chunk 查询从单行分页改为批量 keyset 分页,保持低内存读取",
"module": "similarasin",
"dependency": "8",
"status": "pending"
},
{
"id": 10,
"title": "为 chunk 结果建立按 row key 的批量索引,消除跨 chunk 线性扫描",
"module": "similarasin",
"dependency": "9",
"status": "pending"
},
{
"id": 11,
"title": "将 Coze 结果合并的重复检测从 O(n²) 改为 HashSet/稳定 row key",
"module": "similarasin",
"dependency": "10",
"status": "pending"
},
{
"id": 12,
"title": "扩展 Coze 结果缓冲覆盖范围,减少频繁读写完整 chunk payload",
"module": "similarasin",
"dependency": "11",
"status": "pending"
},
{
"id": 13,
"title": "为 chunk 合并增加单次最大行数与 payload 字节上限",
"module": "similarasin",
"dependency": "12",
"status": "pending"
},
{
"id": 14,
"title": "图片 DB cache 改为批量读取缩略图,并只更新实际命中的 last_used_at",
"module": "similarasin",
"dependency": "13",
"status": "pending"
},
{
"id": 15,
"title": "将图片缓存访问时间更新改为异步批量刷新,减少逐图 UPDATE",
"module": "similarasin",
"dependency": "14",
"status": "pending"
},
{
"id": 16,
"title": "图片预取改为短预算 best-effort,超时后直接回退 URL",
"module": "similarasin",
"dependency": "15",
"status": "pending"
},
{
"id": 17,
"title": "优化图片解码采样、像素上限和 JPEG 质量搜索,降低 CPU 与堆峰值",
"module": "similarasin",
"dependency": "16",
"status": "pending"
},
{
"id": 18,
"title": "统一图片 spool 生命周期,确保超时、取消和异常路径删除临时文件",
"module": "similarasin",
"dependency": "17",
"status": "pending"
},
{
"id": 19,
"title": "将 Coze 请求/响应及 Python 回传日志改为采样、截断和 DEBUG 级别",
"module": "similarasin",
"dependency": "18",
"status": "pending"
},
{
"id": 20,
"title": "完成 Similar ASIN 端到端压测、JFR/GC 分析与结果文件兼容回归",
"module": "similarasin",
"dependency": "19",
"status": "pending"
},
{
"id": 21,
"title": "建立店铺抓取性能基线:单店铺 1k/5k 行、多国家、图片成功/失败场景",
"module": "shopdatacrawl",
"dependency": "无",
"status": "pending"
},
{
"id": 22,
"title": "将店铺 Excel 图片缓存替换为有界字节缓存",
"module": "shopdatacrawl",
"dependency": "21",
"status": "pending"
},
{
"id": 23,
"title": "图片嵌入成功后立即释放外部缩略图字节副本",
"module": "shopdatacrawl",
"dependency": "22",
"status": "pending"
},
{
"id": 24,
"title": "为店铺图片预取增加任务级数量、字节和超时上限",
"module": "shopdatacrawl",
"dependency": "23",
"status": "pending"
},
{
"id": 25,
"title": "评估并实现店铺结果 workbook 的 SXSSF 或 spool 化写入路径",
"module": "shopdatacrawl",
"dependency": "24",
"status": "pending"
},
{
"id": 26,
"title": "为模板 workbook 增加大行数下的样式、图片和工作表兼容测试",
"module": "shopdatacrawl",
"dependency": "25",
"status": "pending"
},
{
"id": 27,
"title": "将 chunk 接收改为原子插入/幂等 upsert,减少先查后插",
"module": "shopdatacrawl",
"dependency": "26",
"status": "pending"
},
{
"id": 28,
"title": "以 scope 计数器替代每个 chunk 的 COUNT(*) 完整统计",
"module": "shopdatacrawl",
"dependency": "27",
"status": "pending"
},
{
"id": 29,
"title": "合并 scope 状态查询与更新,减少单 chunk 数据库往返",
"module": "shopdatacrawl",
"dependency": "28",
"status": "pending"
},
{
"id": 30,
"title": "为国家结果行建立稳定去重键,替换线性重复扫描",
"module": "shopdatacrawl",
"dependency": "29",
"status": "pending"
},
{
"id": 31,
"title": "将任务快照改为轻量进度字段,避免每次写入完整结果 JSON",
"module": "shopdatacrawl",
"dependency": "30",
"status": "pending"
},
{
"id": 32,
"title": "将 task entity 本地缓存替换为有容量和过期回收的实现",
"module": "shopdatacrawl",
"dependency": "31",
"status": "pending"
},
{
"id": 33,
"title": "将店铺源文件 key 映射改为确定路径,取消临时目录递归扫描",
"module": "shopdatacrawl",
"dependency": "32",
"status": "pending"
},
{
"id": 34,
"title": "将 ownerInstanceId 从 JSON 查询迁移到显式列并补充索引",
"module": "shopdatacrawl",
"dependency": "33",
"status": "pending"
},
{
"id": 35,
"title": "将每日累计文件改为数据层增量模型,避免每次下载并重写完整 XLSX",
"module": "shopdatacrawl",
"dependency": "34",
"status": "pending"
},
{
"id": 36,
"title": "为每日累计文件引入版本号/CAS,缩短店铺级锁的持有时间",
"module": "shopdatacrawl",
"dependency": "35",
"status": "pending"
},
{
"id": 37,
"title": "拆分每日累计文件组装与任务结果接收,增加异步文件作业状态",
"module": "shopdatacrawl",
"dependency": "36",
"status": "pending"
},
{
"id": 38,
"title": "历史列表与进度查询增加分页、字段裁剪和批量任务加载",
"module": "shopdatacrawl",
"dependency": "37",
"status": "pending"
},
{
"id": 39,
"title": "补充删除、超时、重复回传和累计文件失败的资源清理测试",
"module": "shopdatacrawl",
"dependency": "38",
"status": "pending"
},
{
"id": 40,
"title": "完成店铺抓取压测并比较内存、CPU、DB QPS、对象存储流量和锁等待",
"module": "shopdatacrawl",
"dependency": "39",
"status": "pending"
},
{
"id": 41,
"title": "建立 Collect Data 1k/10k 行、多个 chunk 和品牌检测场景基线",
"module": "collectdata",
"dependency": "无",
"status": "pending"
},
{
"id": 42,
"title": "限制采集解析的文件大小、最大行数和单 chunk 行数",
"module": "collectdata",
"dependency": "41",
"status": "pending"
},
{
"id": 43,
"title": "将采集源文件查找改为确定路径/索引查询",
"module": "collectdata",
"dependency": "42",
"status": "pending"
},
{
"id": 44,
"title": "保留原始 chunk payload 的同时,减少逐行 extra JSON 的重复序列化",
"module": "collectdata",
"dependency": "43",
"status": "pending"
},
{
"id": 45,
"title": "将 ASIN 去重查询与无效品牌查询统一为批量集合查询",
"module": "collectdata",
"dependency": "44",
"status": "pending"
},
{
"id": 46,
"title": "跳过空品牌批次的无效远程品牌检查请求",
"module": "collectdata",
"dependency": "45",
"status": "pending"
},
{
"id": 47,
"title": "为品牌检查结果增加任务内短期缓存,避免同品牌重复远程调用",
"module": "collectdata",
"dependency": "46",
"status": "pending"
},
{
"id": 48,
"title": "将 invalid ASIN 记录改为批量 INSERT IGNORE/upsert",
"module": "collectdata",
"dependency": "47",
"status": "pending"
},
{
"id": 49,
"title": "将结果明细从逐行 RustFS 对象改为 chunk 级 payload 存储",
"module": "collectdata",
"dependency": "48",
"status": "pending"
},
{
"id": 50,
"title": "为结果明细设计批量 upsert mapper 与幂等唯一键",
"module": "collectdata",
"dependency": "49",
"status": "pending"
},
{
"id": 51,
"title": "将 accepted 行的序列化和 hash 计算改为批量处理",
"module": "collectdata",
"dependency": "50",
"status": "pending"
},
{
"id": 52,
"title": "生成结果文件时按 chunk 一次读取,取消逐行对象读取",
"module": "collectdata",
"dependency": "51",
"status": "pending"
},
{
"id": 53,
"title": "将 rawRows 与 finalRows 的内存生命周期分段,避免同时长期驻留",
"module": "collectdata",
"dependency": "52",
"status": "pending"
},
{
"id": 54,
"title": "将 finalRowCount 从每个 chunk COUNT(*) 改为任务内增量计数",
"module": "collectdata",
"dependency": "53",
"status": "pending"
},
{
"id": 55,
"title": "将进度统计更新改为节流/合并写,减少高频 task UPDATE",
"module": "collectdata",
"dependency": "54",
"status": "pending"
},
{
"id": 56,
"title": "为采集结果文件增加流式写入失败后的临时文件清理",
"module": "collectdata",
"dependency": "55",
"status": "pending"
},
{
"id": 57,
"title": "为采集结果对象增加数据库删除与物理对象删除的一致性处理",
"module": "collectdata",
"dependency": "56",
"status": "pending"
},
{
"id": 58,
"title": "补充外部品牌服务不可用、RustFS 超时和重复 chunk 的降级测试",
"module": "collectdata",
"dependency": "57",
"status": "pending"
},
{
"id": 59,
"title": "完成采集模块数据库索引、批量 SQL 和对象存储调用次数验证",
"module": "collectdata",
"dependency": "58",
"status": "pending"
},
{
"id": 60,
"title": "完成采集模块 10k 行压测并验收结果完整性、内存和吞吐",
"module": "collectdata",
"dependency": "59",
"status": "pending"
},
{
"id": 61,
"title": "建立共享任务链路资源指标基线:线程、连接、队列、GC、Redis、RustFS 和 DB",
"module": "shared",
"dependency": "无",
"status": "pending"
},
{
"id": 62,
"title": "为本地任务实体缓存增加最大条目数、TTL 和定时清理",
"module": "shared",
"dependency": "61",
"status": "pending"
},
{
"id": 63,
"title": "为前端/后端进度快照增加写入去重和最小更新间隔",
"module": "shared",
"dependency": "62",
"status": "pending"
},
{
"id": 64,
"title": "将 transient payload 压缩改为直接 gzip 二进制流上传",
"module": "shared",
"dependency": "63",
"status": "pending"
},
{
"id": 65,
"title": "为 transient payload 读取增加流式解压和解压后字节上限",
"module": "shared",
"dependency": "64",
"status": "pending"
},
{
"id": 66,
"title": "限制 RustFS 并发读写与重试的总资源预算,防止多任务叠加爆发",
"module": "shared",
"dependency": "65",
"status": "pending"
},
{
"id": 67,
"title": "复用 RustFS/MinIO 客户端与 HTTP 连接池,减少每次操作创建客户端",
"module": "shared",
"dependency": "66",
"status": "pending"
},
{
"id": 68,
"title": "将 payload 引用删除改为批量引用检查与异步物理删除",
"module": "shared",
"dependency": "67",
"status": "pending"
},
{
"id": 69,
"title": "为数据库删除任务补充 transient payload 指针收集和清理队列",
"module": "shared",
"dependency": "68",
"status": "pending"
},
{
"id": 70,
"title": "将历史清理改为 keyset 分页、小批量和短事务",
"module": "shared",
"dependency": "69",
"status": "pending"
},
{
"id": 71,
"title": "清理日志改为数量与 sample ID,禁止输出超长任务 ID 列表",
"module": "shared",
"dependency": "70",
"status": "pending"
},
{
"id": 72,
"title": "为文件作业实现数据库原子 claim,避免重复派发同一 job",
"module": "shared",
"dependency": "71",
"status": "pending"
},
{
"id": 73,
"title": "为本地文件作业队列增加 in-flight 去重和队列背压",
"module": "shared",
"dependency": "72",
"status": "pending"
},
{
"id": 74,
"title": "隔离调度线程池、文件作业线程池和外部 Coze/图片执行池",
"module": "shared",
"dependency": "73",
"status": "pending"
},
{
"id": 75,
"title": "为虚拟线程任务增加等待队列上限与拒绝/延迟指标",
"module": "shared",
"dependency": "74",
"status": "pending"
},
{
"id": 76,
"title": "将 JSON owner 查询迁移到显式列并补充任务/状态复合索引",
"module": "shared",
"dependency": "75",
"status": "pending"
},
{
"id": 77,
"title": "统一 Coze、品牌检查和紫鸟 HTTP 客户端的连接复用策略",
"module": "shared",
"dependency": "76",
"status": "pending"
},
{
"id": 78,
"title": "为所有外部调用增加耗时、重试、失败率和 payload 字节指标",
"module": "shared",
"dependency": "77",
"status": "pending"
},
{
"id": 79,
"title": "为对象存储、数据库和队列增加故障注入测试",
"module": "shared",
"dependency": "78",
"status": "pending"
},
{
"id": 80,
"title": "补充 JVM 堆、直接内存、临时磁盘和连接池容量配置说明",
"module": "shared",
"dependency": "79",
"status": "pending"
},
{
"id": 81,
"title": "建立前端任务轮询请求量、响应体大小和页面内存基线",
"module": "frontend",
"dependency": "无",
"status": "pending"
},
{
"id": 82,
"title": "为进度响应 Map 增加 TTL 清理与最大条目数",
"module": "frontend",
"dependency": "81",
"status": "pending"
},
{
"id": 83,
"title": "统一不同页面的轮询去重、in-flight 合并和终态清理",
"module": "frontend",
"dependency": "82",
"status": "pending"
},
{
"id": 84,
"title": "优化店铺抓取队列状态合并,消除 historyItems 的线性重复查找",
"module": "frontend",
"dependency": "83",
"status": "pending"
},
{
"id": 85,
"title": "优化 Similar ASIN 轮询与文件生成等待,避免重复 force 请求",
"module": "frontend",
"dependency": "84",
"status": "pending"
},
{
"id": 86,
"title": "将隐藏页面轮询间隔、前台恢复和退避策略统一配置化",
"module": "frontend",
"dependency": "85",
"status": "pending"
},
{
"id": 87,
"title": "限制 localStorage 中任务、快照和队列数据的最大数量/字节数",
"module": "frontend",
"dependency": "86",
"status": "pending"
},
{
"id": 88,
"title": "解析结果前端只接收预览数据,避免大 payload 进入响应式对象",
"module": "frontend",
"dependency": "87",
"status": "pending"
},
{
"id": 89,
"title": "清理页面卸载时的所有 timer、请求和临时 URL",
"module": "frontend",
"dependency": "88",
"status": "pending"
},
{
"id": 90,
"title": "为进度接口增加断网、超时、服务恢复和重复响应测试",
"module": "frontend",
"dependency": "89",
"status": "pending"
},
{
"id": 91,
"title": "按页面拆分 Element Plus 与公共业务 chunk",
"module": "frontend",
"dependency": "90",
"status": "pending"
},
{
"id": 92,
"title": "配置 Vite manualChunks 并比较各页面首屏传输大小",
"module": "frontend",
"dependency": "91",
"status": "pending"
},
{
"id": 93,
"title": "补充 Similar ASIN、店铺抓取和采集数据页面的 E2E 核心路径",
"module": "frontend",
"dependency": "92",
"status": "pending"
},
{
"id": 94,
"title": "补充移动端与桌面端响应式页面验收截图",
"module": "frontend",
"dependency": "93",
"status": "pending"
},
{
"id": 95,
"title": "补充深色主题、错误提示、重试和终态刷新验收",
"module": "frontend",
"dependency": "94",
"status": "pending"
},
{
"id": 96,
"title": "建立 Java/Python/Vue 三端统一的 API 字段兼容检查",
"module": "frontend",
"dependency": "95",
"status": "pending"
},
{
"id": 97,
"title": "执行 Java 全量测试、Python unittest、Vue 类型检查与构建",
"module": "frontend",
"dependency": "96",
"status": "pending"
},
{
"id": 98,
"title": "执行真实启动、健康检查、核心请求和外部依赖调用验证",
"module": "frontend",
"dependency": "97",
"status": "pending"
},
{
"id": 99,
"title": "执行全链路压测并记录 CPU、内存、GC、DB、Redis、RustFS、网络结果",
"module": "frontend",
"dependency": "98",
"status": "pending"
},
{
"id": 100,
"title": "完成发布前回滚演练、git commit 对应关系检查和交付清单",
"module": "frontend",
"dependency": "99",
"status": "pending"
}
]
}