super/crawler-plugin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(A1/A3+C8): 客户端令牌链路打通（按人鉴权就绪）+ 快照 JSON 写入节流

Browse Source 
A1/A3 客户端令牌链路（服务端守卫已能按 JWT 鉴权，缺口在客户端不带身份）
- 前端：新增 user-token-bridge（纯逻辑，值变化才推送）+ user-token-sync（启动安装，
  pywebviewready/storage/60s 轮询补推），main.ts 接入；桥接口补 set_user_token
- 客户端：新增 crawler_core/user_token.py（持有 + 对自家 Java 端点注入
  Authorization: Bearer <jwt>，Session.request 包装，第三方域名不注入、已有头不覆盖、
  登出清空、SHUFUAI_DISABLE_USER_TOKEN_HOOK 可关）；main.py 暴露桥方法并在启动安装钩子
- 服务端：守卫契约测试补 5 个用户态前缀用例（开关关闭放行 / 打开后匿名 401 /
  用户令牌通过 / 内部令牌仍放行 / /api/ziniao 第二层）
- 翻开关的前置条件（客户端铺开后置 aiimage.security.user-tool-guard-enabled=true）写入报告

C8 快照 JSON 写入节流
- 读端改以 biz_task_result_item 行为准、JSON 仅兜底（历史任务 JSON 仍是唯一副本时可用）
- 整档 JSON 改 30s 节流写，终态路径 force 立即写；新增 2 个契约测试钉住语义

验证：mvn test 2888 全绿；npm test 741 全绿；user_token 钩子自测（注入/归一/第三方跳过/不覆盖/登出）通过
This commit is contained in:
黄自达
2026-09-14 10:20:15 +08:00
parent 7643094f1d
commit c55c4a140b
10 changed files with 423 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-java/src/main/java/com/nanri/aiimage/modules/queryasin/service/QueryAsinTaskService.java
+50 -6
View File
@@ -63,7 +63,20 @@ public class QueryAsinTaskService {
private static final int RESULT_SUCCESS = 1;
private static final String INTERRUPTED_MESSAGE = "Python 在该店铺结果提交完成前中断";
/**
* 整档快照 JSON 的写入节流间隔（2026-09 审查 C8）。
*
* <p>分片回传很频繁时逐次重写整份文档是 O(n²) 序列化开销；读端已改为**以
* biz_task_result_item 行为准、JSON 仅作兜底**，因此允许 JSON 短时滞后。
* 终态（结果文件生成/结束）走 force 立即写，保证历史详情最终一致。
*/
private static final long SNAPSHOT_JSON_WRITE_THROTTLE_MILLIS = 30_000L;
private final FileTaskMapper fileTaskMapper;
/** taskId → 上次整档 JSON 写入时间（仅本进程节流；跨实例由终态强制写兜底） */
private final java.util.concurrent.ConcurrentHashMap<Long, Long> snapshotJsonWrittenAt =
new java.util.concurrent.ConcurrentHashMap<>();
private final FileResultMapper fileResultMapper;
private final QueryAsinResolveService queryAsinResolveService;
private final QueryAsinExcelAssemblyService excelAssemblyService;
@@ -619,9 +632,11 @@ public class QueryAsinTaskService {
private Map<Long, Map<Long, QueryAsinResultItemVo>> buildSnapshotMap(Map<Long, FileTaskEntity> taskMap) {
Map<Long, Map<Long, QueryAsinResultItemVo>> out = new LinkedHashMap<>();
for (Map.Entry<Long, FileTaskEntity> entry : taskMap.entrySet()) {
List<QueryAsinResultItemVo> snapshots = parseTaskSnapshots(entry.getValue().getResultJson());
// 以行表为准（增量真源），JSON 仅作兜底（历史任务的 JSON 是唯一副本）
List<QueryAsinResultItemVo> snapshots =
taskResultItemService.listResultSnapshots(entry.getKey(), MODULE_TYPE, QueryAsinResultItemVo.class);
if (snapshots.isEmpty()) {
snapshots = taskResultItemService.listResultSnapshots(entry.getKey(), MODULE_TYPE, QueryAsinResultItemVo.class);
snapshots = parseTaskSnapshots(entry.getValue().getResultJson());
}
out.put(entry.getKey(), indexSnapshotByResultId(snapshots));
}
@@ -722,7 +737,12 @@ public class QueryAsinTaskService {
}
private List<QueryAsinResultItemVo> buildSnapshotFromDb(FileTaskEntity task, List<FileResultEntity> rows) {
List<QueryAsinResultItemVo> existing = parseTaskSnapshots(task.getResultJson());
// 同上：行表优先，JSON 兜底
List<QueryAsinResultItemVo> existing =
taskResultItemService.listResultSnapshots(task.getId(), MODULE_TYPE, QueryAsinResultItemVo.class);
if (existing.isEmpty()) {
existing = parseTaskSnapshots(task.getResultJson());
}
Map<Long, QueryAsinResultItemVo> snapshotByResultId = indexSnapshotByResultId(existing);
List<QueryAsinResultItemVo> list = new ArrayList<>();
for (FileResultEntity row : rows) {
@@ -962,7 +982,8 @@ public class QueryAsinTaskService {
}
}
persistSnapshotJson(task, snapshots);
// 终态：强制重写整档 JSON，保证历史详情与结果文件口径一致
persistSnapshotJson(task, snapshots, true);
fileTaskMapper.updateById(task);
taskCacheService.deleteTaskCache(task.getId());
}
@@ -1008,7 +1029,7 @@ public class QueryAsinTaskService {
}
updateResultFileColumns(successRowIds, filename, objectKey, fileSize, rowCount);
updateTaskStatusFromRows(task, rows);
persistSnapshotJson(task, buildSnapshotFromDb(task, rows));
persistSnapshotJson(task, buildSnapshotFromDb(task, rows), true);
fileTaskMapper.updateById(task);
} finally {
FileUtil.del(xlsx);
@@ -1090,14 +1111,37 @@ public class QueryAsinTaskService {
}
private void persistSnapshotJson(FileTaskEntity task, List<QueryAsinResultItemVo> snapshots) {
persistSnapshotJson(task, snapshots, false);
}
/**
* @param force true=立即重写整档 JSON（终态路径用）；false=按节流窗口决定是否重写
*/
private void persistSnapshotJson(FileTaskEntity task, List<QueryAsinResultItemVo> snapshots, boolean force) {
try {
task.setResultJson(objectMapper.writeValueAsString(snapshots == null ? List.of() : snapshots));
// 行表始终同步（增量 upsert，内容未变时自身会跳过）
syncSnapshotTables(task, snapshots);
if (!force && !snapshotJsonWriteDue(task)) {
return;
}
task.setResultJson(objectMapper.writeValueAsString(snapshots == null ? List.of() : snapshots));
if (task.getId() != null) {
snapshotJsonWrittenAt.put(task.getId(), System.currentTimeMillis());
}
} catch (Exception ex) {
throw new BusinessException("查询ASIN任务快照保存失败");
}
}
private boolean snapshotJsonWriteDue(FileTaskEntity task) {
if (task == null || task.getId() == null) {
return true;
}
Long lastWrittenAt = snapshotJsonWrittenAt.get(task.getId());
return lastWrittenAt == null
|| System.currentTimeMillis() - lastWrittenAt >= SNAPSHOT_JSON_WRITE_THROTTLE_MILLIS;
}
private void syncSnapshotTables(FileTaskEntity task, List<QueryAsinResultItemVo> snapshots) {
List<QueryAsinResultItemVo> safe = snapshots == null ? List.of() : snapshots;
taskResultItemService.replaceTaskSnapshots(task.getId(), MODULE_TYPE, safe, new TaskResultItemService.SnapshotKeyResolver() {
backend-java/src/test/java/com/nanri/aiimage/config/AdminApiGuardFilterTest.java
+85 -1
View File
@@ -29,9 +29,15 @@ class AdminApiGuardFilterTest {
private final ObjectMapper objectMapper = new ObjectMapper();
private AdminApiGuardFilter newFilter(AdminAuthSupport adminAuthSupport, boolean enabled, String exemptPrefixes) {
return newFilter(adminAuthSupport, enabled, exemptPrefixes, false);
}
private AdminApiGuardFilter newFilter(AdminAuthSupport adminAuthSupport, boolean enabled, String exemptPrefixes,
boolean userToolGuardEnabled) {
AdminApiGuardFilter filter = new AdminApiGuardFilter(adminAuthSupport, objectMapper);
ReflectionTestUtils.setField(filter, "enabled", enabled);
ReflectionTestUtils.setField(filter, "exemptPrefixes", exemptPrefixes);
ReflectionTestUtils.setField(filter, "userToolGuardEnabled", userToolGuardEnabled);
return filter;
}
@@ -251,4 +257,82 @@ class AdminApiGuardFilterTest {
}
verify(authSupport, never()).requireUserOrInternal(any());
}
}
// ---- 用户态工具前缀（A1/A3）：开关默认关闭，打开后按"用户令牌或内部令牌"鉴权 ----
@Test
void userToolPrefixesAreOpenWhileGuardDisabled() throws Exception {
AdminAuthSupport authSupport = mock(AdminAuthSupport.class);
AdminApiGuardFilter filter = newFilter(authSupport, true, "", false);
MockHttpServletRequest request = new MockHttpServletRequest("POST", "/api/patrol-delete/tasks/submit");
MockHttpServletResponse response = new MockHttpServletResponse();
MockFilterChain chain = new MockFilterChain();
filter.doFilter(request, response, chain);
verify(authSupport, never()).requireUserOrInternal(any());
assertThat(chain.getRequest()).isNotNull();
}
@Test
void enabledUserToolGuardRejectsAnonymous() throws Exception {
AdminAuthSupport authSupport = mock(AdminAuthSupport.class);
when(authSupport.requireUserOrInternal(any())).thenThrow(new BusinessException(401, "未登录"));
AdminApiGuardFilter filter = newFilter(authSupport, true, "", true);
MockHttpServletRequest request = new MockHttpServletRequest("POST", "/api/patrol-delete/tasks/submit");
MockHttpServletResponse response = new MockHttpServletResponse();
MockFilterChain chain = new MockFilterChain();
filter.doFilter(request, response, chain);
assertThat(chain.getRequest()).isNull();
assertThat(response.getStatus()).isEqualTo(200);
JsonNode body = objectMapper.readTree(response.getContentAsString());
assertThat(body.path("success").asBoolean()).isFalse();
assertThat(body.path("code").asInt()).isEqualTo(401);
}
@Test
void enabledUserToolGuardAcceptsUserToken() throws Exception {
AdminAuthSupport authSupport = mock(AdminAuthSupport.class);
when(authSupport.requireUserOrInternal(any())).thenReturn(new AdminUserEntity());
AdminApiGuardFilter filter = newFilter(authSupport, true, "", true);
MockHttpServletRequest request = new MockHttpServletRequest("POST", "/api/withdraw/tasks/submit");
MockHttpServletResponse response = new MockHttpServletResponse();
MockFilterChain chain = new MockFilterChain();
filter.doFilter(request, response, chain);
assertThat(chain.getRequest()).isNotNull();
verify(authSupport).requireUserOrInternal(any());
}
@Test
void enabledUserToolGuardStillLetsTrustedInternalTokenThrough() throws Exception {
AdminAuthSupport authSupport = mock(AdminAuthSupport.class);
when(authSupport.isTrustedInternalToken(any())).thenReturn(true);
AdminApiGuardFilter filter = newFilter(authSupport, true, "", true);
MockHttpServletRequest request = new MockHttpServletRequest("POST", "/api/collect-data/tasks/submit");
MockHttpServletResponse response = new MockHttpServletResponse();
MockFilterChain chain = new MockFilterChain();
filter.doFilter(request, response, chain);
verify(authSupport, never()).requireUserOrInternal(any());
assertThat(chain.getRequest()).isNotNull();
}
@Test
void enabledUserToolGuardCoversZiniaoPrefixAsSecondLayer() throws Exception {
AdminAuthSupport authSupport = mock(AdminAuthSupport.class);
when(authSupport.requireUserOrInternal(any())).thenThrow(new BusinessException(401, "未登录"));
AdminApiGuardFilter filter = newFilter(authSupport, true, "", true);
MockHttpServletRequest request = new MockHttpServletRequest("GET", "/api/ziniao/staff");
MockHttpServletResponse response = new MockHttpServletResponse();
MockFilterChain chain = new MockFilterChain();
filter.doFilter(request, response, chain);
assertThat(chain.getRequest()).isNull();
assertThat(response.getContentAsString()).contains("401");
}
}
backend-java/src/test/java/com/nanri/aiimage/modules/queryasin/service/QueryAsinSnapshotJsonThrottleTest.java
+95
View File
@@ -0,0 +1,95 @@
package com.nanri.aiimage.modules.queryasin.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.nanri.aiimage.modules.queryasin.model.vo.QueryAsinResultItemVo;
import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity;
import com.nanri.aiimage.modules.task.service.TaskProgressSnapshotService;
import com.nanri.aiimage.modules.task.service.TaskResultItemService;
import org.junit.jupiter.api.Test;
import org.springframework.test.util.ReflectionTestUtils;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
/**
* 快照 JSON 整档写入的节流语义（2026-09 审查 C8）。
*
* <p>读端已改为"以 biz_task_result_item 行为准、JSON 兜底"，因此允许 JSON 短时滞后：
* 节流窗口内只同步行表、不重写整档 JSON；终态路径传 force=true 立即重写。
* 本测试直接钉住这两条契约（其余依赖与本次语义无关，传 null/mock）。
*/
class QueryAsinSnapshotJsonThrottleTest {
private final ObjectMapper objectMapper = new ObjectMapper();
private final TaskResultItemService taskResultItemService = mock(TaskResultItemService.class);
private final TaskProgressSnapshotService taskProgressSnapshotService = mock(TaskProgressSnapshotService.class);
private QueryAsinTaskService service() {
// 构造顺序：mapper/resultMapper/resolve/excel/cache/oss/resultDownloadResolver/ziniaoSwitch/
// objectMapper/pressure/jobService/resultItemService/snapshot/lock/lightAssembler
return new QueryAsinTaskService(
null, null, null, null, null, null, null, null,
objectMapper, null, null, taskResultItemService, taskProgressSnapshotService, null, null);
}
private static void persist(QueryAsinTaskService service, FileTaskEntity task,
List<QueryAsinResultItemVo> snapshots, boolean force) throws Exception {
Method method = QueryAsinTaskService.class.getDeclaredMethod(
"persistSnapshotJson", FileTaskEntity.class, List.class, boolean.class);
method.setAccessible(true);
method.invoke(service, task, snapshots, force);
}
private static QueryAsinResultItemVo item(long resultId, String shopName) {
QueryAsinResultItemVo vo = new QueryAsinResultItemVo();
vo.setResultId(resultId);
vo.setShopName(shopName);
vo.setSuccess(true);
return vo;
}
@Test
void firstWriteGoesThroughThenThrottledUntilForced() throws Exception {
QueryAsinTaskService service = service();
FileTaskEntity task = new FileTaskEntity();
task.setId(9001L);
List<QueryAsinResultItemVo> first = List.of(item(1L, "店铺A"));
persist(service, task, first, false);
String afterFirst = task.getResultJson();
assertTrue(afterFirst != null && afterFirst.contains("店铺A"), "首次写入立即生效（无历史写入时间）");
// 节流窗口内：行表仍同步，但 JSON 不重写（新数据要等窗口结束或 force）
List<QueryAsinResultItemVo> second = List.of(item(1L, "店铺A"), item(2L, "店铺B"));
persist(service, task, second, false);
assertEquals(afterFirst, task.getResultJson(), "窗口内不重写整档 JSON");
verify(taskResultItemService, times(2))
.replaceTaskSnapshots(anyLong(), anyString(), any(), any());
// 终态：force 立即重写
persist(service, task, second, true);
assertTrue(task.getResultJson().contains("店铺B"), "force 立即重写整档 JSON");
}
@Test
void emptySnapshotStillSyncsRowTable() throws Exception {
QueryAsinTaskService service = service();
FileTaskEntity task = new FileTaskEntity();
task.setId(9002L);
persist(service, task, new ArrayList<>(), false);
verify(taskResultItemService).replaceTaskSnapshots(anyLong(), anyString(), any(), any());
assertEquals("[]", task.getResultJson(), "空快照序列化为空数组");
}
}