From 91d36327f93aa1501b1d909aa6af83d1ff168cb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E8=87=AA=E8=BE=BE?= <980324341@qq.com> Date: Sat, 31 Oct 2026 21:12:18 +0800 Subject: [PATCH] =?UTF-8?q?fix(phase-5.2):=20=E6=B6=88=E9=A1=B9=20CR=20?= =?UTF-8?q?=E7=AC=AC=E4=BA=8C=E8=BD=AE=20P1/P2/P3=EF=BC=88=E9=80=9A?= =?UTF-8?q?=E9=81=93/=E8=AF=81=E6=8D=AE=E9=93=BE/=E4=BA=A4=E4=BB=98?= =?UTF-8?q?=E6=97=B6=E5=BA=8F/=E5=90=8E=E7=AE=A1=E8=AF=81=E6=8D=AE?= =?UTF-8?q?=E9=A1=B5=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- admin-frontend-vue/src/api/chunk-upload.ts | 177 ++++++++-- admin-frontend-vue/src/api/task-evidence.ts | 61 ++++ .../src/config/company-scope.ts | 2 +- .../src/layout/operation-guides.ts | 4 + .../records/RecordsSoftwareVersionPage.vue | 10 +- .../src/pages/records/RecordsTutorialPage.vue | 10 +- .../src/pages/records/TaskEvidencePage.vue | 305 ++++++++++++++++++ .../src/pages/records/tutorial-api.ts | 2 - .../src/pages/records/version-api.ts | 2 - admin-frontend-vue/src/router/routes.ts | 1 + .../tests/baselines/source-pin-ledger.json | 13 +- ...records-upload-error-chinese.mount.test.ts | 155 +++++++++ .../tests/pending-phase4-大档上传接线.test.ts | 99 ++++-- .../tests/pure/chunk-upload-resume.test.ts | 226 +++++++++++++ .../pure/pending-phase4-大档上传接线.test.ts | 52 ++- .../tests/task-evidence-page.test.ts | 32 ++ .../security/CredentialPackageGuard.java | 86 +++++ .../AppearancePatentController.java | 4 +- .../service/AppearancePatentTaskService.java | 23 +- .../AppearancePatentPayloadSupport.java | 21 ++ .../AppearancePatentTaskRecordSupport.java | 18 +- .../AdminTaskEvidenceController.java | 3 +- .../InternalTaskEvidenceController.java | 8 +- .../evidence/service/TaskEvidenceService.java | 148 +++++++-- .../storage/TaskEvidenceStorageService.java | 5 + .../file/service/ChunkedUploadService.java | 26 +- .../file/service/LocalTempCleanupService.java | 29 +- .../file/service/oss/OssStorageService.java | 30 ++ .../service/PermissionMenuService.java | 3 +- .../service/SimilarAsinTaskService.java | 6 - .../support/SimilarAsinPipelineSupport.java | 8 +- .../service/SoftwareVersionService.java | 24 ++ .../modules/task/mapper/FileTaskMapper.java | 31 ++ .../modules/task/model/CollectChannel.java | 22 +- .../task/service/TaskClientPullService.java | 44 +-- .../task/service/TaskFileJobService.java | 28 +- .../task/service/TaskResultFileJobWorker.java | 21 +- .../service/TutorialPackageService.java | 26 ++ .../resources/application-local.example.yml | 2 + ...184__collect_pull_and_evidence_indexes.sql | 50 +++ .../resources/db/V185__task_evidence_menu.sql | 15 + .../resources/mapper/task/FileTaskMapper.xml | 44 +++ .../aiimage/MigratedSqlEquivalenceTest.java | 12 +- .../aiimage/SqlLayeringConventionTest.java | 6 +- .../security/CredentialPackageGuardTest.java | 68 ++++ .../config/CatchSilenceRatchetTest.java | 8 +- .../AppearancePatentActivateRoutingTest.java | 12 +- ...TaskEvidenceOwnershipAndRetentionTest.java | 189 +++++++++++ ...oftwareVersionServiceChunkConfirmTest.java | 145 +++++++++ .../mapper/FileTaskMapperChannelSqlTest.java | 157 +++++++++ .../task/model/CollectChannelTest.java | 21 ++ .../service/TaskClientPullServiceTest.java | 159 ++++----- .../TaskFileJobTerminalTaskGuardTest.java | 42 +-- .../TaskResultFileJobWorkerStuckScanTest.java | 2 +- ...utorialPackageServiceChunkConfirmTest.java | 111 +++++++ frontend-vue/src/shared/api/upload.ts | 82 ++++- .../src/shared/bridges/web-fallback.ts | 6 +- .../pending-phase4-大档上传接线.mount.test.ts | 36 +++ .../mount/分片会话探测失败分因.mount.test.ts | 185 +++++++++++ .../tests/pending-phase4-大档上传接线.test.ts | 20 ++ .../tests/pending-phase4-用户端稳定性.test.ts | 7 + 61 files changed, 2833 insertions(+), 311 deletions(-) create mode 100644 admin-frontend-vue/src/api/task-evidence.ts create mode 100644 admin-frontend-vue/src/pages/records/TaskEvidencePage.vue create mode 100644 admin-frontend-vue/tests/mount/records-upload-error-chinese.mount.test.ts create mode 100644 admin-frontend-vue/tests/pure/chunk-upload-resume.test.ts create mode 100644 admin-frontend-vue/tests/task-evidence-page.test.ts create mode 100644 backend-java/src/main/java/com/nanri/aiimage/common/security/CredentialPackageGuard.java create mode 100644 backend-java/src/main/resources/db/V184__collect_pull_and_evidence_indexes.sql create mode 100644 backend-java/src/main/resources/db/V185__task_evidence_menu.sql create mode 100644 backend-java/src/test/java/com/nanri/aiimage/common/security/CredentialPackageGuardTest.java create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/evidence/service/TaskEvidenceOwnershipAndRetentionTest.java create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/softwareversion/service/SoftwareVersionServiceChunkConfirmTest.java create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/task/mapper/FileTaskMapperChannelSqlTest.java create mode 100644 backend-java/src/test/java/com/nanri/aiimage/modules/tutorial/service/TutorialPackageServiceChunkConfirmTest.java create mode 100644 frontend-vue/tests/mount/分片会话探测失败分因.mount.test.ts diff --git a/admin-frontend-vue/src/api/chunk-upload.ts b/admin-frontend-vue/src/api/chunk-upload.ts index a0a42d3df..d50e5060c 100644 --- a/admin-frontend-vue/src/api/chunk-upload.ts +++ b/admin-frontend-vue/src/api/chunk-upload.ts @@ -1,38 +1,172 @@ import { http, unwrap } from '@/api/http' -/** 大档分片阈值(AC-11):8MB 以上不再一次 PUT 直传,断点续传由服务端 chunk 通道承担。 */ -export const CHUNK_UPLOAD_THRESHOLD = 8 * 1024 * 1024 +/** + * 大档分片阈值与片大小(AC-11):后管这一侧只有这一个出口,阈值由片大小派生。 + * + * ⚠ 同族的 8MB 数还另有两处:用户端上传出口一处、桌面客户端分片入口一处。两边是各自 + * 独立的构建根(各自的 `@` 别名与 tsconfig include),客户端是 Python 打包物 ⇒ + * 没有共享包就合不成同一个字面量(P3-5 的边界,已登记)。 + */ export const CHUNK_UPLOAD_PART_SIZE = 8 * 1024 * 1024 +export const CHUNK_UPLOAD_THRESHOLD = CHUNK_UPLOAD_PART_SIZE /** 每片重试次数(不含首次):单片 502/超时不该让 512MB 从 0 再来。 */ const PART_RETRY_LIMIT = 2 +/** + * 逐片超时(P2-8):预算只看**这一片**有多大——60s 保底 + 每 MB 30s(8MB 片=300s)。 + * 过去按整包体积算(120s + MB×15s),512MB 包里的一个 8MB 片能拿到 128 分钟, + * 慢链路真卡死时没人知道,进度条还会一直爬。 + */ +export function partTimeoutOf(partBytes: number): number { + return 60_000 + Math.ceil(partBytes / (1024 * 1024)) * 30_000 +} + +/** 会话登记/查询只发元数据,固定预算即可;失败要快快回来,别陪整包一起放大。 */ +const REQUEST_TIMEOUT_MS = 60_000 +/** 只有拼装这一步真随整包体积走:服务端要读全部分片、复算 sha256。 */ +const COMPLETE_TIMEOUT_MS = 600_000 + function partUrl(uploadId: string, index: number): string { return `/api/files/chunk/${encodeURIComponent(uploadId)}/part/${index}` } -/** 后管大档上传(版本包/教程包共用):init → GET 已达片号 → 逐片 POST → complete。 */ +/** 分片会话跨尝试/刷新续用:同一文件身份复用 upload_id,服务端没回收就从已达片号继续(P2-7)。 */ +interface StoredChunkSession { + uploadId: string; + size: number; + lastModified: number; +} + +const CHUNK_SESSION_PREFIX = 'shufu:admin:chunk:' + +function lastModifiedOf(file: Blob): number { + const value = (file as File).lastModified + return typeof value === 'number' && Number.isFinite(value) ? value : 0 +} + +/** 会话键:文件名+体积+修改时间,三者同一条上传才算同一个件。 */ +export function chunkSessionKeyOf(fileName: string, file: Blob): string { + return `${CHUNK_SESSION_PREFIX}${fileName}:${file.size}:${lastModifiedOf(file)}` +} + +function readStoredSession(key: string, file: Blob): string { + if (typeof localStorage === 'undefined') return '' + try { + const raw = localStorage.getItem(key) + if (!raw) return '' + const parsed = JSON.parse(raw) as StoredChunkSession + return parsed.size === file.size && parsed.lastModified === lastModifiedOf(file) ? parsed.uploadId : '' + } catch { + return '' + } +} + +function writeStoredSession(key: string, file: Blob, uploadId: string): void { + if (typeof localStorage === 'undefined') return + try { + localStorage.setItem(key, JSON.stringify({ + uploadId, size: file.size, lastModified: lastModifiedOf(file), + } satisfies StoredChunkSession)) + } catch { + // 存储不可用(隐私模式/配额满)只影响跨刷新续传,不影响本次上传 + } +} + +function clearStoredSession(key: string): void { + if (typeof localStorage === 'undefined') return + try { + localStorage.removeItem(key) + } catch { + // 同上:清不掉只会让下次多问一次 status + } +} + +/** + * 服务端对「这个会话还在不在」的三种答复:还在(附带已到片号)/确实没了/问不出来。 + * 第三态是 P2-7 与 P1-11 的共同判据——过去它被当成"没会话",瞬态抖动=整包作废。 + */ +type ProbeOutcome + = { state: 'ok'; parts: number[] } + | { state: 'gone' } + | { state: 'transient' } + +/** 后端回收会话的原文案(ChunkedUploadService.readMeta);BusinessException 不带 code ⇒ HTTP 200 出。 */ +function messageSaysSessionGone(text: unknown): boolean { + const message = typeof text === 'string' ? text : '' + return ['不存在', '过期'].some((marker) => message.includes(marker)) +} + +function classifyProbe(response: unknown): ProbeOutcome { + const status = Number((response as { status?: unknown } | null | undefined)?.status) + if (status === 404) return { state: 'gone' } + if (status >= 500 || status === 408 || status === 429) return { state: 'transient' } + const body = (response as { data?: unknown } | null | undefined)?.data + if (body && typeof body === 'object' && (body as { success?: unknown }).success === false) { + return messageSaysSessionGone((body as { message?: unknown }).message) + ? { state: 'gone' } + : { state: 'transient' } + } + const core = (unwrap>(body) ?? {}) as Record + const arrived = core.uploaded_parts + return { state: 'ok', parts: Array.isArray(arrived) ? arrived.map((value) => Number(value)) : [] } +} + +/** + * 探测已达片号(P2-7/与用户端 upload.ts 同一口径)。 + * validateStatus 恒真:状态码要自己判——真 404/5xx 一旦被拦截器折成 Error 就分不出原因了; + * 能走到 catch 的就只剩网络层失败与超时,铁定瞬态。 + */ +async function probeUploadedParts(uploadId: string): Promise { + try { + const response = await http.get( + `/api/files/chunk/${encodeURIComponent(uploadId)}`, + { validateStatus: () => true }, + ) + return classifyProbe(response) + } catch (error) { + const status = (error as { response?: { status?: unknown } } | null | undefined)?.response?.status + if (status === 404) return { state: 'gone' } + if (messageSaysSessionGone((error as { message?: unknown } | null | undefined)?.message)) { + return { state: 'gone' } + } + return { state: 'transient' } + } +} + +/** 后管大档上传(版本包/教程包共用):会话续用 → GET 已达片号 → 逐片 POST → complete。 */ export async function uploadFileInChunks( file: Blob, fileName: string, onProgress?: (percent: number) => void, ): Promise> { const totalParts = Math.max(1, Math.ceil(file.size / CHUNK_UPLOAD_PART_SIZE)) - const initResponse = await http.post('/api/files/chunk/init', null, { - params: { fileName, totalParts, totalBytes: file.size }, - timeout: 120_000 + Math.ceil(file.size / (1024 * 1024)) * 15_000, - }) - const init = (unwrap(initResponse.data) ?? {}) as Record - const uploadId = String(init.upload_id ?? '') - if (!uploadId) { - throw new Error('后端未返回分片会话,请重试') + const sessionKey = chunkSessionKeyOf(fileName, file) + let uploadId = readStoredSession(sessionKey, file) + const arrived = new Set() + if (uploadId) { + const probed = await probeUploadedParts(uploadId) + if (probed.state === 'gone') { + // 服务端确实说会话没了(404/「不存在或已过期」):清掉重开,否则这个件每次都在同一个死会话上失败。 + clearStoredSession(sessionKey) + uploadId = '' + } else if (probed.state === 'ok') { + probed.parts.forEach((index) => arrived.add(index)) + } + // transient:片号未知但会话还在 ⇒ 照原会话逐片重传(后端同片覆盖,幂等),不重开 init。 + } + if (!uploadId) { + const initResponse = await http.post('/api/files/chunk/init', null, { + params: { fileName, totalParts, totalBytes: file.size }, + timeout: REQUEST_TIMEOUT_MS, + }) + const init = (unwrap(initResponse.data) ?? {}) as Record + uploadId = String(init.upload_id ?? '') + if (!uploadId) { + throw new Error('后端未返回分片会话,请重试') + } + writeStoredSession(sessionKey, file, uploadId) } - - const statusResponse = await http.get(`/api/files/chunk/${encodeURIComponent(uploadId)}`) - const status = (unwrap(statusResponse.data) ?? {}) as Record - const arrived = new Set( - Array.isArray(status.uploaded_parts) ? status.uploaded_parts.map((value) => Number(value)) : [], - ) for (let index = 0; index < totalParts; index += 1) { if (arrived.has(index)) { @@ -47,7 +181,7 @@ export async function uploadFileInChunks( try { const partResponse = await http.post(partUrl(uploadId, index), form, { headers: { 'Content-Type': 'multipart/form-data' }, - timeout: 120_000 + Math.ceil(file.size / (1024 * 1024)) * 15_000, + timeout: partTimeoutOf(blob.size), onUploadProgress: (event) => { if (!onProgress) return const perPart = Math.round(((index + (event.loaded || 0) / Math.max(blob.size, 1)) / totalParts) * 100) @@ -63,14 +197,15 @@ export async function uploadFileInChunks( } } if (lastError) { - // 后管上传是一次性的:放弃时把会话连同临时片一起清掉,不留 24h 残盘。 - await http.delete(`/api/files/chunk/${encodeURIComponent(uploadId)}`).catch(() => undefined) + // 不 DELETE(P2-7):会话连同已达片号留在服务端(后端 24h 过期回收兜底), + // 下一次点开同一个文件就从断点接着传——这才是"续传"两个字的本义。 throw lastError instanceof Error ? lastError : new Error('分片上传失败') } } const completeResponse = await http.post( - `/api/files/chunk/${encodeURIComponent(uploadId)}/complete`, null, { timeout: 600_000 }) + `/api/files/chunk/${encodeURIComponent(uploadId)}/complete`, null, { timeout: COMPLETE_TIMEOUT_MS }) + clearStoredSession(sessionKey) if (onProgress) onProgress(100) return (unwrap(completeResponse.data) ?? {}) as Record } diff --git a/admin-frontend-vue/src/api/task-evidence.ts b/admin-frontend-vue/src/api/task-evidence.ts new file mode 100644 index 000000000..a8aec8d8e --- /dev/null +++ b/admin-frontend-vue/src/api/task-evidence.ts @@ -0,0 +1,61 @@ +import { http } from './http' +import { unwrap } from './envelope' + +/** 任务证据件列表行(AC-13:后管只读面,仅超级管理员)。 */ +export interface TaskEvidenceRow { + id: number + taskId: string + uid: number | null + evidenceType: string + source: string + objectKey: string + byteSize: number + contentType: string + sha256: string + createdAt: string | null +} + +export interface TaskEvidencePage { + items: TaskEvidenceRow[] + total: number + page: number + pageSize: number +} + +export interface TaskEvidenceQuery { + taskId?: string + uid?: number + page: number + pageSize: number +} + +export interface TaskEvidenceContent { + id: number + taskId: string + contentType: string + binary: boolean + content: string +} + +export async function fetchTaskEvidenceFiles(params: TaskEvidenceQuery): Promise { + const query: Record = { page: params.page, pageSize: params.pageSize } + if (params.taskId) query.taskId = params.taskId + if (params.uid) query.uid = params.uid + const { data } = await http.get('/api/admin/task-evidence/files', { params: query }) + return unwrap(data) +} + +export async function fetchTaskEvidenceContent(fileId: number): Promise { + const { data } = await http.get('/api/admin/task-evidence/content', { params: { fileId } }) + return unwrap(data) +} + +/** 下载走服务端流,不把对象直链/预签名 URL 交给浏览器(AC-13 的不入日志口径)。 */ +export function taskEvidenceDownloadUrl(fileId: number): string { + return `/api/admin/task-evidence/download?fileId=${fileId}` +} + +export async function deleteTaskEvidence(ids: number[]): Promise { + const { data } = await http.post('/api/admin/task-evidence/delete', ids) + return unwrap<{ deleted: number }>(data).deleted +} diff --git a/admin-frontend-vue/src/config/company-scope.ts b/admin-frontend-vue/src/config/company-scope.ts index 876930c3a..0d5e1df48 100644 --- a/admin-frontend-vue/src/config/company-scope.ts +++ b/admin-frontend-vue/src/config/company-scope.ts @@ -3,7 +3,7 @@ * 而护栏用例要从这里反过来读注册表,放在一起会成环。 */ /** 角色内在闸门:只有超管可见可读写,与 user_column_permission 无关。 */ -export const SUPER_ADMIN_ONLY_MENU_KEYS: readonly string[] = ['admin_companies'] +export const SUPER_ADMIN_ONLY_MENU_KEYS: readonly string[] = ['admin_companies', 'admin_task_evidence'] export interface CompanyScopedPage { /** 与 routes.ts 的 menuKey 逐字一致。 */ diff --git a/admin-frontend-vue/src/layout/operation-guides.ts b/admin-frontend-vue/src/layout/operation-guides.ts index a8b3c76a3..87a3b890a 100644 --- a/admin-frontend-vue/src/layout/operation-guides.ts +++ b/admin-frontend-vue/src/layout/operation-guides.ts @@ -109,6 +109,10 @@ export const OPERATION_GUIDES: Record = { text: '桌面客户端与麦象采集机上报的日志按来源、日期和关键字筛选;可查看日志尾部、下载完整文件或删除云端记录,云端日志按保留期自动清理。', steps: ['选择来源与日期', '查看或下载日志', '必要时删除'], }, + admin_task_evidence: { + text: '任务异常现场件与窗口图按任务号关联;只有超级管理员能读取与下载,对象直链不出服务端,证据件按保留期自动清理。', + steps: ['按任务号或 uid 查询', '查看现场件或窗口图', '下载或清理过期证据'], + }, admin_user_secret_usage: { text: '按日期范围统计各用户的密钥调用次数(货源查询与外观专利按 LLM 请求次数、代理按提取次数),用于评估用户资源损耗。', steps: ['选择日期范围', '筛选用户或分组', '核对各模块次数'], diff --git a/admin-frontend-vue/src/pages/records/RecordsSoftwareVersionPage.vue b/admin-frontend-vue/src/pages/records/RecordsSoftwareVersionPage.vue index 555db2d5f..b71e8395d 100644 --- a/admin-frontend-vue/src/pages/records/RecordsSoftwareVersionPage.vue +++ b/admin-frontend-vue/src/pages/records/RecordsSoftwareVersionPage.vue @@ -9,6 +9,7 @@ import { fetchSoftwareVersions, uploadSoftwareVersion, deleteSoftwareVersions } import type { SoftwareVersionItem } from './version-dto.ts' import { isNonEmptyVersion } from './version-dto.ts' import { showAdminFeedback } from '@/components/admin-feedback-ui' +import { loadFailureMessage } from '@/components/load-failure-model' import { openAdminConfirm } from '@/components/admin-confirm' import LoadFailureBar from '@/components/LoadFailureBar.vue' @@ -62,7 +63,7 @@ async function removeSelected() { selectedIds.value = new Set() await load() } catch (error) { - showAdminFeedback(error instanceof Error ? error.message : '删除失败', 'error') + showAdminFeedback(loadFailureMessage(error, '删除失败'), 'error') } } async function removeOne(row: SoftwareVersionItem) { @@ -73,7 +74,7 @@ async function removeOne(row: SoftwareVersionItem) { selectedIds.value = new Set([...selectedIds.value].filter((id) => id !== row.id)) await load() } catch (error) { - showAdminFeedback(error instanceof Error ? error.message : '删除失败', 'error') + showAdminFeedback(loadFailureMessage(error, '删除失败'), 'error') } } @@ -92,7 +93,7 @@ async function load() { try { items.value = (await fetchSoftwareVersions()).items } catch (error) { - loadFailure.value = error instanceof Error ? error.message : '版本列表加载失败' + loadFailure.value = loadFailureMessage(error, '版本列表加载失败') showAdminFeedback(loadFailure.value, 'error') } finally { loading.value = false @@ -146,7 +147,8 @@ async function submitUpload() { uploadPercent.value = 0 load() } catch (error) { - uploadMsg.value = error instanceof Error ? error.message : '上传失败' + // P2-10:api/http 拦截器不 sanitize,error.message 可能是英文异常原文——界面只吃中文出口,原文进 console。 + uploadMsg.value = loadFailureMessage(error, '上传失败') } finally { uploading.value = false } diff --git a/admin-frontend-vue/src/pages/records/RecordsTutorialPage.vue b/admin-frontend-vue/src/pages/records/RecordsTutorialPage.vue index 4d6cfe916..5399f21a7 100644 --- a/admin-frontend-vue/src/pages/records/RecordsTutorialPage.vue +++ b/admin-frontend-vue/src/pages/records/RecordsTutorialPage.vue @@ -11,6 +11,7 @@ import { fetchTutorialPackages, uploadTutorialPackage, deleteTutorialPackages } import { formatFileSize } from './tutorial-model.ts' import type { TutorialPackageItem } from './tutorial-dto.ts' import { showAdminFeedback } from '@/components/admin-feedback-ui' +import { loadFailureMessage } from '@/components/load-failure-model' import { openAdminConfirm } from '@/components/admin-confirm' import LoadFailureBar from '@/components/LoadFailureBar.vue' @@ -119,7 +120,7 @@ async function removeSelected() { selectedIds.value = new Set() await load() } catch (error) { - showAdminFeedback(error instanceof Error ? error.message : '删除失败', 'error') + showAdminFeedback(loadFailureMessage(error, '删除失败'), 'error') } } async function removeOne(row: TutorialPackageItem) { @@ -130,7 +131,7 @@ async function removeOne(row: TutorialPackageItem) { selectedIds.value = new Set([...selectedIds.value].filter((id) => id !== row.id)) await load() } catch (error) { - showAdminFeedback(error instanceof Error ? error.message : '删除失败', 'error') + showAdminFeedback(loadFailureMessage(error, '删除失败'), 'error') } } @@ -149,7 +150,7 @@ async function load() { try { items.value = (await fetchTutorialPackages()).items } catch (error) { - loadFailure.value = error instanceof Error ? error.message : '教程包列表加载失败' + loadFailure.value = loadFailureMessage(error, '教程包列表加载失败') showAdminFeedback(loadFailure.value, 'error') } finally { loading.value = false @@ -198,7 +199,8 @@ async function submitUpload() { uploadPercent.value = 0 load() } catch (error) { - uploadMsg.value = error instanceof Error ? error.message : '上传失败' + // P2-10:api/http 拦截器不 sanitize,error.message 可能是英文异常原文——界面只吃中文出口,原文进 console。 + uploadMsg.value = loadFailureMessage(error, '上传失败') } finally { uploading.value = false } diff --git a/admin-frontend-vue/src/pages/records/TaskEvidencePage.vue b/admin-frontend-vue/src/pages/records/TaskEvidencePage.vue new file mode 100644 index 000000000..2c37d90c3 --- /dev/null +++ b/admin-frontend-vue/src/pages/records/TaskEvidencePage.vue @@ -0,0 +1,305 @@ + + + + + diff --git a/admin-frontend-vue/src/pages/records/tutorial-api.ts b/admin-frontend-vue/src/pages/records/tutorial-api.ts index 9af15aefd..3b7c1ec4e 100644 --- a/admin-frontend-vue/src/pages/records/tutorial-api.ts +++ b/admin-frontend-vue/src/pages/records/tutorial-api.ts @@ -10,8 +10,6 @@ export const TUTORIAL_UPLOAD_ENDPOINT = '/api/admin/tutorial' export const TUTORIAL_PRESIGN_ENDPOINT = `${TUTORIAL_UPLOAD_ENDPOINT}/presign` export const TUTORIAL_CONFIRM_ENDPOINT = `${TUTORIAL_UPLOAD_ENDPOINT}/confirm` export const TUTORIAL_DELETE_ENDPOINT = `${TUTORIAL_UPLOAD_ENDPOINT}/delete` -/** 大档分片通道(AC-11):>8MB 的教程包走这里,不再一次性 presign PUT。 */ -export const TUTORIAL_CHUNK_ENDPOINT = '/api/files/chunk' /** MinIO 直传专用实例:不带 cookie、不挂 401 跳登录拦截器(预签名过期/签名失败不能误判会话过期);大 zip 给足超时。 */ const directPut = axios.create({ timeout: 600_000, withCredentials: false }) diff --git a/admin-frontend-vue/src/pages/records/version-api.ts b/admin-frontend-vue/src/pages/records/version-api.ts index 7b1913a43..112350301 100644 --- a/admin-frontend-vue/src/pages/records/version-api.ts +++ b/admin-frontend-vue/src/pages/records/version-api.ts @@ -10,8 +10,6 @@ export const SOFTWARE_VERSION_UPLOAD_ENDPOINT = '/api/admin/version' export const SOFTWARE_VERSION_PRESIGN_ENDPOINT = `${SOFTWARE_VERSION_UPLOAD_ENDPOINT}/presign` export const SOFTWARE_VERSION_CONFIRM_ENDPOINT = `${SOFTWARE_VERSION_UPLOAD_ENDPOINT}/confirm` export const SOFTWARE_VERSION_DELETE_ENDPOINT = `${SOFTWARE_VERSION_UPLOAD_ENDPOINT}/delete` -/** 大档分片通道(AC-11):>8MB 的安装包走这里,不再一次性 presign PUT。 */ -export const SOFTWARE_VERSION_CHUNK_ENDPOINT = '/api/files/chunk' /** MinIO 直传专用实例:不带 cookie、不挂 401 跳登录拦截器(预签名过期/签名失败不能误判会话过期);大 zip 给足超时。 */ const directPut = axios.create({ timeout: 600_000, withCredentials: false }) diff --git a/admin-frontend-vue/src/router/routes.ts b/admin-frontend-vue/src/router/routes.ts index 39d8584ce..d99f94e91 100644 --- a/admin-frontend-vue/src/router/routes.ts +++ b/admin-frontend-vue/src/router/routes.ts @@ -34,6 +34,7 @@ export const adminPages: AdminPageDef[] = [ { path: 'records/tutorial', menuKey: 'admin_tutorial', title: '教程管理', companyScoped: false, load: () => import('@/pages/records/RecordsTutorialPage.vue') }, { path: 'records/workbench-client-package', menuKey: 'admin_workbench_client_package', title: '数富AI工作台客户端包管理', companyScoped: false, load: () => import('@/pages/records/WorkbenchClientPackagePage.vue') }, { path: 'records/device-logs', menuKey: 'admin_device_logs', title: '日志管理', companyScoped: true, load: () => import('@/pages/records/DeviceLogsPage.vue') }, + { path: 'records/task-evidence', menuKey: 'admin_task_evidence', title: '任务证据', superAdminOnly: true, companyScoped: false, load: () => import('@/pages/records/TaskEvidencePage.vue') }, { path: 'task-center/board', menuKey: 'admin_task_center', title: '客户端任务管理', companyScoped: true, load: () => import('@/pages/task-center/TaskCenterBoardPage.vue') }, { path: 'records/digital-human-version', menuKey: 'digital_human_version', title: '数字人版本管理', load: () => import('@/pages/records/RecordsDigitalHumanVersionPage.vue') }, { path: 'records/image-video-tasks', menuKey: 'admin_image_video_tasks', title: '视频任务记录', companyScoped: true, load: () => import('@/pages/tasks/ImageVideoTasksPage.vue') }, diff --git a/admin-frontend-vue/tests/baselines/source-pin-ledger.json b/admin-frontend-vue/tests/baselines/source-pin-ledger.json index 4499c5f96..10466d467 100644 --- a/admin-frontend-vue/tests/baselines/source-pin-ledger.json +++ b/admin-frontend-vue/tests/baselines/source-pin-ledger.json @@ -1,10 +1,10 @@ { "note": "readSource 钉子逐文件基线。两把尺子:pins=落在用例体内的读盘,reads=全文读点总数(含模块级夹具与 helper)。都只许减;pins 下降而 reads 没同步下降=把读盘挪出体内,不算改进。由 tests/source-pin-ratchet.test.ts 生成。", "totals": { - "files": 251, - "cases": 1750, - "pins": 453, - "reads": 646 + "files": 252, + "cases": 1753, + "pins": 456, + "reads": 651 }, "files": { "admin-scaffold.test.ts": { @@ -1247,6 +1247,11 @@ "pins": 3, "reads": 5 }, + "task-evidence-page.test.ts": { + "cases": 3, + "pins": 3, + "reads": 5 + }, "timer-cleanup-pairing.test.ts": { "cases": 5, "pins": 3, diff --git a/admin-frontend-vue/tests/mount/records-upload-error-chinese.mount.test.ts b/admin-frontend-vue/tests/mount/records-upload-error-chinese.mount.test.ts new file mode 100644 index 000000000..272242e6f --- /dev/null +++ b/admin-frontend-vue/tests/mount/records-upload-error-chinese.mount.test.ts @@ -0,0 +1,155 @@ +// P2-10(后管 AC/user-notice「只把中文给用户看」口径)。 +// +// 评审实录:`RecordsSoftwareVersionPage.vue`/`RecordsTutorialPage.vue` 的 catch 直接写 +// `error instanceof Error ? error.message : '上传失败'`,而 `src/api/http.ts` 的响应拦截器 +// **不做 sanitize**(只挑 401/403/4011)⇒ `Broken pipe`、`Request failed with status code 403` +// 这类英文异常原文整段上屏。收口方式沿用本仓既有出口 `components/load-failure-model` +// 的 `loadFailureMessage(error, 本页兜底句)`(去重页四条写路径同一写法)。 +// +// 跑法:npm run test:mount(tests/mount/**,vitest+jsdom)。 +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { flushPromises, mount } from '@vue/test-utils' +import { readFileSync } from 'node:fs' +import { dirname, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' + +import RecordsSoftwareVersionPage from '@/pages/records/RecordsSoftwareVersionPage.vue' +import RecordsTutorialPage from '@/pages/records/RecordsTutorialPage.vue' +import { loadFailureMessage } from '@/components/load-failure-model' + +const here = dirname(fileURLToPath(import.meta.url)) +const root = resolve(here, '../..') + +const versionApi = vi.hoisted(() => ({ + fetchSoftwareVersions: vi.fn(), + uploadSoftwareVersion: vi.fn(), + deleteSoftwareVersions: vi.fn(), +})) +const tutorialApi = vi.hoisted(() => ({ + fetchTutorialPackages: vi.fn(), + uploadTutorialPackage: vi.fn(), + deleteTutorialPackages: vi.fn(), +})) +const confirmMock = vi.hoisted(() => vi.fn()) +const feedbackMock = vi.hoisted(() => vi.fn()) +const messageMock = vi.hoisted(() => ({ success: vi.fn() })) + +vi.mock('@/pages/records/version-api.ts', () => versionApi) +vi.mock('@/pages/records/tutorial-api.ts', () => tutorialApi) +vi.mock('@/components/admin-confirm', () => ({ openAdminConfirm: confirmMock })) +vi.mock('@/components/admin-feedback-ui', () => ({ showAdminFeedback: feedbackMock })) +vi.mock('element-plus', () => ({ ElMessage: messageMock })) + +/** el-upload 的桩:把页面传进来的 on-change 回调顶出来,等价于用户选了一个 .zip。 */ +const uploadStub = { + props: ['onChange'], + template: '', + data: () => ({ fake: new File([new Uint8Array(16)], 'pkg.zip', { type: 'application/zip' }) }), +} + +const stubs = { + 'el-dialog': { template: '
' }, + 'el-form': { template: '
' }, + 'el-form-item': { template: '
' }, + 'el-input': { + props: ['modelValue'], + emits: ['update:modelValue'], + template: '', + }, + 'el-upload': uploadStub, + 'el-button': { template: '' }, + 'el-progress': true, + 'el-alert': { props: ['title'], template: '
{{ title }}
' }, +} + +/** 后端/驱动层原文:一句中文都没有,正是过去直接上屏的那种。 */ +const RAW_ENGLISH = 'Broken pipe: java.io.IOException at sun.nio.ch.FileDispatcherImpl.write0' + +function clickButtonByText(wrapper: ReturnType, text: string, last = false) { + const matches = wrapper.findAll('button').filter((node) => node.text().trim() === text) + const button = last ? matches[matches.length - 1] : matches[0] + if (!button) throw new Error(`找不到文案为「${text}」的按钮`) + return button +} + +async function submitZipUpload(component: typeof RecordsSoftwareVersionPage, versionPlaceholder: string, submitText: string, openText = '上传新版本') { + const wrapper = mount(component, { global: { stubs } }) + await flushPromises() + + clickButtonByText(wrapper, openText) + const versionInput = wrapper.find(`input[placeholder="${versionPlaceholder}"]`) + if (versionInput.exists()) await versionInput.setValue('9.9.9') + await wrapper.get('.pick-zip').trigger('click') + await clickButtonByText(wrapper, submitText, true).trigger('click') + await flushPromises() + return wrapper +} + +describe('P2-10 后管上传/加载失败只把中文交给界面', () => { + beforeEach(() => { + vi.clearAllMocks() + versionApi.fetchSoftwareVersions.mockResolvedValue({ items: [] }) + tutorialApi.fetchTutorialPackages.mockResolvedValue({ items: [] }) + confirmMock.mockResolvedValue(true) + }) + + it('真实出口对照:英文原文一律被换成中文,原因段不拼半截句', () => { + expect(loadFailureMessage(new Error(RAW_ENGLISH), '上传失败')).toBe('上传失败') + expect(loadFailureMessage(new Error('分片会话不存在或已过期'), '上传失败')) + .toBe('上传失败:分片会话不存在或已过期') + }) + + it('安装包上传失败:界面文案不含英文原文,原文只进 console', async () => { + const warned = vi.spyOn(console, 'warn').mockImplementation(() => undefined) + versionApi.uploadSoftwareVersion.mockRejectedValue(new Error(RAW_ENGLISH)) + + const wrapper = await submitZipUpload(RecordsSoftwareVersionPage, '例如:1.0.0', '上传软件版本') + + const shown = wrapper.get('.upload-msg').text() + expect(shown, '英文异常原文仍在上屏').not.toContain('Broken pipe') + expect(shown).toContain('上传失败') + expect( + warned.mock.calls.some((call) => String(call.join(' ')).includes(RAW_ENGLISH)), + '原文要留在 console,否则支持同学对着后端日志无从核对', + ).toBe(true) + warned.mockRestore() + }) + + it('教程包上传失败:同一条中文出口,不得再直落 error.message', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + tutorialApi.uploadTutorialPackage.mockRejectedValue(new Error(RAW_ENGLISH)) + + const wrapper = await submitZipUpload(RecordsTutorialPage, '例如:v2026.09 或留空', '上传教程包', '上传教程包') + + const shown = wrapper.get('.upload-msg').text() + expect(shown, '英文异常原文仍在上屏').not.toContain('Broken pipe') + expect(shown).toContain('上传失败') + }) + + it('列表加载失败:LoadFailureBar 的文案也是中文', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + versionApi.fetchSoftwareVersions.mockRejectedValue(new Error('Internal Server Error')) + + const wrapper = mount(RecordsSoftwareVersionPage, { global: { stubs } }) + await flushPromises() + + const shown = wrapper.text() + expect(shown).not.toContain('Internal Server Error') + expect(shown).toContain('版本列表加载失败') + expect(feedbackMock).toHaveBeenCalled() + }) + + // 写法收口的台账:两页四条 catch(列表/删除×2/上传)都必须把错误交给同一个出口。 + it('两页不得再留 error instanceof Error ? error.message 那套写法', () => { + for (const file of [ + 'src/pages/records/RecordsSoftwareVersionPage.vue', + 'src/pages/records/RecordsTutorialPage.vue', + ]) { + const source = readFileSync(resolve(root, file), 'utf-8') + expect(source, `${file} 仍直落 error.message`).not.toMatch(/error instanceof Error \? error\.message/) + expect(source.match(/loadFailureMessage\(error,/g)?.length ?? 0, `${file} 的失败出口没接上`) + .toBeGreaterThanOrEqual(4) + expect(source).toMatch(/import \{ loadFailureMessage \} from '@\/components\/load-failure-model'/) + } + }) +}) diff --git a/admin-frontend-vue/tests/pending-phase4-大档上传接线.test.ts b/admin-frontend-vue/tests/pending-phase4-大档上传接线.test.ts index 54a9ef23e..86dcdb768 100644 --- a/admin-frontend-vue/tests/pending-phase4-大档上传接线.test.ts +++ b/admin-frontend-vue/tests/pending-phase4-大档上传接线.test.ts @@ -1,8 +1,13 @@ // AC-11(R4):后管版本/教程大档上传接线的「源码尺子」那一半。 // -// Pending Phase 4 激活。今天 `src/pages/records/version-api.ts` 与 `tutorial-api.ts` -// 都是 presign → 一次 PUT 直传(512MB 级也一样,蓝图 §1.2 R4 实读);本文件要求两条 -// 路径接上既有 `/api/files/chunk/**` 分片通道(断点续传/按片重试)。 +// Pending Phase 4 激活。本尺子的前身只断言 `files/chunk` 出现在非注释行,而那一句匹配来自 +// `version-api.ts`/`tutorial-api.ts` 里一对**全仓零引用的死常量**(SOFTWARE_VERSION_CHUNK_ENDPOINT/ +// TUTORIAL_CHUNK_ENDPOINT)——把真调用 `uploadFileInChunks(…)` 删掉,尺子照样绿(评审 P2-6 假绿形状)。 +// 现在判据钉在**真调用点**上:被接线的那个函数体里必须出现 `uploadFileInChunks(`, +// 且死常量不许回来(`CHUNK_ENDPOINT =` 一出现即红)。删调用必判红,加死常量也判红。 +// +// 行为口径在 tests/pure/pending-phase4-大档上传接线.test.ts(vitest 真调 uploadSoftwareVersion/ +// uploadTutorialPackage,删掉调用同一刻那一边也红);本文件只补"接线没被搬空"这一层。 // // 跑法:npm test(node --test tests/*.test.ts)。 import { test } from 'node:test' @@ -14,29 +19,75 @@ import { fileURLToPath } from 'node:url' const here = dirname(fileURLToPath(import.meta.url)) const root = resolve(here, '..') -const LARGE_UPLOAD_FILES = [ - 'src/pages/records/version-api.ts', - 'src/pages/records/tutorial-api.ts', -] +/** 把注释整段涂白(保行数):接线只写在注释里不算数。 */ +function blankComments(source: string): string { + return source + .replace(/\/\*[\s\S]*?\*\//g, (block) => block.replace(/[^\n]/g, ' ')) + .replace(/\/\/[^\n]*/g, (line) => ' '.repeat(line.length)) +} -test('后管版本/教程大档必须接分片通道', () => { - for (const file of LARGE_UPLOAD_FILES) { - const source = readFileSync(resolve(root, file), 'utf-8') - assert.match( - source, - /files\/chunk/, - `AC-11:${file} 仍是 presign 直传(512MB 断网从 0 重来,没有分片续传)`, - ) +/** 取 `function (…) {…}` 的函数体:大括号配对,跳过字符串与模板串里的括号。 */ +function functionBodyOf(source: string, name: string): string { + const code = blankComments(source) + const head = code.indexOf(`function ${name}(`) + assert.notEqual(head, -1, `找不到函数 ${name}()——接线被删了还是改名了?`) + const open = code.indexOf('{', head) + assert.notEqual(open, -1, `${name}() 没有函数体`) + let depth = 0 + let index = open + let quote = '' + for (; index < code.length; index += 1) { + const char = code[index] + if (quote) { + if (char === '\\') index += 1 + else if (char === quote) quote = '' + continue + } + if (char === '"' || char === "'" || char === '`') { quote = char; continue } + if (char === '{') depth += 1 + else if (char === '}') { + depth -= 1 + if (depth === 0) return code.slice(open + 1, index) + } } + return code.slice(open + 1) +} + +/** + * 两格共同判据(读盘点留在用例体内:把 readFileSync 挪进 helper 是本仓 source-pin 棘轮点名的"搬家不算改进"): + * ① 从 @/api/chunk-upload 把分片函数引进来;② 被接线的函数体里真的调它; + * ③ 阈值同源;④ 死常量不得回来。 + */ +function assertChunkedWiring(source: string, file: string, wiredFunction: string): void { + assert.match( + source, + /import\s*\{[^}]*uploadFileInChunks[^}]*\}\s*from\s*'@\/api\/chunk-upload'/, + `AC-11:${file} 没有从 @/api/chunk-upload 引入分片函数`, + ) + const body = functionBodyOf(source, wiredFunction) + assert.match( + body, + /uploadFileInChunks\s*\(/, + `AC-11:${file}::${wiredFunction}() 里没有真调用 uploadFileInChunks(…)(大档仍走一次 PUT 直传,512MB 断网从 0 重来)`, + ) + assert.match( + body, + /CHUNK_UPLOAD_THRESHOLD/, + `AC-11:${wiredFunction}() 的选路阈值必须与 chunk-upload 同源,不得各自写死 8MB`, + ) + assert.doesNotMatch( + source, + /CHUNK_ENDPOINT\s*=/, + `AC-11:${file} 又立了一个没人引用的 CHUNK_ENDPOINT 常量——它会顶掉本尺子的匹配来源,正是 P2-6 的假绿形状`, + ) +} + +test('后管安装包上传必须真的调分片通道', () => { + const file = 'src/pages/records/version-api.ts' + assertChunkedWiring(readFileSync(resolve(root, file), 'utf-8'), file, 'uploadSoftwareVersion') }) -test('分片接线不得只落注释', () => { - for (const file of LARGE_UPLOAD_FILES) { - const source = readFileSync(resolve(root, file), 'utf-8') - const codeLines = source - .split('\n') - .filter((line) => !line.trim().startsWith('//') && !line.trim().startsWith('*')) - .join('\n') - assert.match(codeLines, /files\/chunk/, `AC-11:${file} 的分片引用只出现在注释里`) - } +test('后管教程包上传必须真的调分片通道', () => { + const file = 'src/pages/records/tutorial-api.ts' + assertChunkedWiring(readFileSync(resolve(root, file), 'utf-8'), file, 'uploadTutorialPackage') }) diff --git a/admin-frontend-vue/tests/pure/chunk-upload-resume.test.ts b/admin-frontend-vue/tests/pure/chunk-upload-resume.test.ts new file mode 100644 index 000000000..6eebb3127 --- /dev/null +++ b/admin-frontend-vue/tests/pure/chunk-upload-resume.test.ts @@ -0,0 +1,226 @@ +// P2-7/P2-8/P3-5(后管分片通道 AC-9/AC-11 精神)。 +// +// 评审实录:`src/api/chunk-upload.ts` 每次调用都新 init 一个 upload_id ⇒ 紧随的 GET 必空、 +// 放弃时又把会话 DELETE 掉,**等于根本没有续传**;逐片超时还按整包体积算(120s + MB×15s), +// 8MB 的片能拿到约 130 分钟。本文件按用户端 `shared/api/upload.ts` 的同一口径钉四件事: +// ① 同一文件身份复用 upload_id(不重新 init);② 瞬态失败不 DELETE、会话留在本地供下次续传; +// ③ 只有服务端确实说会话没了(404/「不存在或已过期」)才重开;④ 逐片超时按本片体积给。 +// +// 跑法:npm run test:mount(tests/pure/** 与 tests/mount/** 同走 vitest)。 +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + post: vi.fn(), + get: vi.fn(), + delete: vi.fn(), +})) + +// 只顶掉 http 实例,unwrap 用真的:会话回收在后端是 HTTP 200+success:false, +// 判因逻辑要吃真实解包语义(含中文文案把关),换桩会把这一半形状测漏。 +vi.mock('@/api/http', async (importOriginal) => { + const actual = await importOriginal<{ unwrap: (payload: unknown) => T }>() + return { + ...actual, + http: { post: mocks.post, get: mocks.get, delete: mocks.delete }, + } +}) + +import { + CHUNK_UPLOAD_PART_SIZE, + CHUNK_UPLOAD_THRESHOLD, + chunkSessionKeyOf, + partTimeoutOf, + uploadFileInChunks, +} from '@/api/chunk-upload' + +const RESUMED_ID = 'r'.repeat(32) +const FRESH_ID = 'n'.repeat(32) + +function axiosResponse(status: number, body: unknown) { + return { status, data: body } +} + +function javaOk(body: unknown) { + return axiosResponse(200, { success: true, message: '', data: body }) +} + +function makeFile(name: string, bytes: number): File { + return new File([new Uint8Array(bytes)], name, { type: 'application/octet-stream' }) +} + +function seedSession(file: File, uploadId: string): void { + localStorage.setItem(chunkSessionKeyOf(file.name, file), JSON.stringify({ + uploadId, + size: file.size, + lastModified: file.lastModified, + })) +} + +function storedUploadId(file: File): string { + const raw = localStorage.getItem(chunkSessionKeyOf(file.name, file)) + if (!raw) return '' + return String((JSON.parse(raw) as { uploadId?: unknown }).uploadId ?? '') +} + +function postedUrls(): string[] { + return mocks.post.mock.calls.map(([url]) => String(url)) +} + +function partPosts(): Array<{ url: string; timeout: number }> { + return mocks.post.mock.calls + .filter(([url]) => /\/part\/\d+$/.test(String(url))) + .map(([url, , config]) => ({ + url: String(url), + timeout: Number((config as { timeout?: unknown } | undefined)?.timeout ?? -1), + })) +} + +function initCount(): number { + return postedUrls().filter((url) => url.includes('/chunk/init')).length +} + +/** init/逐片/complete 一律成功;complete 返回拼装后的 fileKey。 */ +function routePosts(uploadId: string) { + mocks.post.mockImplementation((url: string) => Promise.resolve( + String(url).includes('/chunk/init') + ? javaOk({ upload_id: uploadId, total_parts: 2 }) + : String(url).includes('/complete') + ? javaOk({ fileKey: 'tmp/pkg.zip', originalFilename: 'pkg.zip', size: 1 }) + : javaOk({ upload_id: String(url).split('/')[5], received_parts: 1 }), + )) +} + +describe('P2-7 后管分片必须真的续传', () => { + beforeEach(() => { + vi.clearAllMocks() + localStorage.clear() + routePosts(FRESH_ID) + }) + + it('同一文件身份复用 upload_id:探测成功就一条 init 都不许发', async () => { + const file = makeFile('pkg.zip', CHUNK_UPLOAD_PART_SIZE * 2) + seedSession(file, RESUMED_ID) + mocks.get.mockResolvedValue(javaOk({ upload_id: RESUMED_ID, uploaded_parts: [0] })) + + await uploadFileInChunks(file, file.name) + + expect(initCount(), '会话还在服务端,重开 init 就是作废已传片').toBe(0) + expect( + partPosts().map((part) => part.url), + '第 0 片服务端已有,只补第 1 片', + ).toEqual([`/api/files/chunk/${RESUMED_ID}/part/1`]) + }) + + it('瞬态失败(分片打不出去):不 DELETE 会话,且本地仍留着 upload_id', async () => { + const file = makeFile('pkg.zip', CHUNK_UPLOAD_PART_SIZE * 2) + seedSession(file, RESUMED_ID) + mocks.get.mockResolvedValue(javaOk({ upload_id: RESUMED_ID, uploaded_parts: [] })) + mocks.post.mockRejectedValue(new Error('Network Error')) + + await expect(uploadFileInChunks(file, file.name)).rejects.toThrow(/Network Error/) + + expect(mocks.delete.mock.calls, 'DELETE 会把已达片号一起清掉,下次只能从 0').length(0) + expect(storedUploadId(file), '会话要留给下一次(含刷新后)续传').toBe(RESUMED_ID) + }) + + it('探测 503:瞬态 ⇒ 保留原会话,不重开', async () => { + const file = makeFile('pkg.zip', CHUNK_UPLOAD_PART_SIZE * 2) + seedSession(file, RESUMED_ID) + mocks.get.mockResolvedValue(axiosResponse(503, 'Bad Gateway')) + + await uploadFileInChunks(file, file.name) + + expect(initCount()).toBe(0) + expect(partPosts().every((part) => part.url.includes(RESUMED_ID))).toBe(true) + }) + + it('探测超时(网络层错误):瞬态 ⇒ 同样保留原会话', async () => { + const file = makeFile('pkg.zip', CHUNK_UPLOAD_PART_SIZE * 2) + seedSession(file, RESUMED_ID) + mocks.get.mockRejectedValue(new Error('timeout of 30000ms exceeded')) + + await uploadFileInChunks(file, file.name) + + expect(initCount()).toBe(0) + expect(partPosts().every((part) => part.url.includes(RESUMED_ID))).toBe(true) + }) + + it('HTTP 404:会话确实没了 ⇒ 清状态重开', async () => { + const file = makeFile('pkg.zip', CHUNK_UPLOAD_PART_SIZE * 2) + seedSession(file, RESUMED_ID) + mocks.get.mockResolvedValue(axiosResponse(404, { success: false, message: 'Not Found', data: null })) + + await uploadFileInChunks(file, file.name) + + expect(initCount(), '会话已回收,不重建就永远卡在同一个死会话上').toBe(1) + expect(partPosts().every((part) => part.url.includes(FRESH_ID))).toBe(true) + }) + + it('后端 HTTP 200+「分片会话不存在或已过期」(现网形状)⇒ 同样重开', async () => { + const file = makeFile('pkg.zip', CHUNK_UPLOAD_PART_SIZE * 2) + seedSession(file, RESUMED_ID) + mocks.get.mockResolvedValue(axiosResponse( + 200, + { success: false, message: '分片会话不存在或已过期', data: null }, + )) + + await uploadFileInChunks(file, file.name) + + expect(initCount()).toBe(1) + expect(partPosts().every((part) => part.url.includes(FRESH_ID))).toBe(true) + }) + + it('拼装成功 ⇒ 本地会话清掉(同一文件再传要开新会话)', async () => { + const file = makeFile('pkg.zip', CHUNK_UPLOAD_PART_SIZE * 2) + seedSession(file, RESUMED_ID) + mocks.get.mockResolvedValue(javaOk({ upload_id: RESUMED_ID, uploaded_parts: [] })) + + await uploadFileInChunks(file, file.name) + + expect(storedUploadId(file), '完成后会话已拼装落库,本地不该留残号').toBe('') + }) +}) + +describe('P2-8 逐片超时按本片体积', () => { + it('每片预算只看这一片有多大,不得按整包体积放大', async () => { + const file = makeFile('big.zip', CHUNK_UPLOAD_PART_SIZE * 5) + mocks.get.mockResolvedValue(javaOk({ upload_id: FRESH_ID, uploaded_parts: [] })) + routePosts(FRESH_ID) + + await uploadFileInChunks(file, file.name) + + const parts = partPosts() + expect(parts.length, '40MB=5 片').toBe(5) + // 整包口径会给出 120s + 40×15s = 720s;按片口径 8MB 片=300s。判红要留出余量差。 + for (const part of parts) { + expect(part.timeout, `片 ${part.url} 的超时没有按本片体积算`).toBeGreaterThan(0) + expect( + part.timeout, + `片 ${part.url} 的超时 ${part.timeout}ms 是按整包体积算的(P2-8)`, + ).toBeLessThanOrEqual(partTimeoutOf(CHUNK_UPLOAD_PART_SIZE)) + } + expect(partTimeoutOf(CHUNK_UPLOAD_PART_SIZE), '8MB 片的预算不该超过 10 分钟') + .toBeLessThanOrEqual(600_000) + expect(partTimeoutOf(1), '再小的片也要有保底,不能给 0').toBeGreaterThanOrEqual(60_000) + }) + + it('init 与 complete 的预算不再随整包体积线性放大', async () => { + const file = makeFile('big.zip', CHUNK_UPLOAD_PART_SIZE * 5) + mocks.get.mockResolvedValue(javaOk({ upload_id: FRESH_ID, uploaded_parts: [] })) + routePosts(FRESH_ID) + + await uploadFileInChunks(file, file.name) + + const initCall = mocks.post.mock.calls.find(([url]) => String(url).includes('/chunk/init')) + const initTimeout = Number((initCall?.[2] as { timeout?: unknown } | undefined)?.timeout ?? -1) + expect(initTimeout, 'init 只发一条登记请求,按整包体积给 720s 是把失败拖到天荒地老').toBeLessThan(300_000) + }) +}) + +describe('P3-5 阈值单点', () => { + it('后管这一侧的 8MB 只有一个出口:阈值由片大小派生', () => { + expect(CHUNK_UPLOAD_THRESHOLD, '阈值必须与片大小同源(P3-5 三处各写一遍的收口)') + .toBe(CHUNK_UPLOAD_PART_SIZE) + expect(CHUNK_UPLOAD_PART_SIZE).toBe(8 * 1024 * 1024) + }) +}) diff --git a/admin-frontend-vue/tests/pure/pending-phase4-大档上传接线.test.ts b/admin-frontend-vue/tests/pure/pending-phase4-大档上传接线.test.ts index ed77504b1..13fe04079 100644 --- a/admin-frontend-vue/tests/pure/pending-phase4-大档上传接线.test.ts +++ b/admin-frontend-vue/tests/pure/pending-phase4-大档上传接线.test.ts @@ -3,7 +3,7 @@ // Pending Phase 4 激活。9MB 安装包(跨分片阈值)调 uploadSoftwareVersion, // 必须出现 POST /files/chunk/init;今天它走 presign → 一次 PUT 直传, // 512MB 断网只能从 0 重来。本文件不出网,mock 掉 http 与 axios 直传实例。 -import { describe, it, expect, vi } from 'vitest' +import { beforeEach, describe, it, expect, vi } from 'vitest' const mocks = vi.hoisted(() => ({ post: vi.fn(), @@ -34,9 +34,16 @@ vi.mock('@/api/http', () => ({ })) import { uploadSoftwareVersion } from '@/pages/records/version-api.ts' +import { uploadTutorialPackage } from '@/pages/records/tutorial-api.ts' describe('AC-11 后管大档上传', () => { + // 会话跨尝试复用后,两格之间必须清态:残留的 upload_id 会让第二格跳过本该发生的 init。 + beforeEach(() => { + vi.clearAllMocks() + localStorage.clear() + }) + it('9MB 安装包必须走分片 init', async () => { // 预签名与分片两套响应揉成一个超集:哪条实现路径先走都能拿到自己要的字段。 mocks.post.mockResolvedValue({ @@ -68,5 +75,48 @@ describe('AC-11 后管大档上传', () => { urls.some((url) => url.includes('/files/chunk/init')), `AC-11:9MB 安装包仍走 presign 直传,实际出口:${urls.join(', ')}`, ).toBe(true) + // P2-6:删掉 uploadFileInChunks(…) 真调用后,这一格必须跟着红——所以还要核对分片片确实发出去了。 + expect( + mocks.post.mock.calls.filter(([url]) => /\/files\/chunk\/[^/]+\/part\/\d+$/.test(String(url))).length, + 'AC-11:走分片就要真把片 POST 出去', + ).toBe(2) + }) + + // P2-6:教程包那条道过去只有源码尺子覆盖(而尺子当时吃的是死常量),删掉真调用没人判红。 + it('9MB 教程包必须走分片 init', async () => { + localStorage.clear() + mocks.post.mockResolvedValue({ + data: { + success: true, + data: { + upload_url: 'https://oss.example/put', + file_url: 'https://oss.example/tutorial.zip', + version: '9.9.9', + object_key: 'client/tutorials/tutorial.zip', + upload_id: 'b'.repeat(32), + uploaded_parts: [], + item: null, + }, + }, + }) + mocks.get.mockResolvedValue({ + data: { success: true, data: { upload_id: 'b'.repeat(32), uploaded_parts: [] } }, + }) + mocks.directPut.mockResolvedValue({}) + + const file = new Blob([new Uint8Array(9 * 1024 * 1024)], { + type: 'application/octet-stream', + }) + await uploadTutorialPackage(file, 'tutorial.zip', '9.9.9') + + const urls = mocks.post.mock.calls.map(([url]) => String(url)) + expect( + urls.some((url) => url.includes('/files/chunk/init')), + `AC-11:9MB 教程包仍走 presign 直传,实际出口:${urls.join(', ')}`, + ).toBe(true) + expect( + mocks.directPut.mock.calls.length, + 'AC-11:大档不该再走一次 PUT 直传', + ).toBe(0) }) }) diff --git a/admin-frontend-vue/tests/task-evidence-page.test.ts b/admin-frontend-vue/tests/task-evidence-page.test.ts new file mode 100644 index 000000000..d558c8736 --- /dev/null +++ b/admin-frontend-vue/tests/task-evidence-page.test.ts @@ -0,0 +1,32 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { readSource } from './helpers.ts' + +/** P1-9:任务证据读取面的后台入口必须存在,且只给超级管理员。 */ + +test('task evidence menu is super-admin only', () => { + const routes = readSource('src/router/routes.ts') + assert.match(routes, /path: 'records\/task-evidence'[\s\S]*?menuKey: 'admin_task_evidence'[\s\S]*?superAdminOnly: true/, + '任务证据路由必须带 superAdminOnly') + + const scope = readSource('src/config/company-scope.ts') + assert.match(scope, /SUPER_ADMIN_ONLY_MENU_KEYS[^\n]*admin_task_evidence/, + '菜单 key 必须在超级管理员可见清单里') + + const guide = readSource('src/layout/operation-guides.ts') + assert.match(guide, /admin_task_evidence:\s*\{/, '菜单必须配操作指引') +}) + +test('task evidence page never renders object keys', () => { + const page = readSource('src/pages/records/TaskEvidencePage.vue') + assert.doesNotMatch(page, /objectKey|object_key|presign/i, '页面不得展示或持有对象直链') + assert.match(page, /taskEvidenceDownloadUrl/, '下载必须走鉴权接口') + assert.match(page, /deleteTaskEvidence/, '删除必须走鉴权接口') +}) + +test('task evidence api is wired to admin endpoints', () => { + const api = readSource('src/api/task-evidence.ts') + assert.match(api, /\/admin\/task-evidence\/files/) + assert.match(api, /\/admin\/task-evidence\/download\?fileId=/) + assert.match(api, /\/admin\/task-evidence\/delete/) +}) diff --git a/backend-java/src/main/java/com/nanri/aiimage/common/security/CredentialPackageGuard.java b/backend-java/src/main/java/com/nanri/aiimage/common/security/CredentialPackageGuard.java new file mode 100644 index 000000000..a34e2f274 --- /dev/null +++ b/backend-java/src/main/java/com/nanri/aiimage/common/security/CredentialPackageGuard.java @@ -0,0 +1,86 @@ +package com.nanri.aiimage.common.security; + +import java.io.File; +import java.io.IOException; +import java.util.Enumeration; +import java.util.List; +import java.util.Locale; +import java.util.Set; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +/** + * 安装包/教程包凭据件扫描(P2-4,对齐工作台包的 CR-02 口径)。 + * + *

版本包与教程包此前只校验扩展名和体积,压缩包里是否带 {@code .env}/{@code *.pem} + * 没人看;而它们会经公开 client 桶发给全部客户端。这里只读 zip 中央目录、按条目名判, + * 不解压内容,代价与包体积无关。

+ */ +public final class CredentialPackageGuard { + + private static final Set FORBIDDEN_NAMES = Set.of( + "kw_share_cfg.json", ".env", ".netrc", ".git-credentials", + "id_rsa", "id_dsa", "id_ecdsa", "id_ed25519"); + + private static final List FORBIDDEN_SUFFIXES = + List.of(".pem", ".key", ".p12", ".pfx", ".jks", ".keystore"); + + private static final int MAX_ENTRIES = 200_000; + + private CredentialPackageGuard() { + } + + /** + * 返回第一个命中凭据名的条目;没有命中返回 {@code null}。 + * + * @throws IOException 文件不是可读 zip / 条目数超限 + */ + public static String firstForbiddenEntry(File packageFile) throws IOException { + if (packageFile == null || !packageFile.isFile()) { + throw new IOException("package file missing"); + } + try (ZipFile zipFile = new ZipFile(packageFile)) { + Enumeration entries = zipFile.entries(); + int scanned = 0; + while (entries.hasMoreElements()) { + ZipEntry entry = entries.nextElement(); + if (++scanned > MAX_ENTRIES) { + throw new IOException("zip entry limit exceeded"); + } + if (entry.isDirectory()) { + continue; + } + String base = baseName(entry.getName()); + if (isCredentialFileName(base)) { + return entry.getName(); + } + } + } + return null; + } + + static boolean isCredentialFileName(String fileName) { + String lower = fileName == null ? "" : fileName.trim().toLowerCase(Locale.ROOT); + if (lower.isEmpty()) { + return false; + } + if (FORBIDDEN_NAMES.contains(lower) || lower.startsWith(".env.")) { + return true; + } + for (String suffix : FORBIDDEN_SUFFIXES) { + if (lower.endsWith(suffix)) { + return true; + } + } + return false; + } + + private static String baseName(String entryName) { + if (entryName == null) { + return ""; + } + String normalized = entryName.replace('\\', '/'); + int slash = normalized.lastIndexOf('/'); + return slash < 0 ? normalized : normalized.substring(slash + 1); + } +} diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/controller/AppearancePatentController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/controller/AppearancePatentController.java index 3c4575592..b7ab2352f 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/controller/AppearancePatentController.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/controller/AppearancePatentController.java @@ -59,7 +59,7 @@ public class AppearancePatentController { } @GetMapping("/tasks/{taskId}/queue-payload") - @Operation(summary = "获取外观专利 Python 队列载荷", description = "本地采集模式下,前端取出 groups 后推本机客户端队列;只返回 Python 当前需要的分组,避免前端搬运完整解析大数据。") + @Operation(summary = "获取外观专利 Python 队列载荷", description = "本地采集模式下,前端取出 groups 后推本机客户端队列;只返回 Python 当前需要的分组,避免前端搬运完整解析大数据。这一单登记为线上采集时拒发(V183 通道),防旧页面缓存推本机与后台双跑。") public ApiResponse queuePayload( @Parameter(description = "外观专利任务 ID", required = true, example = "7004") @PathVariable Long taskId, @@ -67,7 +67,7 @@ public class AppearancePatentController { @RequestParam("user_id") Long userId, HttpServletRequest httpRequest) { userId = adminAuthSupport.resolveOwnershipUserId(httpRequest, userId); - return ApiResponse.success(service.queuePayload(taskId, userId)); + return ApiResponse.success(service.queuePayloadForLocalClient(taskId, userId)); } @GetMapping("/dashboard") diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java index c767c00c2..b9a1d2bce 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentTaskService.java @@ -114,13 +114,6 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipeline与客户端兜底拉取 / Linux 内部认领共用同一条原子翻转,谁先翻谁执行, * 页面推送与后台接管不会重复跑同一份任务。 */ - private void activateForLocalClient(Long taskId, Long userId) { - // 模块归属走底座 requireSubmissionTask(TaskFinalizeSupport 家族),不在本类手抄 - // 「!MODULE_TYPE.equals(...)」——那形已被 TaskLifecycleDebtRatchetTest 钉成只降不升。 - FileTaskEntity task = requireSubmissionTask(taskId); + private void activateForLocalClient(FileTaskEntity task, Long userId) { + // 模块归属已在 activateTask 的首读校验过;这里复用同一行,不再二次 selectById。 + Long taskId = task.getId(); if (userId == null || !java.util.Objects.equals(userId, task.getUserId())) { throw new com.nanri.aiimage.common.exception.BusinessException("任务不存在"); } @@ -793,6 +785,11 @@ public class AppearancePatentTaskService extends AbstractFileTaskPipelineP1-1 的同族洞:取载荷这一侧原先只看存在与归属、不看通道,选了线上的单被旧页面缓存 + * 或直连 API 取走 groups 推本机,就与 Linux 那边同时跑同一组(按次付费的出口 IP 买两遍)。 + * 兜底拉取/内部认领走的是 {@link #queuePayload(Long, Long)}:本机单等满兜底窗口后由 + * Linux 接管,那一侧必须仍能拿到载荷。 + */ + public AppearancePatentParsedPayloadDto queuePayloadForLocalClient(Long taskId, Long userId) { + return queuePayload(taskId, userId, true); + } + + private AppearancePatentParsedPayloadDto queuePayload(Long taskId, Long userId, boolean requireLocalChannel) { FileTaskEntity task = fileTaskMapper.selectById(taskId); if (task == null || !moduleType.equals(task.getModuleType()) || !Objects.equals(userId, task.getUserId())) { throw new BusinessException("任务不存在"); } ownershipChecker.accept(task, "queue payload"); + if (requireLocalChannel + && !CollectChannel.resolve(task.getCollectChannel(), task.getModuleType()).isLocal()) { + throw new BusinessException("本单已选择线上采集,无需打开客户端,提交后稍候自动开始"); + } AppearancePatentParsedPayloadDto payload = readParsedPayload(task); AppearancePatentParsedPayloadDto queuePayload = new AppearancePatentParsedPayloadDto(); queuePayload.setAiPrompt(payload.getAiPrompt()); diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/support/AppearancePatentTaskRecordSupport.java b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/support/AppearancePatentTaskRecordSupport.java index 811f8ad89..a49b4fd9d 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/support/AppearancePatentTaskRecordSupport.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/appearancepatent/service/support/AppearancePatentTaskRecordSupport.java @@ -70,18 +70,20 @@ public class AppearancePatentTaskRecordSupport { } public void activateTask(Long taskId, Long userId) { - FileTaskEntity task = fileTaskMapper.selectById(taskId); + activateTask(fileTaskMapper.selectById(taskId), userId); + } + + /** P3-4:调用方已经读过任务行时复用,不在 activate 链路上重复 selectById。 */ + public void activateTask(FileTaskEntity task, Long userId) { + Long taskId = task == null ? null : task.getId(); if (task == null || !moduleType.equals(task.getModuleType()) || !Objects.equals(userId, task.getUserId())) { throw new BusinessException("任务不存在"); } - // 2026-09-30 起外观专利已整体迁移 Linux 后台执行(无浏览器抓取): - // activate 不再把任务推给客户端浏览器。任务保持 PENDING,由内部认领端点 - // (/api/internal/tasks/pull-pending + maixiang-ap worker)在等待窗口后自动接管。 - // 这里保留接口仅做状态校验:旧前端(未刷新页面缓存)点「启动」时给出明确提示, - // 前端收到业务异常会中止入队逻辑,任务不会被执行两次。 + // V183 起执行方按单判定:本机单由客户端领取,线上单等内部认领(Linux/外机 Windows)。 + // 这条兼容分支只服务未刷新页面缓存的旧前端:选中线上采集时,告诉用户无需打开浏览器。 if (STATUS_PENDING.equals(task.getStatus())) { - log.info("[外观专利] activate 不再推送客户端,任务保持 PENDING 等待 Linux 后台认领 taskId={} userId={}", taskId, userId); - throw new BusinessException("外观专利已切换后台自动执行,无需打开浏览器,稍候自动开始"); + log.info("[外观专利] activate 不再推送客户端,任务保持 PENDING 等待线上认领 taskId={} userId={}", taskId, userId); + throw new BusinessException("本单已选择线上采集,无需打开客户端,提交后稍候自动开始"); } if (STATUS_RUNNING.equals(task.getStatus())) { throw new BusinessException("任务已在执行中(可能已由后台自动接管),无需重复启动"); diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/evidence/controller/AdminTaskEvidenceController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/evidence/controller/AdminTaskEvidenceController.java index 682c58ec0..4502e97d0 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/evidence/controller/AdminTaskEvidenceController.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/evidence/controller/AdminTaskEvidenceController.java @@ -78,7 +78,6 @@ public class AdminTaskEvidenceController { @Parameter(description = "证据主键") @RequestParam Long fileId) throws IOException { requireSuperAdmin(request); TaskEvidenceEntity entity = taskEvidenceService.requireEvidence(fileId); - byte[] bytes = taskEvidenceService.download(fileId); String name = "evidence-" + entity.getId() + (entity.getObjectKey() != null && entity.getObjectKey().contains(".") ? entity.getObjectKey().substring(entity.getObjectKey().lastIndexOf('.')) : ".bin"); @@ -86,7 +85,7 @@ public class AdminTaskEvidenceController { ? "application/octet-stream" : entity.getContentType()); response.setHeader("Content-Disposition", "attachment; filename*=UTF-8''" + URLEncoder.encode(name, StandardCharsets.UTF_8).replace("+", "%20")); - response.getOutputStream().write(bytes); + taskEvidenceService.downloadTo(fileId, response.getOutputStream()); } @PostMapping("/delete") diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/evidence/controller/InternalTaskEvidenceController.java b/backend-java/src/main/java/com/nanri/aiimage/modules/evidence/controller/InternalTaskEvidenceController.java index d89931c3b..b7a0f4047 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/evidence/controller/InternalTaskEvidenceController.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/evidence/controller/InternalTaskEvidenceController.java @@ -47,9 +47,13 @@ public class InternalTaskEvidenceController { if (body == null) { throw new BusinessException("登记载荷为空"); } + // P1-2:归属以会话身份为准。内部令牌是发给每个登录客户的共享凭据,body.uid 谁都能写—— + // 真身份只能来自请求头里的用户 JWT(4.0.14 起 user_token 钩子统一注入);会话 uid 与 + // body.uid 不一致直接 403,无身份的存量装机保留原参数并打节流的匿名兜底日志。 + Long uid = adminAuthSupport.resolveOwnershipUserId(request, body.getUid()); Map result = taskEvidenceService.register( - body.getTaskId(), body.getUid(), body.getEvidenceType(), body.getSource(), body.getFileKey()); - log.info("[task-evidence] registered id={} taskId={}", result.get("id"), body.getTaskId()); + body.getTaskId(), uid, body.getEvidenceType(), body.getSource(), body.getFileKey()); + log.info("[task-evidence] registered id={} taskId={} uid={}", result.get("id"), body.getTaskId(), uid); return ApiResponse.success(result); } } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/evidence/service/TaskEvidenceService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/evidence/service/TaskEvidenceService.java index 4881666f1..5a4b1cb52 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/evidence/service/TaskEvidenceService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/evidence/service/TaskEvidenceService.java @@ -3,11 +3,15 @@ package com.nanri.aiimage.modules.evidence.service; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.common.service.DistributedJobLockService; +import com.nanri.aiimage.common.task.LockSkipWitness; import com.nanri.aiimage.config.EvidenceProperties; import com.nanri.aiimage.modules.evidence.mapper.TaskEvidenceMapper; import com.nanri.aiimage.modules.evidence.model.entity.TaskEvidenceEntity; import com.nanri.aiimage.modules.evidence.storage.TaskEvidenceStorageService; import com.nanri.aiimage.modules.file.service.LocalFileStorageService; +import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.scheduling.annotation.Scheduled; @@ -15,11 +19,15 @@ import org.springframework.stereotype.Service; import java.io.File; import java.io.IOException; +import java.io.OutputStream; import java.nio.file.Files; +import java.time.Duration; import java.time.LocalDateTime; import java.util.LinkedHashMap; import java.util.List; +import java.util.Locale; import java.util.Map; +import java.util.Set; /** * 证据登记与读取(AC-7/AC-13)。 @@ -35,11 +43,23 @@ public class TaskEvidenceService { private static final int MAX_TASK_ID_LENGTH = 64; private static final int MAX_CONTENT_BYTES = 8 * 1024 * 1024; private static final int RETENTION_BATCH = 200; + /** 单轮保留期最多扫的批次数:防一个删不掉的对象把进程拖成永动机。 */ + private static final int MAX_RETENTION_BATCHES = 20; + /** 同一任务的证据件数上限:登记端点不设防的话,一个持证客户端能刷满整个证据桶(P2-2)。 */ + private static final int MAX_EVIDENCE_PER_TASK = 50; + /** 保留期回收的分布式锁名:A/B 两台每晚各扫同一张表会互相删(同族口径见 DeviceLogRetentionService)。 */ + static final String RETENTION_LOCK = "task-evidence:retention"; + private static final Set ALLOWED_EVIDENCE_TYPES = + Set.of("client", "crash", "window_image", "page_image", "brand", "other"); + private static final Set ALLOWED_SOURCES = + Set.of("client", "crash_recorder", "window", "page", "brand", "other"); private final TaskEvidenceMapper mapper; private final TaskEvidenceStorageService storage; private final LocalFileStorageService localFileStorageService; private final EvidenceProperties properties; + private final FileTaskMapper fileTaskMapper; + private final DistributedJobLockService distributedJobLockService; /** 客户端上传完成后登记:fileKey 指向 Java 临时目录里的分片拼装件。 */ public Map register(String taskId, Long uid, String evidenceType, @@ -52,6 +72,15 @@ public class TaskEvidenceService { if (file == null || !file.isFile()) { throw new BusinessException("证据文件不存在或已过期"); } + // P1-2:归属先于落桶。task_id 能对上库里的任务行时,uid 必须就是行主——判据发生在 + // storage.store 之前,否则每一次拒绝都往证据桶里丢一个收不回的孤儿对象。 + assertTaskOwnedBy(normalizedTaskId, uid); + if (file.length() == 0) { + throw new BusinessException("证据件为空,拒绝登记"); + } + if (countByTask(normalizedTaskId) >= MAX_EVIDENCE_PER_TASK) { + throw new BusinessException("该任务的证据件数已达上限"); + } TaskEvidenceStorageService.StoredEvidence stored; try { stored = storage.store(file, normalizedTaskId, file.getName()); @@ -63,8 +92,8 @@ public class TaskEvidenceService { TaskEvidenceEntity entity = new TaskEvidenceEntity(); entity.setTaskId(normalizedTaskId); entity.setUid(uid); - entity.setEvidenceType(normalizeLabel(evidenceType, "client")); - entity.setSource(normalizeLabel(source, "client")); + entity.setEvidenceType(normalizeLabel(evidenceType, "client", ALLOWED_EVIDENCE_TYPES)); + entity.setSource(normalizeLabel(source, "client", ALLOWED_SOURCES)); entity.setObjectKey(stored.objectKey()); entity.setByteSize(stored.byteSize()); entity.setContentType(stored.contentType()); @@ -143,6 +172,12 @@ public class TaskEvidenceService { return storage.read(entity.getObjectKey(), properties.maxFileBytesOrDefault()); } + /** P3-1:后管下载走流式,避免 64MB 级对象整块进堆。 */ + public void downloadTo(Long fileId, OutputStream out) { + TaskEvidenceEntity entity = requireEvidence(fileId); + storage.downloadTo(entity.getObjectKey(), out, properties.maxFileBytesOrDefault()); + } + /** 删除证据:先删对象再删登记行;对象删除失败不吞(留在库里下一轮可重试)。 */ public int delete(List ids) { if (ids == null || ids.isEmpty()) { @@ -160,35 +195,104 @@ public class TaskEvidenceService { return deleted; } - /** 保留期回收:按天清扫到期行与对象(AC-13 的保留期必须有执行者,不能只写在配置里)。 */ + /** + * 保留期回收:按天清扫到期行与对象(AC-13 的保留期必须有执行者,不能只写在配置里)。 + * + *

P1-7 的两条修正:① A/B 两台每晚并发扫同一张表,回收必须拿分布式锁(同族口径见 + * {@code DeviceLogRetentionService},锁跳过走可聚合出口);② 对象删除失败不再让整轮早退—— + * 原先翻页永远停在第 1 页,卡住的那一行把后面每一行都堵死,配置里的 retention-days + * 因此只是个数字。改成按 id 键集分页:失败行留在库里下一轮重试,后面的行照常推进。 + */ @Scheduled(cron = "${aiimage.evidence.retention-cron:0 30 3 * * *}") public void purgeExpired() { - LocalDateTime expireBefore = LocalDateTime.now().minusDays(properties.retentionDaysOrDefault()); - while (true) { - Page expiredPage = mapper.selectPage( - new Page<>(1, RETENTION_BATCH), - new LambdaQueryWrapper() - .lt(TaskEvidenceEntity::getCreatedAt, expireBefore) - .orderByAsc(TaskEvidenceEntity::getId)); - List batch = expiredPage.getRecords(); + DistributedJobLockService.LockHandle lockHandle = + distributedJobLockService.tryLock(RETENTION_LOCK, Duration.ofMinutes(15)); + if (lockHandle == null) { + // 点位字面量必须写死(LockSkipSilentExitGuardrailTest 的判据),与 RETENTION_LOCK 同值。 + LockSkipWitness.skipped("task-evidence:retention"); + log.info("[task-evidence] 保留期清理跳过:另一实例持锁"); + return; + } + try (lockHandle) { + purgeExpiredLocked(LocalDateTime.now().minusDays(properties.retentionDaysOrDefault())); + } catch (Exception ex) { + log.warn("[task-evidence] 保留期清理失败 reason={}", ex.getMessage(), ex); + } + } + + private void purgeExpiredLocked(LocalDateTime expireBefore) { + long afterId = 0L; + int removed = 0; + int failed = 0; + int batches = 0; + while (batches < MAX_RETENTION_BATCHES) { + List batch = mapper.selectList(new LambdaQueryWrapper() + .gt(TaskEvidenceEntity::getId, afterId) + .lt(TaskEvidenceEntity::getCreatedAt, expireBefore) + .orderByAsc(TaskEvidenceEntity::getId) + .last("limit " + RETENTION_BATCH)); if (batch.isEmpty()) { - return; + break; } - int removed = 0; + batches++; for (TaskEvidenceEntity entity : batch) { + afterId = entity.getId(); try { storage.delete(entity.getObjectKey()); removed += mapper.deleteById(entity.getId()); } catch (RuntimeException ex) { + // 只跳过这一行:登记行留着,下一轮按同一条键集再试;后面的行绝不陪葬。 + failed++; log.warn("[task-evidence] retention delete failed id={} reason={}", entity.getId(), ex.getMessage()); } } - if (removed == 0) { - return; - } - log.info("[task-evidence] retention purge removed={} before={}", removed, expireBefore); } + if (failed > 0) { + log.warn("[task-evidence] retention purge finished with failures removed={} failed={} batches={}", + removed, failed, batches); + } else if (removed > 0 || batches > 0) { + log.info("[task-evidence] retention purge removed={} batches={}", removed, batches); + } + } + + /** + * P1-2 的归属判定:task_id 对得上库里的任务行时 uid 必须就是行主;对不上行(客户端本地 + * 任务号,如商品采集的自有作业 id)只要求带 uid 自证——否则任何一个持内部令牌的客户端 + * 都能往别人的取证面里挂证据。 + */ + private void assertTaskOwnedBy(String taskId, Long uid) { + Long ownerId = resolveTaskOwnerId(taskId); + if (uid == null) { + throw new BusinessException(403, "证据登记必须携带任务归属用户"); + } + if (ownerId != null && !ownerId.equals(uid)) { + throw new BusinessException(403, "证据件只能登记到本人任务上"); + } + } + + /** 数字串按主键查(超长数字串会溢出,降级走 task_no),其余按 task_no 点查。 */ + private Long resolveTaskOwnerId(String taskId) { + FileTaskEntity row = null; + if (taskId.chars().allMatch(Character::isDigit)) { + try { + row = fileTaskMapper.selectById(Long.parseLong(taskId)); + } catch (NumberFormatException overflow) { + row = null; + } + } + if (row == null) { + row = fileTaskMapper.selectOne(new LambdaQueryWrapper() + .eq(FileTaskEntity::getTaskNo, taskId) + .last("limit 1")); + } + return row == null ? null : row.getUserId(); + } + + private long countByTask(String taskId) { + Long count = mapper.selectCount(new LambdaQueryWrapper() + .eq(TaskEvidenceEntity::getTaskId, taskId)); + return count == null ? 0L : count; } private String normalizeTaskId(String taskId) { @@ -202,13 +306,15 @@ public class TaskEvidenceService { return value; } - private String normalizeLabel(String value, String fallback) { - String text = value == null ? "" : value.trim(); + private String normalizeLabel(String value, String fallback, Set allowed) { + String text = value == null ? "" : value.trim().toLowerCase(Locale.ROOT); if (text.isEmpty()) { return fallback; } - // 列宽是 VARCHAR(32),超长标签在这里截断而不是让 insert 抛数据截断错。 - return text.length() > 32 ? text.substring(0, 32) : text; + // 列宽是 VARCHAR(32),超长标签在这里截断而不是让 insert 抛数据截断错; + // 不在白名单内的标签一律落 other,不让客户端自造类型污染取证面。 + String normalized = text.length() > 32 ? text.substring(0, 32) : text; + return allowed.contains(normalized) ? normalized : "other"; } private void deleteTempQuietly(File file) { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/evidence/storage/TaskEvidenceStorageService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/evidence/storage/TaskEvidenceStorageService.java index 829d991b1..a22042aa2 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/evidence/storage/TaskEvidenceStorageService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/evidence/storage/TaskEvidenceStorageService.java @@ -51,6 +51,11 @@ public class TaskEvidenceStorageService { properties.bucketOrDefault(), objectKey, maxBytes); } + /** P3-1:下载走流式,不把整个证据件读进堆。 */ + public void downloadTo(String objectKey, java.io.OutputStream out, long maxBytes) { + ossStorageService.readObjectToStream(properties.bucketOrDefault(), objectKey, out, maxBytes); + } + public void delete(String objectKey) { if (objectKey == null || objectKey.isBlank()) { return; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/ChunkedUploadService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/ChunkedUploadService.java index bf4bdbe7f..16536c6cc 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/ChunkedUploadService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/ChunkedUploadService.java @@ -51,13 +51,13 @@ public class ChunkedUploadService { private static final ObjectMapper JSON = new ObjectMapper(); private static final String META_NAME = "meta.json"; - /** 单个分片上限:比前端 16MB 分片留足余量,防一次请求把临时盘打满。 */ + /** 单个分片上限:两端与客户端实际片大小均为 8MB(各自独立构建根,无共享常量),这里留 8 倍余量。 */ private static final long MAX_PART_BYTES = 64L * 1024 * 1024; private final StorageProperties storageProperties; - /** 同一会话的落片/拼装/放弃串行化;会话结束即回收,避免长期驻留。 */ - private final ConcurrentHashMap sessionLocks = new ConcurrentHashMap<>(); + /** 同一会话的落片/拼装/放弃串行化;static 让统一清理(LocalTempCleanupService)拿到同一把锁。 */ + private static final ConcurrentHashMap SESSION_LOCKS = new ConcurrentHashMap<>(); @Value("${aiimage.file.chunk-upload.max-total-bytes:2147483648}") private long maxTotalBytes = 2L * 1024 * 1024 * 1024; @@ -194,7 +194,7 @@ public class ChunkedUploadService { "拼装后大小与声明不一致:声明 " + meta.totalBytes + ",实际 " + total); } deleteQuietly(sessionDir(uploadId)); - sessionLocks.remove(uploadId); + SESSION_LOCKS.remove(uploadId); String sha256 = HexFormat.of().formatHex(digest.digest()); log.info("[chunk-upload] 拼装完成 uploadId={} fileName={} bytes={} sha256={}", uploadId, meta.fileName, total, sha256); @@ -208,7 +208,7 @@ public class ChunkedUploadService { synchronized (lockOf(uploadId)) { deleteQuietly(sessionDir(uploadId)); } - sessionLocks.remove(uploadId); + SESSION_LOCKS.remove(uploadId); log.info("[chunk-upload] 已放弃分片会话 uploadId={} fileName={}", uploadId, meta.fileName); } @@ -273,7 +273,21 @@ public class ChunkedUploadService { } private Object lockOf(String uploadId) { - return sessionLocks.computeIfAbsent(uploadId, key -> new Object()); + return sessionLockOf(uploadId); + } + + /** + * P2-3:统一清理与 savePart/complete 必须是同一把锁,所以锁身份收在这里而不是 + * 各自 new 一份 map(两份 map 等于没锁)。锁的是盘上目录名,进程内互斥即够—— + * 每台实例只清自己的 local-temp 目录,跨实例不存在同一块盘。 + */ + public static Object sessionLockOf(String uploadId) { + return SESSION_LOCKS.computeIfAbsent(uploadId, key -> new Object()); + } + + /** 会话目录名形状(32 位十六进制);统一清理按它决定要不要拿会话锁。 */ + public static boolean isSessionDirectoryName(String name) { + return name != null && UPLOAD_ID_PATTERN.matcher(name).matches(); } /** diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/LocalTempCleanupService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/LocalTempCleanupService.java index d603ae50c..c3b888806 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/LocalTempCleanupService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/LocalTempCleanupService.java @@ -88,7 +88,7 @@ public class LocalTempCleanupService { continue; } if (child.isDirectory() && CHUNK_UPLOAD_DIR_NAME.equals(child.getName())) { - deletedSourceCount += deleteExpiredChildrenRecursively(child, sourceExpireBefore); + deletedSourceCount += deleteExpiredChunkUploads(child, sourceExpireBefore); // 删片会刷新会话目录 mtime,按"过期"再判目录会留下空壳;这里按空目录自底向上收口。 deleteEmptyDirectoriesDeep(child, tempDir); } @@ -126,6 +126,33 @@ public class LocalTempCleanupService { return 0; } + /** + * P2-3:分片会话的过期件回收必须与 savePart/complete 共用同一把会话锁——原来裸删, + * 在飞上传的半截会话可能被整点回收腰斩,用户的大档从 0 重来(正是断点续传要消灭的事)。 + * 非会话名的残留(历史上漏下的散件)保持原路径。 + */ + private int deleteExpiredChunkUploads(File chunkRoot, Instant expireBefore) { + int deletedCount = 0; + File[] sessions = chunkRoot.listFiles(); + if (sessions == null) { + return 0; + } + for (File session : sessions) { + if (!PathSafetyGuard.isInside(chunkRoot, session)) { + log.warn("local temp cleanup skipped unsafe path: {}", session.getAbsolutePath()); + continue; + } + if (ChunkedUploadService.isSessionDirectoryName(session.getName())) { + synchronized (ChunkedUploadService.sessionLockOf(session.getName())) { + deletedCount += deleteExpiredChildrenRecursively(session, expireBefore); + } + } else { + deletedCount += deleteExpiredChildrenRecursively(session, expireBefore); + } + } + return deletedCount; + } + private void deleteEmptyDirectories(File directory, File stopAt) { File current = directory; while (current != null && !current.equals(stopAt) && isDirectoryEmpty(current)) { diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/oss/OssStorageService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/oss/OssStorageService.java index c266f8c46..307cf4769 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/oss/OssStorageService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/file/service/oss/OssStorageService.java @@ -21,6 +21,7 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.InputStream; +import java.io.OutputStream; import java.net.URI; import java.net.URLDecoder; import java.nio.charset.StandardCharsets; @@ -541,6 +542,35 @@ public class OssStorageService { } } + /** + * 流式读对象(P3-1):证据件下载不再把整对象读进堆;超上限时中止并抛 IllegalArgumentException。 + */ + public void readObjectToStream(String bucket, String objectKey, OutputStream out, long maxBytes) { + String normalizedBucket = requireStorageName(bucket, "bucket"); + String normalizedObjectKey = requireStorageName(objectKey, "objectKey"); + try (var stream = buildClient().getObject(GetObjectArgs.builder() + .bucket(normalizedBucket) + .object(normalizedObjectKey) + .build())) { + byte[] buffer = new byte[64 * 1024]; + long total = 0; + int read; + while ((read = stream.read(buffer)) != -1) { + total += read; + if (total > maxBytes) { + throw new IllegalArgumentException("对象过大,超过读取上限 " + maxBytes + " 字节: " + + normalizedBucket + "/" + normalizedObjectKey); + } + out.write(buffer, 0, read); + } + out.flush(); + } catch (IllegalArgumentException ex) { + throw ex; + } catch (Exception ex) { + throw storageFailure("read", normalizedBucket + "/" + normalizedObjectKey, ex); + } + } + /** * Reads a managed result object from either its object key or stored public URL. * This overload keeps callers from depending on the configured bucket name. diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/permission/service/PermissionMenuService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/permission/service/PermissionMenuService.java index 40976fefc..423493a80 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/permission/service/PermissionMenuService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/permission/service/PermissionMenuService.java @@ -54,7 +54,8 @@ public class PermissionMenuService { /** 角色内在闸门菜单:非超管看不见、谁也授不出去(设计文档 §8.1)。 * 与前端 src/config/company-scope.ts 的 SUPER_ADMIN_ONLY_MENU_KEYS 必须同值,由 Task 7 的跨端用例钉住。 */ - private static final Set SUPER_ADMIN_ONLY_KEYS = Set.of("admin_companies"); + private static final Set SUPER_ADMIN_ONLY_KEYS = + Set.of("admin_companies", "admin_task_evidence"); /** * 管理/平台配置类菜单:出口仍是 {@code requireAdmin}(用户、菜单、分组、密钥、公司、版本、教程、 diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java index 7844cab20..2e27a20f7 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/SimilarAsinTaskService.java @@ -214,12 +214,6 @@ public class SimilarAsinTaskService implements SimilarAsinPipelineHost { private final SimilarAsinImagePrefetchService imagePrefetchService; private final UserApiSecretService userApiSecretService; - /** - * 采集路由开关:字段注入,避免给 {@code @RequiredArgsConstructor} 加参数而牵连一批 - * 手工 new 本类的测试。null(单测未注入)按「本地」处理,保持 activate 的原有行为。 - */ - @org.springframework.beans.factory.annotation.Autowired(required = false) - private com.nanri.aiimage.config.CollectRoutingProperties collectRoutingProperties; /** * Task 89:Excel 行解析器(表头/单元格读取、别名匹配、空行跳过、单字段截断)。 * 由 parseAndCreateTask 委托;解析语义与搬移前 parseWorkbook 完全一致。 diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinPipelineSupport.java b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinPipelineSupport.java index dd58b16c6..8d380bae6 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinPipelineSupport.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/similarasin/service/support/SimilarAsinPipelineSupport.java @@ -1409,7 +1409,9 @@ public class SimilarAsinPipelineSupport { } LlmBatchContext context = SimilarAsinPayloadSupport.readLlmBatchContext(objectMapper, state); if (context == null || context.resultId() == null) { - markLlmStateTerminal(state, LLM_STATUS_FAILED, "LLM 批次上下文缺失"); + markLlmStateTerminal(state, LLM_STATUS_FAILED, "LLM 异步工作流丢失批次上下文"); + // S-4:异步分支同样要给"欠费/坏密钥"留整单判死出口;非致命错误时该方法是 no-op。 + host.terminalizeOnFatalLlmFailure(taskForPoll(taskId), state.getLlmError()); continue; } List batchRows = readLlmBatchRows(state); @@ -1423,8 +1425,10 @@ public class SimilarAsinPipelineSupport { allRowsByBaseId); } markLlmStateTerminal(state, LLM_STATUS_FAILED, "LLM 异步工作流轮询超时"); + // S-4:轮询超时本身不是欠费;但若 state 上已带 LLM 欠费/坏密钥归因,走同一判死出口。 + host.terminalizeOnFatalLlmFailure(task, state.getLlmError()); maybeFinalizeLlmJob(taskId, context); - log.warn("[similar-asin] 文件任务超时兜底已将 LLM pending 批次置为失败 taskId={} stateId={} jobId={}", + log.warn("[similar-asin] 异步工作流轮询超时,本批按行失败收口 taskId={} stateId={} jobId={}", taskId, state.getId(), context.jobId()); } } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/softwareversion/service/SoftwareVersionService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/softwareversion/service/SoftwareVersionService.java index 1d9362797..ee5673d56 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/softwareversion/service/SoftwareVersionService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/softwareversion/service/SoftwareVersionService.java @@ -1,6 +1,7 @@ package com.nanri.aiimage.modules.softwareversion.service; import cn.hutool.crypto.digest.DigestUtil; +import com.nanri.aiimage.common.security.CredentialPackageGuard; import com.nanri.aiimage.common.text.UserMessages; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.nanri.aiimage.common.exception.BusinessException; @@ -281,6 +282,7 @@ public class SoftwareVersionService { deleteTempQuietly(source); return toItemMap(existing); } + rejectCredentialFiles(source); String objectKey = softwareVersionObjectKey(normalizedVersion); String fileUrl = ossStorageService.uploadSoftwareVersionPackage(source, objectKey); SoftwareVersionEntity entity = new SoftwareVersionEntity(); @@ -296,6 +298,28 @@ public class SoftwareVersionService { return toItemMap(saved == null ? entity : saved); } + /** + * P2-4:版本包会进公开 client 桶、发给所有客户端;压缩包里带 .env/*.pem 的一律拒绝。 + * 只读 zip 条目名,不解压内容。 + */ + private void rejectCredentialFiles(File source) { + String entry; + try { + entry = CredentialPackageGuard.firstForbiddenEntry(source); + } catch (IOException ex) { + log.warn("[software-version] 安装包无法读取,拒绝发布 name={} reason={}", + source.getName(), ex.getMessage()); + deleteTempQuietly(source); + throw new BusinessException("安装包无法读取或不是有效 zip,拒绝发布"); + } + if (entry != null) { + log.warn("[software-version] 安装包含凭据文件,拒绝发布 name={} entry={}", + source.getName(), entry); + deleteTempQuietly(source); + throw new BusinessException("安装包含凭据文件,拒绝发布"); + } + } + private void deleteTempQuietly(File file) { if (file != null && file.exists() && !file.delete()) { log.warn("[software-version] 清理分片临时件失败 path={}", file.getAbsolutePath()); diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/mapper/FileTaskMapper.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/mapper/FileTaskMapper.java index fa0fba717..3769b50dd 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/mapper/FileTaskMapper.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/mapper/FileTaskMapper.java @@ -25,4 +25,35 @@ public interface FileTaskMapper extends BaseMapper { List selectExpiredTerminalTaskIds(@Param("moduleType") String moduleType, @Param("cutoff") LocalDateTime cutoff, @Param("batchSize") int batchSize); + + /** + * 客户端兜底拉取候选:本用户 + 指定状态 + 白名单模块 + 创建早于 cutoff,且这一单走本机通道。 + * + *

通道判定必须在 SQL 里、在 {@code LIMIT} 之前。2026-10-31 的 P1-1 把它写在 Java 侧 + * (先 {@code limit 5} 取页、再 {@code filter(isLocal())}),混通道时只要排队里较老的 5 条是 + * ONLINE,本机通道的单本轮及以后都领不到——不是延后,是排到队尾之前永久饿死。 + * + *

只投影认领与组装载荷要用的列(其余字段由调用方 {@code selectById} 再取), + * 与迁移前的 Wrapper {@code select(...)} 逐字同形。 + */ + List selectLocalChannelPendingTasks(@Param("userId") Long userId, + @Param("status") String status, + @Param("moduleTypes") List moduleTypes, + @Param("localDefaultModules") List localDefaultModules, + @Param("cutoff") LocalDateTime cutoff, + @Param("limit") int limit); + + /** + * 服务端内部认领候选(Linux 后台 worker):指定状态 + 白名单模块 + 创建早于 cutoff, + * 且「不是本机通道」或「本机通道已等满兜底窗口 {@code localCutoff}」。 + * + *

同样是 SQL 侧谓词:原先先按 {@code cutoff}(2 分钟)取页、再在 Java 里按 30 分钟让位, + * 一页里全是让位中的本机单时本轮线上单零吞吐。 + */ + List selectInternalClaimCandidates(@Param("status") String status, + @Param("moduleTypes") List moduleTypes, + @Param("localDefaultModules") List localDefaultModules, + @Param("cutoff") LocalDateTime cutoff, + @Param("localCutoff") LocalDateTime localCutoff, + @Param("limit") int limit); } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/model/CollectChannel.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/model/CollectChannel.java index 2531412b4..8529c050b 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/model/CollectChannel.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/model/CollectChannel.java @@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.task.model; import com.nanri.aiimage.config.CollectRoutingProperties; +import java.util.List; import java.util.Locale; /** @@ -23,17 +24,28 @@ public enum CollectChannel { /** 线上采集:Linux 后台 worker 或外机 Windows。 */ ONLINE; + private static final List LOCAL_DEFAULT_MODULES = List.of( + CollectRoutingProperties.MODULE_APPEARANCE_PATENT, + CollectRoutingProperties.MODULE_SIMILAR_ASIN); + + /** + * 未登记通道时读作 {@link #LOCAL} 的模块清单。 + * + *

{@link #defaultFor(String)} 与 mapper 里那条通道谓词({@code FileTaskMapper.xml} 的 + * {@code resolvedChannelCase})共用这一处真源:SQL 侧要的是同一份「哪些模块默认本机」, + * 两边各写一套就会出现「Java 认为该让位、SQL 却把它捞进候选」的分裂判据。 + */ + public static List localDefaultModuleTypes() { + return LOCAL_DEFAULT_MODULES; + } + /** 该模块在没有显式登记通道时的默认执行方。 */ public static CollectChannel defaultFor(String moduleType) { if (moduleType == null || moduleType.isBlank()) { return ONLINE; } String key = moduleType.trim().toUpperCase(Locale.ROOT); - if (CollectRoutingProperties.MODULE_APPEARANCE_PATENT.equals(key) - || CollectRoutingProperties.MODULE_SIMILAR_ASIN.equals(key)) { - return LOCAL; - } - return ONLINE; + return LOCAL_DEFAULT_MODULES.contains(key) ? LOCAL : ONLINE; } /** 解析库里存的通道值;空或不可识别时回落到该模块的默认(不让一个脏值把单子永久饿死)。 */ diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskClientPullService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskClientPullService.java index 77a3d3b51..25d1116e3 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskClientPullService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskClientPullService.java @@ -1,6 +1,5 @@ package com.nanri.aiimage.modules.task.service; -import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; import com.nanri.aiimage.config.CollectRoutingProperties; import com.nanri.aiimage.config.InstanceMetadata; @@ -139,22 +138,14 @@ public class TaskClientPullService { log.warn("[client-task-pull] 模块白名单为空,本次不拉取 userId={}", userId); return List.of(); } - // 通道由这一单自己说(V183 collect_channel):选了线上的单子客户端不能领,领了就是抢跑。 + // 通道由这一单自己说(V183 collect_channel):选了线上的单子客户端不能领,领了就是抢跑, + // 所以判定必须发生在 SQL 里——先按 limit 取页、回到 Java 再 filter 的话,混通道时较老的 + // N 条是 ONLINE 就把本机单永久挡在队尾之后(不是延后,是饿死)。 long startedAt = System.currentTimeMillis(); LocalDateTime cutoff = LocalDateTime.now(DB_ZONE).minusMinutes(Math.max(1L, minPendingMinutes)); int safeLimit = Math.max(1, Math.min(limit, MAX_LIMIT)); - List candidates = fileTaskMapper.selectList(new LambdaQueryWrapper() - .select(FileTaskEntity::getId, FileTaskEntity::getModuleType, - FileTaskEntity::getCollectChannel, FileTaskEntity::getCreatedAt) - .eq(FileTaskEntity::getUserId, userId) - .eq(FileTaskEntity::getStatus, STATUS_PENDING) - .in(FileTaskEntity::getModuleType, types) - .lt(FileTaskEntity::getCreatedAt, cutoff) - .orderByAsc(FileTaskEntity::getCreatedAt) - .last("limit " + safeLimit)) - .stream() - .filter(task -> CollectChannel.resolve(task.getCollectChannel(), task.getModuleType()).isLocal()) - .toList(); + List candidates = fileTaskMapper.selectLocalChannelPendingTasks( + userId, STATUS_PENDING, types, CollectChannel.localDefaultModuleTypes(), cutoff, safeLimit); if (candidates.isEmpty() && !types.isEmpty()) { log.info("[client-task-pull] 白名单内没有本机通道的排队任务,本次不拉取 userId={} moduleTypes={}", userId, moduleTypes); @@ -195,27 +186,10 @@ public class TaskClientPullService { Math.max(1L, internalMinPendingMinutes)); LocalDateTime localCutoff = now.minusMinutes(localFallbackMinutes); int safeLimit = Math.max(1, Math.min(limit, MAX_LIMIT)); - List candidates = fileTaskMapper.selectList(new LambdaQueryWrapper() - .select(FileTaskEntity::getId, FileTaskEntity::getModuleType, - FileTaskEntity::getCollectChannel, FileTaskEntity::getCreatedAt) - .eq(FileTaskEntity::getStatus, STATUS_PENDING) - .in(FileTaskEntity::getModuleType, types) - .lt(FileTaskEntity::getCreatedAt, cutoff) - .orderByAsc(FileTaskEntity::getCreatedAt) - .last("limit " + safeLimit)); - if (!candidates.isEmpty()) { - int before = candidates.size(); - // 让位只看这一单的通道:本机通道的单子等满兜底窗口才接管,选了线上的立刻可领 - candidates = candidates.stream() - .filter(task -> !CollectChannel.resolve(task.getCollectChannel(), task.getModuleType()).isLocal() - || task.getCreatedAt() == null - || !task.getCreatedAt().isAfter(localCutoff)) - .toList(); - if (candidates.size() < before) { - log.info("[client-task-pull] 内部认领让位本机:本轮跳过 {} 条 LOCAL 通道的排队任务(兜底窗口 {} 分钟)", - before - candidates.size(), localFallbackMinutes); - } - } + // 让位判定同样在 SQL 里(与客户端拉取同一个形状):原先按 cutoff 取一页再在 Java 里按 + // localCutoff 让位,整页都是「本机通道且还没等满兜底窗口」时,本轮线上单一条也领不到。 + List candidates = fileTaskMapper.selectInternalClaimCandidates( + STATUS_PENDING, types, CollectChannel.localDefaultModuleTypes(), cutoff, localCutoff, safeLimit); if (candidates.isEmpty()) { return List.of(); } diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobService.java index 0bc352a3c..3ed2698f1 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskFileJobService.java @@ -240,17 +240,20 @@ public class TaskFileJobService { } /** - * 终态父任务的 job 直接拉满重试:不执行、不再被下一轮扫到。 + * 终态父任务的 job 也放行认领(P1-13 根治):这张表只有 ASSEMBLE_RESULT 一种作业, + * 它的唯一职责是把已回传的数据拼成表,不消耗任何外部资源——父任务已终态恰恰 + * 意味着数据已定型,此时出表才是对的。原先在这里无条件判死,出表只剩「卡死扫描 + * 捞 FAILED+retry 到顶」一条补偿路径:至少一个扫描周期的窗口里用户停在红字, + * 兜底再一失败那张本可以出的表就永远出不来(2026-10-31 现网任务 33062 取证)。 * - *

终态回调要留给卡死扫描的失败兜底({@code fallbackAssembleOnFailure}):外观专利等模块 - * 靠它用已回传分片出部分结果文件,「起码失败了也可以下载」才不会被 B28 的终态守卫顺手掐掉。 - * 这里若直接打 {@code terminal_callback_at},兜底扫描再也捞不到这行,用户只剩红字、连半张表都没有。 + *

终态守卫的本意是「终态后别再继续检测、停心跳续期」,保留在 + * {@code TaskResultFileJobWorker} 的心跳层(stage=TERMINAL_STOP_RENEW); + * 出表执行与任务结论脱钩(铁律 14):组装只写 biz_file_result,biz_file_task + * 的 status/error_message 一字不动。 */ - private void skipJobOfTerminalTask(TaskFileJobEntity job, String stage) { - String message = "父任务已终态,结果作业不再执行"; - markFailedPermanentAwaitingFallback(job, message); - log.info("[task-file-job] skipped because parent task is terminal jobId={} taskId={} moduleType={} stage={}", - job.getId(), job.getTaskId(), job.getModuleType(), stage); + private void noteTerminalTaskStillDeliverable(TaskFileJobEntity job) { + log.info("[task-file-job] parent task terminal; ASSEMBLE_RESULT still delivers jobId={} taskId={} moduleType={} stage=TERMINAL_ASSEMBLE", + job.getId(), job.getTaskId(), job.getModuleType()); } private List claimCandidates(List candidates) { @@ -263,8 +266,7 @@ public class TaskFileJobService { continue; } if (isOwningTaskTerminal(candidate.getTaskId())) { - skipJobOfTerminalTask(candidate, "CLAIM"); - continue; + noteTerminalTaskStillDeliverable(candidate); } if (markRunning(candidate.getId())) { claimedIds.add(candidate.getId()); @@ -629,8 +631,8 @@ public class TaskFileJobService { } TaskFileJobEntity existing = taskFileJobMapper.selectById(jobId); if (existing != null && isOwningTaskTerminal(existing.getTaskId())) { - skipJobOfTerminalTask(existing, "CLAIM_RUNNING"); - return null; + // P1-13:终态父任务也放行认领(见 noteTerminalTaskStillDeliverable),出表不因终态停摆。 + noteTerminalTaskStillDeliverable(existing); } if (!markRunning(jobId)) { return null; diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorker.java b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorker.java index 012a428f6..a3b0325a9 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorker.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorker.java @@ -260,12 +260,9 @@ public class TaskResultFileJobWorker { } private void processClaimedWithHeartbeat(TaskFileJobEntity job, TaskFileJobEntity claim) { - if (taskFileJobService.isOwningTaskTerminal(job.getTaskId())) { - taskFileJobService.markFailedPermanentAwaitingFallback(job, "父任务已终态,结果作业不再执行"); - log.info("[task-file-job] skipped because parent task is terminal jobId={} taskId={} moduleType={} stage=TERMINAL_SKIP", - job.getId(), job.getTaskId(), job.getModuleType()); - return; - } + // P1-13:不再在执行入口按父任务终态判死——这张表只有纯出表作业,终态恰恰说明数据 + // 已定型,此时出表才是对的(2026-10-31 现网任务 33062:守卫判死 + 兜底 50 秒后才有表, + // 窗口里用户只剩红字)。终态的「别再检测」语义保留在心跳层(stage=TERMINAL_STOP_RENEW)。 if (!taskFileJobService.activateRunningClaim(claim)) { log.info("[task-file-job] queued claim expired before execution jobId={} taskId={} moduleType={} claimUpdatedAt={}", job.getId(), job.getTaskId(), job.getModuleType(), claim.getUpdatedAt()); @@ -295,7 +292,9 @@ public class TaskResultFileJobWorker { job.getId(), job.getTaskId(), job.getModuleType()); return; } - taskFileJobService.markFailedPermanentAwaitingFallback(job, "父任务已终态,结果作业不再执行"); + // P1-13:这条只该在「执行真的没产出结果」时走到——出表已放行,没出成表才是遗骸, + // 留给兜底扫描出部分结果(幂等重入)。 + taskFileJobService.markFailedPermanentAwaitingFallback(job, "父任务已终态且本次执行未产出结果文件"); log.info("[task-file-job] parent task terminal after execution; failed job for fallback jobId={} taskId={} moduleType={} stage=TERMINAL_AFTER_EXECUTION", job.getId(), job.getTaskId(), job.getModuleType()); } @@ -328,14 +327,6 @@ public class TaskResultFileJobWorker { job.getId(), job.getTaskId(), job.getModuleType(), job.getResultId()); return; } - if (taskFileJobService.isOwningTaskTerminal(job.getTaskId())) { - try (lockHandle) { - taskFileJobService.markFailedPermanentAwaitingFallback(job, "父任务已终态,结果作业不再执行"); - } - log.info("[task-file-job] skipped after acquiring lock because parent task is terminal jobId={} taskId={} moduleType={} stage=TERMINAL_AFTER_LOCK", - job.getId(), job.getTaskId(), job.getModuleType()); - return; - } activeLockHandles.put(job.getId(), lockHandle); try (lockHandle) { latest = taskFileJobService.findById(job.getId()); diff --git a/backend-java/src/main/java/com/nanri/aiimage/modules/tutorial/service/TutorialPackageService.java b/backend-java/src/main/java/com/nanri/aiimage/modules/tutorial/service/TutorialPackageService.java index 2c91143e3..d94108199 100644 --- a/backend-java/src/main/java/com/nanri/aiimage/modules/tutorial/service/TutorialPackageService.java +++ b/backend-java/src/main/java/com/nanri/aiimage/modules/tutorial/service/TutorialPackageService.java @@ -2,6 +2,7 @@ package com.nanri.aiimage.modules.tutorial.service; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.common.security.CredentialPackageGuard; import com.nanri.aiimage.modules.file.service.LocalFileStorageService; import com.nanri.aiimage.modules.file.service.oss.OssStorageService; import com.nanri.aiimage.modules.tutorial.mapper.TutorialPackageMapper; @@ -11,6 +12,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import java.io.IOException; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.util.ArrayList; @@ -160,6 +162,7 @@ public class TutorialPackageService { } return toItemMap(existing); } + rejectCredentialFiles(source); String generatedKey = OBJECT_KEY_PREFIX + OBJECT_NAME_FORMATTER.format(LocalDateTime.now()) + "-" + safeFileName(normalizedName); ossStorageService.uploadTutorialPackage(source, generatedKey); @@ -262,6 +265,29 @@ public class TutorialPackageService { } } + /** P2-4:教程包同样进公开 client 桶,压缩包里带 .env/*.pem 的一律拒绝。 */ + private void rejectCredentialFiles(java.io.File source) { + String entry; + try { + entry = CredentialPackageGuard.firstForbiddenEntry(source); + } catch (IOException ex) { + log.warn("[tutorial] 教程包无法读取,拒绝发布 name={} reason={}", + source.getName(), ex.getMessage()); + if (source.exists() && !source.delete()) { + log.warn("[tutorial] 清理拒绝的临时件失败 path={}", source.getAbsolutePath()); + } + throw new BusinessException("教程包无法读取或不是有效 zip,拒绝发布"); + } + if (entry != null) { + log.warn("[tutorial] 教程包含凭据文件,拒绝发布 name={} entry={}", + source.getName(), entry); + if (source.exists() && !source.delete()) { + log.warn("[tutorial] 清理拒绝的临时件失败 path={}", source.getAbsolutePath()); + } + throw new BusinessException("教程包含凭据文件,拒绝发布"); + } + } + private String requireZipFileName(String fileName) { String normalized = fileName == null ? "" : fileName.trim(); if (normalized.isEmpty()) { diff --git a/backend-java/src/main/resources/application-local.example.yml b/backend-java/src/main/resources/application-local.example.yml index cf15f8c20..f5b93b945 100644 --- a/backend-java/src/main/resources/application-local.example.yml +++ b/backend-java/src/main/resources/application-local.example.yml @@ -37,6 +37,8 @@ AIIMAGE_OSS_ACCESS_KEY_SECRET=change-me # 任务异常现场证据件(v1.2 Phase 3):独立桶与保留期必须显式配置;桶不得复用 nanri-ai-images。 AIIMAGE_EVIDENCE_OSS_BUCKET=task-evidence AIIMAGE_EVIDENCE_RETENTION_DAYS=7 +AIIMAGE_EVIDENCE_RETENTION_CRON=0 30 3 * * * +AIIMAGE_EVIDENCE_MAX_FILE_BYTES=67108864 AIIMAGE_EVIDENCE_KEY_PREFIX=task-evidence/ # 工作台客户端包直传预签名有效期(秒);90MB 级包在慢公网入口需要更长窗口。 diff --git a/backend-java/src/main/resources/db/V184__collect_pull_and_evidence_indexes.sql b/backend-java/src/main/resources/db/V184__collect_pull_and_evidence_indexes.sql new file mode 100644 index 000000000..24267b393 --- /dev/null +++ b/backend-java/src/main/resources/db/V184__collect_pull_and_evidence_indexes.sql @@ -0,0 +1,50 @@ +-- V184: 兜底拉取与证据保留期两条取页路径的缺失索引(CR 第二轮 P1-1 同批 / P2-1) +-- +-- 1) biz_file_task(status, module_type, created_at) +-- TaskClientPullService 的通道谓词从 Java 侧下推进 SQL 之后,内部认领(Linux worker)这一支 +-- 的真实谓词就是 status='PENDING' AND module_type IN (...) AND created_at < ? ORDER BY created_at。 +-- 现有索引里没有一条以 status 打头再跟 module_type(V120 是 (status, updated_at)、 +-- V34 是 (module_type, user_id, created_at)、V92 是 (owner_instance_id, status, updated_at)), +-- 于是每轮按 PENDING 全集排序。 +-- collect_channel 有意不进索引:通道判定写成 CASE(NULL 值在 SQL 三值逻辑下必须兜成模块默认, +-- 直接比值会让「未登记通道的非本机模块」被 NOT 判成 UNKNOWN 而永久领不到),表达式谓词不吃索引; +-- PENDING 本身是被本索引筛小的那一层,量级 = 在排队的单子。 +-- 2) biz_task_evidence(created_at) +-- V182 只建了 (task_id, created_at):保留期回收的谓词是单列 created_at < ?,前导列不符 ⇒ +-- purgeExpired 每一批全表扫,而这张表按「异常即产件」只增不减。 +-- +-- 风险:ALTER TABLE ADD INDEX 需申请元数据锁(MySQL 8 在线 DDL,INPLACE),建议低峰执行; +-- 本迁移按 information_schema 判存,重跑不会因索引已存在而失败。 +-- 回滚(手工): +-- ALTER TABLE biz_file_task DROP INDEX idx_biz_file_task_status_module_created; +-- ALTER TABLE biz_task_evidence DROP INDEX idx_task_evidence_created_at; + +SET @db_name = DATABASE(); + +-- 1) biz_file_task(status, module_type, created_at) +SET @idx_exists := ( + SELECT COUNT(*) FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = @db_name AND TABLE_NAME = 'biz_file_task' + AND INDEX_NAME = 'idx_biz_file_task_status_module_created' +); +SET @sql := IF(@idx_exists = 0, + 'ALTER TABLE biz_file_task ADD INDEX idx_biz_file_task_status_module_created (status, module_type, created_at)', + 'SELECT 1' +); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +-- 2) biz_task_evidence(created_at) +SET @idx_exists := ( + SELECT COUNT(*) FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = @db_name AND TABLE_NAME = 'biz_task_evidence' + AND INDEX_NAME = 'idx_task_evidence_created_at' +); +SET @sql := IF(@idx_exists = 0, + 'ALTER TABLE biz_task_evidence ADD INDEX idx_task_evidence_created_at (created_at)', + 'SELECT 1' +); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; diff --git a/backend-java/src/main/resources/db/V185__task_evidence_menu.sql b/backend-java/src/main/resources/db/V185__task_evidence_menu.sql new file mode 100644 index 000000000..abaac86e8 --- /dev/null +++ b/backend-java/src/main/resources/db/V185__task_evidence_menu.sql @@ -0,0 +1,15 @@ +-- 迁移 V185:后台菜单「任务证据」(v1.2 P1-9 / R3 后管可视) +-- +-- 证据件读取面 /api/admin/task-evidence/** 已按 requireSuperAdmin 收口,但后台没有入口, +-- 等于「能上传、能登记、没人看得见」。这里只新增一条菜单行,挂在「记录与版本」分组下, +-- 幂等插入:column_key 已存在时不再写第二行。 +-- +-- 回滚:DELETE FROM columns WHERE column_key='admin_task_evidence'; + +INSERT INTO columns (name, column_key, menu_type, route_path, sort_order, parent_id) +SELECT '任务证据', 'admin_task_evidence', 'admin', 'records/task-evidence', 84, parent.id +FROM columns parent +WHERE parent.column_key = 'admin_group_record' + AND NOT EXISTS ( + SELECT 1 FROM columns WHERE column_key = 'admin_task_evidence' + ); diff --git a/backend-java/src/main/resources/mapper/task/FileTaskMapper.xml b/backend-java/src/main/resources/mapper/task/FileTaskMapper.xml index 7bc6c0821..a87bec344 100644 --- a/backend-java/src/main/resources/mapper/task/FileTaskMapper.xml +++ b/backend-java/src/main/resources/mapper/task/FileTaskMapper.xml @@ -11,4 +11,48 @@ ORDER BY id ASC LIMIT #{batchSize} + + + + CASE + WHEN collect_channel IN ('LOCAL', 'ONLINE') THEN collect_channel + WHEN module_type IN + #{localDefaultModule} + THEN 'LOCAL' + ELSE 'ONLINE' + END + + + + + + + diff --git a/backend-java/src/test/java/com/nanri/aiimage/MigratedSqlEquivalenceTest.java b/backend-java/src/test/java/com/nanri/aiimage/MigratedSqlEquivalenceTest.java index 370baa97c..93bdf6265 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/MigratedSqlEquivalenceTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/MigratedSqlEquivalenceTest.java @@ -74,7 +74,9 @@ class MigratedSqlEquivalenceTest { // 99 → 100:Phase 5(AC-19)为任务中心深翻页新增窄锚点语句 {@code TaskCenterQueryMapper#selectTaskPageAnchor} // (两列投影算页首游标),同轮点名进 NOT_MIGRATED;同轮的 selectTaskPage 改写让它从账本搬进 // NOT_MIGRATED(select 总数不变,只是记账位置换了)。 - private static final int XML_SELECT_TOTAL = 100; + // 100 → 102:P1-1 把两条认领查询的通道谓词下推进 SQL,FileTaskMapper 新写 2 条 select, + // 逐条点名进 NOT_MIGRATED(SqlLayeringConventionTest.XML_STATEMENT_TOTAL 同轮 144 → 146)。 + private static final int XML_SELECT_TOTAL = 102; /** * 不是「从注解搬过来」的语句:注解时代没有原文可对撞,等价性这支只能跳过它们。 @@ -135,7 +137,13 @@ class MigratedSqlEquivalenceTest { // 所以整行从 migrated-statements.tsv 摘掉、点名到这里(LEDGER_ROWS 85→84)。 // selectTaskPageAnchor —— 本轮新写的两列窄投影,从来没有注解原文可对撞。 "TaskCenterQueryMapper#selectTaskPage", - "TaskCenterQueryMapper#selectTaskPageAnchor"); + "TaskCenterQueryMapper#selectTaskPageAnchor", + // P1-1(2026-10-31):客户端兜底拉取与服务端内部认领的通道谓词从 Java 层下推进 SQL + // ——先 limit 取页再 filter(isLocal()) 会让本机单被前面的线上单永久挡死。两条都是 + // 本轮新写的 select,从来没有注解原文可对撞(XML_SELECT_TOTAL 同轮 100 → 102, + // SqlLayeringConventionTest.XML_STATEMENT_TOTAL 同轮 144 → 146)。 + "FileTaskMapper#selectLocalChannelPendingTasks", + "FileTaskMapper#selectInternalClaimCandidates"); /** 账本 5 列:接口 / 方法 / @Select 种类 / 是否含常量插值 / 注解原文(Java 反转义后那条串)。 */ private static final int LEDGER_COLUMNS = 5; diff --git a/backend-java/src/test/java/com/nanri/aiimage/SqlLayeringConventionTest.java b/backend-java/src/test/java/com/nanri/aiimage/SqlLayeringConventionTest.java index e701673fd..c7f2f55d3 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/SqlLayeringConventionTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/SqlLayeringConventionTest.java @@ -91,7 +91,11 @@ class SqlLayeringConventionTest { // 143 → 144:Phase 5(AC-19/P1-B1)为任务中心深翻页新增窄锚点语句 // {@code TaskCenterQueryMapper#selectTaskPageAnchor}(两列投影算页首游标), // 同轮登记进 MigratedSqlEquivalenceTest.NOT_MIGRATED(该文件的 XML_SELECT_TOTAL 同轮 99 → 100)。 - private static final int XML_STATEMENT_TOTAL = 144; + // 144 → 146:P1-1(2026-10-31)把「这一单交本机还是交线上」的判定从 Java 层下推进 SQL—— + // 原形状是先 limit 取页再 filter(isLocal()),混通道时较老的 N 条是 ONLINE 就把本机单永久饿死。 + // FileTaskMapper 新写 2 条 select(客户端兜底拉取 / 服务端内部认领),同轮登记进 + // MigratedSqlEquivalenceTest.NOT_MIGRATED(该文件 XML_SELECT_TOTAL 同轮 100 → 102)。 + private static final int XML_STATEMENT_TOTAL = 146; /** * wrapper 里夹带函数级 SQL 的现状("这不是简单单表"的证据),文件名 → 条数: diff --git a/backend-java/src/test/java/com/nanri/aiimage/common/security/CredentialPackageGuardTest.java b/backend-java/src/test/java/com/nanri/aiimage/common/security/CredentialPackageGuardTest.java new file mode 100644 index 000000000..db3532a2f --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/common/security/CredentialPackageGuardTest.java @@ -0,0 +1,68 @@ +package com.nanri.aiimage.common.security; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** P2-4:版本包/教程包发布前的压缩包凭据件扫描。 */ +class CredentialPackageGuardTest { + + @TempDir + Path tempDir; + + private File zipWith(String... entries) throws IOException { + File zip = tempDir.resolve("pkg-" + System.nanoTime() + ".zip").toFile(); + try (ZipOutputStream out = new ZipOutputStream(Files.newOutputStream(zip.toPath()))) { + for (String entry : entries) { + out.putNextEntry(new ZipEntry(entry)); + out.write("x".getBytes(StandardCharsets.UTF_8)); + out.closeEntry(); + } + } + return zip; + } + + @Test + void nestedDotEnvIsRejected() throws IOException { + File pkg = zipWith("app/readme.txt", "app/conf/.env"); + assertEquals("app/conf/.env", CredentialPackageGuard.firstForbiddenEntry(pkg)); + } + + @Test + void envVariantAndKeyMaterialAreRejected() throws IOException { + assertEquals("docs/.env.production", + CredentialPackageGuard.firstForbiddenEntry(zipWith("docs/.env.production"))); + assertEquals("certs/server.PEM", + CredentialPackageGuard.firstForbiddenEntry(zipWith("certs/server.PEM"))); + assertEquals("tools/kw_share_cfg.json", + CredentialPackageGuard.firstForbiddenEntry(zipWith("tools/kw_share_cfg.json"))); + } + + @Test + void ordinaryPackagePasses() throws IOException { + assertNull(CredentialPackageGuard.firstForbiddenEntry( + zipWith("app/readme.txt", "app/config/app.yml", "app/update/update.exe"))); + } + + @Test + void missingOrNonZipFileFailsClosed() throws IOException { + File notZip = tempDir.resolve("not-a-zip.zip").toFile(); + Files.writeString(notZip.toPath(), "plain text"); + IOException missing = assertThrows(IOException.class, + () -> CredentialPackageGuard.firstForbiddenEntry(tempDir.resolve("absent.zip").toFile())); + assertTrue(missing.getMessage().contains("missing")); + assertThrows(IOException.class, () -> CredentialPackageGuard.firstForbiddenEntry(notZip)); + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/config/CatchSilenceRatchetTest.java b/backend-java/src/test/java/com/nanri/aiimage/config/CatchSilenceRatchetTest.java index bcad2089d..a094cec77 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/config/CatchSilenceRatchetTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/config/CatchSilenceRatchetTest.java @@ -62,7 +62,8 @@ class CatchSilenceRatchetTest { // 10-30 用户端代理白名单(极客/青果)抬 591 -> 598(7 处逐一记名见 STRICT_TOTAL 上方注释; // 实测量到 599,多出的 1 格来自已提交的 79b04afd(AdminApiGuardFilter 编码绕过 catch),见同处说明)。 // 10-31 Phase 5.1:证据登记失败回收对象一枚(内层回收失败只 WARN),599 -> 600。 - static final int GIVEN_NOT_RETHROW = 600; + // 10-31 CR 消项:证据保留期清理失败 WARN 兜底 + 任务号溢出回落 task_no,新增 2 处非抛分支,600 -> 602。 + static final int GIVEN_NOT_RETHROW = 602; static final int GIVEN_LOG_ONLY = 249; static final int GIVEN_EMPTY = 56; @@ -280,7 +281,8 @@ class CatchSilenceRatchetTest { * 后转业务错误(RETHROW+rethrow 401);内层 catch 是对象回收本身失败只 WARN * (LOG_ONLY+logOnly 190)。两处误差方向都是「不让孤儿对象永久占桶」。 */ - static final int STRICT_TOTAL = 1057; + // 10-31 CR 消项:版本包/教程包凭据扫描新增 6 处 catch(均转业务异常或带 WARN),1057 -> 1063。 + static final int STRICT_TOTAL = 1063; /** * 重抛型 catch 的**下限**(不是上限):本轮把两处吞异常改成重抛之后实测 388。 * 这一格只能涨不能跌——它是"异常会往外走"的总量,把它钉成"只许减"就等于拦着别人修吞异常。 @@ -404,7 +406,7 @@ class CatchSilenceRatchetTest { * (畸形编码保持原样交给容器 400),非本批新增,说明见 STRICT_TOTAL 上方。 */ // 10-31 Phase 5.1:同 GIVEN_NOT_RETHROW,证据登记补偿回收失败只 WARN,599 -> 600。 - static final int STRICT_NOT_RETHROW = 600; + static final int STRICT_NOT_RETHROW = 602; /** * 10-17 抬 172 -> 173,出处就是上面点名过的 {@code TaskHeartbeatLivenessService.stamp()}: * 块体只有「取到 key 名 + log.warn」一句,写心跳证据失败时既不重试也不抛出—— diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentActivateRoutingTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentActivateRoutingTest.java index 27e3c3353..522807767 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentActivateRoutingTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/appearancepatent/service/AppearancePatentActivateRoutingTest.java @@ -9,7 +9,6 @@ import com.nanri.aiimage.common.exception.BusinessException; import com.nanri.aiimage.common.service.DistributedJobLockService; import com.nanri.aiimage.common.service.ResultDownloadResolver; import com.nanri.aiimage.config.AppearancePatentProperties; -import com.nanri.aiimage.config.CollectRoutingProperties; import com.nanri.aiimage.config.InstanceMetadata; import com.nanri.aiimage.config.StorageProperties; import com.nanri.aiimage.modules.appearancepatent.client.AppearancePatentLlmClient; @@ -68,7 +67,7 @@ class AppearancePatentActivateRoutingTest { FileTaskEntity task = pendingTask(com.nanri.aiimage.modules.task.model.CollectChannel.LOCAL); when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task); when(fileTaskMapper.update(isNull(), any())).thenReturn(1); - AppearancePatentTaskService service = service(new CollectRoutingProperties()); + AppearancePatentTaskService service = service(); service.activateTask(TASK_ID, USER_ID); @@ -85,12 +84,12 @@ class AppearancePatentActivateRoutingTest { void 线上通道的单_activate_保持后台语义并明确拒绝() { when(fileTaskMapper.selectById(TASK_ID)) .thenReturn(pendingTask(com.nanri.aiimage.modules.task.model.CollectChannel.ONLINE)); - AppearancePatentTaskService service = service(new CollectRoutingProperties()); + AppearancePatentTaskService service = service(); BusinessException error = assertThrows(BusinessException.class, () -> service.activateTask(TASK_ID, USER_ID)); - assertTrue(String.valueOf(error.getMessage()).contains("后台自动执行"), error.getMessage()); + assertTrue(String.valueOf(error.getMessage()).contains("线上采集"), error.getMessage()); verify(fileTaskMapper, never()).update(isNull(), any()); } @@ -100,7 +99,7 @@ class AppearancePatentActivateRoutingTest { when(fileTaskMapper.selectById(TASK_ID)).thenReturn(task); when(fileTaskMapper.update(isNull(), any())).thenReturn(1); - service(new CollectRoutingProperties()).activateTask(TASK_ID, USER_ID); + service().activateTask(TASK_ID, USER_ID); verify(taskCacheService).touchTaskHeartbeat(TASK_ID); } @@ -115,7 +114,7 @@ class AppearancePatentActivateRoutingTest { return task; } - private AppearancePatentTaskService service(CollectRoutingProperties routing) { + private AppearancePatentTaskService service() { AppearancePatentTaskService service = new AppearancePatentTaskService( mock(com.nanri.aiimage.modules.file.service.LocalFileStorageService.class), null, @@ -139,7 +138,6 @@ class AppearancePatentActivateRoutingTest { mock(InstanceMetadata.class), mock(TaskProgressLightAssembler.class), mock(UserApiSecretService.class)); - ReflectionTestUtils.setField(service, "collectRoutingProperties", routing); return service; } diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/evidence/service/TaskEvidenceOwnershipAndRetentionTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/evidence/service/TaskEvidenceOwnershipAndRetentionTest.java new file mode 100644 index 000000000..b6ea5f345 --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/evidence/service/TaskEvidenceOwnershipAndRetentionTest.java @@ -0,0 +1,189 @@ +package com.nanri.aiimage.modules.evidence.service; + +import com.nanri.aiimage.common.exception.BusinessException; +import com.nanri.aiimage.common.service.DistributedJobLockService; +import com.nanri.aiimage.common.task.LockSkipWitness; +import com.nanri.aiimage.config.EvidenceProperties; +import com.nanri.aiimage.modules.evidence.mapper.TaskEvidenceMapper; +import com.nanri.aiimage.modules.evidence.model.entity.TaskEvidenceEntity; +import com.nanri.aiimage.modules.evidence.storage.TaskEvidenceStorageService; +import com.nanri.aiimage.modules.file.service.LocalFileStorageService; +import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; +import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +/** + * 证据登记的归属与保留期的执行保证(CR 第二轮 P1-2/P1-7/P2-2)。 + * + *

P1-2 的判据是「归属先于落桶」:task_id 对得上任务行而 uid 不是行主 ⇒ 403 且 + * {@code storage.store} 一次都不许被调——校验晚于落桶的话,每一次拒绝都在证据桶里 + * 留一个收不回的孤儿对象。P1-7 的判据是「保留期必须有执行者且不堵车」:拿不到分布式锁 + * 要走可聚合出口;单个对象删除失败只牺牲那一行,键集继续往前走。 + */ +class TaskEvidenceOwnershipAndRetentionTest { + + private static final String TASK_ID = "33062"; + + private final TaskEvidenceMapper mapper = mock(TaskEvidenceMapper.class); + private final TaskEvidenceStorageService storage = mock(TaskEvidenceStorageService.class); + private final LocalFileStorageService localFiles = mock(LocalFileStorageService.class); + private final EvidenceProperties properties = new EvidenceProperties(); + private final FileTaskMapper fileTaskMapper = mock(FileTaskMapper.class); + private final DistributedJobLockService locks = mock(DistributedJobLockService.class); + + @TempDir + Path tempDir; + + @Test + void 他人任务的证据必须403且一次都不落桶() throws Exception { + stubTaskOwner(7L); + stubUploadableFile(); + + BusinessException ex = assertThrows(BusinessException.class, + () -> service().register(TASK_ID, 8L, "client", "goods-spider", "evidence.json")); + + assertEquals(403, ex.getCode().intValue(), "冒名登记必须 403,不是 500 也不是静默成功"); + verify(storage, never()).store(any(), any(), any()); + verify(mapper, never()).insert(any(TaskEvidenceEntity.class)); + } + + @Test + void 无归属uid的登记必须拒绝() throws Exception { + stubTaskOwner(7L); + stubUploadableFile(); + + assertThrows(BusinessException.class, + () -> service().register(TASK_ID, null, "client", "goods-spider", "evidence.json")); + verify(storage, never()).store(any(), any(), any()); + } + + @Test + void 本人任务照常登记且登记行带上uid() throws Exception { + stubTaskOwner(7L); + Path file = stubUploadableFile(); + when(storage.store(any(), eq(TASK_ID), eq("evidence.json"))).thenReturn( + new TaskEvidenceStorageService.StoredEvidence("task-evidence/k", 3L, "sha", "application/json")); + when(mapper.insert(any(TaskEvidenceEntity.class))).thenReturn(1); + + service().register(TASK_ID, 7L, "client", "goods-spider", "evidence.json"); + + org.mockito.ArgumentCaptor captor = + org.mockito.ArgumentCaptor.forClass(TaskEvidenceEntity.class); + verify(mapper).insert(captor.capture()); + assertEquals(7L, captor.getValue().getUid(), "登记行的 uid 必须是行主本人"); + assertEquals(TASK_ID, captor.getValue().getTaskId()); + assertTrue(Files.notExists(file) || Files.exists(file), "临时件清理失败只留 warn,不影响登记"); + } + + @Test + void 客户端本地任务号带uid自证即可登记() throws Exception { + when(fileTaskMapper.selectById(anyLong())).thenReturn(null); + when(fileTaskMapper.selectOne(any())).thenReturn(null); + stubUploadableFile(); + when(storage.store(any(), any(), any())).thenReturn( + new TaskEvidenceStorageService.StoredEvidence("task-evidence/k", 3L, "sha", "application/json")); + when(mapper.insert(any(TaskEvidenceEntity.class))).thenReturn(1); + + service().register("local-job-abc", 5L, "client", "goods-spider", "evidence.json"); + + verify(storage).store(any(), eq("local-job-abc"), eq("evidence.json")); + } + + @Test + void 零字节证据件拒绝登记() throws Exception { + when(fileTaskMapper.selectById(anyLong())).thenReturn(null); + when(fileTaskMapper.selectOne(any())).thenReturn(null); + Path empty = Files.createFile(tempDir.resolve("empty.png")); + when(localFiles.findLocalSourceFile("empty.png")).thenReturn(empty.toFile()); + + assertThrows(BusinessException.class, + () -> service().register(TASK_ID, 7L, "client", "goods-spider", "empty.png")); + verify(storage, never()).store(any(), any(), any()); + } + + @Test + void 同任务证据件数到上限必须拒绝() throws Exception { + when(fileTaskMapper.selectById(anyLong())).thenReturn(null); + when(fileTaskMapper.selectOne(any())).thenReturn(null); + when(mapper.selectCount(any())).thenReturn(50L); + stubUploadableFile(); + + assertThrows(BusinessException.class, + () -> service().register(TASK_ID, 7L, "client", "goods-spider", "evidence.json")); + verify(storage, never()).store(any(), any(), any()); + } + + @Test + void 清理拿不到锁必须走可聚合出口且不碰库() { + long before = LockSkipWitness.countOf(TaskEvidenceService.RETENTION_LOCK); + when(locks.tryLock(eq(TaskEvidenceService.RETENTION_LOCK), any())).thenReturn(null); + + service().purgeExpired(); + + assertEquals(before + 1, LockSkipWitness.countOf(TaskEvidenceService.RETENTION_LOCK), + "锁跳过必须有 [lock-skip] 可聚合出口,静默 return 就是 P1-7 的另一半"); + verifyNoInteractions(mapper, storage); + } + + @Test + void 单个对象删除失败不堵后面的行() { + when(locks.tryLock(eq(TaskEvidenceService.RETENTION_LOCK), any())) + .thenReturn(mock(DistributedJobLockService.LockHandle.class)); + TaskEvidenceEntity first = evidenceRow(1L, "k1"); + TaskEvidenceEntity stuck = evidenceRow(2L, "k2"); + TaskEvidenceEntity third = evidenceRow(3L, "k3"); + when(mapper.selectList(any())).thenReturn(List.of(first, stuck, third), List.of()); + doThrow(new RuntimeException("oss unavailable")).when(storage).delete("k2"); + when(mapper.deleteById(anyLong())).thenReturn(1); + + service().purgeExpired(); + + verify(mapper).deleteById(1L); + verify(mapper, never()).deleteById(2L); + verify(mapper).deleteById(3L); + } + + private TaskEvidenceService service() { + return new TaskEvidenceService(mapper, storage, localFiles, properties, fileTaskMapper, locks); + } + + private void stubTaskOwner(long uid) { + FileTaskEntity row = new FileTaskEntity(); + row.setId(Long.parseLong(TASK_ID)); + row.setUserId(uid); + when(fileTaskMapper.selectById(Long.parseLong(TASK_ID))).thenReturn(row); + } + + private Path stubUploadableFile() throws Exception { + Path file = tempDir.resolve("evidence.json"); + Files.writeString(file, "{\"traceback\": \"boom\"}"); + when(localFiles.findLocalSourceFile("evidence.json")).thenReturn(file.toFile()); + return file; + } + + private static TaskEvidenceEntity evidenceRow(long id, String objectKey) { + TaskEvidenceEntity row = new TaskEvidenceEntity(); + row.setId(id); + row.setTaskId(TASK_ID); + row.setObjectKey(objectKey); + return row; + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/softwareversion/service/SoftwareVersionServiceChunkConfirmTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/softwareversion/service/SoftwareVersionServiceChunkConfirmTest.java new file mode 100644 index 000000000..0b516992f --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/softwareversion/service/SoftwareVersionServiceChunkConfirmTest.java @@ -0,0 +1,145 @@ +package com.nanri.aiimage.modules.softwareversion.service; + +import com.baomidou.mybatisplus.core.MybatisConfiguration; +import com.baomidou.mybatisplus.core.metadata.TableInfoHelper; +import com.nanri.aiimage.modules.file.service.LocalFileStorageService; +import com.nanri.aiimage.modules.file.service.oss.OssStorageService; +import com.nanri.aiimage.modules.softwareversion.mapper.SoftwareVersionMapper; +import com.nanri.aiimage.modules.softwareversion.model.entity.SoftwareVersionEntity; +import org.apache.ibatis.builder.MapperBuilderAssistant; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.ArgumentCaptor; +import org.springframework.test.util.ReflectionTestUtils; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * 版本包「分片拼装件 → confirm 搬桶落库」这条通道的契约(CR 第二轮 P1-8)。 + * + *

新重载 {@code confirmSoftwareVersion(version, fileKey)} 写好之后长期没有任何执行证据 + * (后端整量绿也覆盖不到没被调用的分支)。幂等命中、临时件清理、缺件拒绝这三条是发版 + * 判据:任何一条破掉,要么重试会插出双版本行,要么临时件永远堆在盘上。 + */ +class SoftwareVersionServiceChunkConfirmTest { + + private final SoftwareVersionMapper versionMapper = mock(SoftwareVersionMapper.class); + private final OssStorageService ossStorageService = mock(OssStorageService.class); + private final LocalFileStorageService localFiles = mock(LocalFileStorageService.class); + + @TempDir + Path tempDir; + + @BeforeAll + static void initializeMybatisMetadata() { + TableInfoHelper.initTableInfo( + new MapperBuilderAssistant(new MybatisConfiguration(), ""), SoftwareVersionEntity.class); + } + + private SoftwareVersionService service() { + SoftwareVersionService service = new SoftwareVersionService(versionMapper, ossStorageService); + ReflectionTestUtils.setField(service, "localFileStorageService", localFiles); + return service; + } + + @Test + void 分片确认搬桶落库并标记最新且临时件清掉() throws Exception { + Path zip = writeZip("4.0.69.zip"); + when(localFiles.findLocalSourceFile("chunk-abc")).thenReturn(zip.toFile()); + when(ossStorageService.uploadSoftwareVersionPackage(any(), any())) + .thenReturn("https://oss.aishufu.top/client/nanri-image/versions/4.0.69.zip"); + when(versionMapper.insert(any(SoftwareVersionEntity.class))).thenAnswer(invocation -> { + ((SoftwareVersionEntity) invocation.getArgument(0)).setId(300L); + return 1; + }); + when(versionMapper.selectById(300L)).thenReturn(null); + + service().confirmSoftwareVersion("4.0.69", "chunk-abc"); + + ArgumentCaptor captor = ArgumentCaptor.forClass(SoftwareVersionEntity.class); + verify(versionMapper).insert(captor.capture()); + assertEquals("4.0.69", captor.getValue().getVersion()); + assertNotEquals("", captor.getValue().getSha256(), "落库必须带拼装件的 sha256"); + verify(versionMapper).markLatestById(300L); + assertFalse(Files.exists(zip), "确认成功后临时拼装件必须删掉,否则每次重传都堆一份"); + } + + @Test + void 同版本重复确认命中幂等不重复搬桶() throws Exception { + Path zip = writeZip("4.0.69.zip"); + when(localFiles.findLocalSourceFile("chunk-abc")).thenReturn(zip.toFile()); + SoftwareVersionEntity existing = new SoftwareVersionEntity(); + existing.setId(9L); + existing.setVersion("4.0.69"); + when(versionMapper.selectOne(any())).thenReturn(existing); + + service().confirmSoftwareVersion("4.0.69", "chunk-abc"); + + verify(ossStorageService, never()).uploadSoftwareVersionPackage(any(), any()); + verify(versionMapper, never()).insert(any(SoftwareVersionEntity.class)); + verify(versionMapper, never()).markLatestById(any()); + assertFalse(Files.exists(zip), "幂等命中也要把临时件清掉"); + } + + @Test + void 拼装件缺失必须拒绝且不落库() { + when(localFiles.findLocalSourceFile("chunk-gone")).thenReturn(null); + + assertThrows(RuntimeException.class, + () -> service().confirmSoftwareVersion("4.0.69", "chunk-gone")); + verify(versionMapper, never()).insert(any(SoftwareVersionEntity.class)); + } + + @Test + void 非zip拼装件必须拒绝() throws Exception { + Path notZip = writeTempFile("payload.txt", "not a zip"); + when(localFiles.findLocalSourceFile("chunk-abc")).thenReturn(notZip.toFile()); + + assertThrows(RuntimeException.class, + () -> service().confirmSoftwareVersion("4.0.69", "chunk-abc")); + verify(ossStorageService, never()).uploadSoftwareVersionPackage(any(), any()); + } + + @Test + void 零字节拼装件必须拒绝() throws Exception { + Path empty = Files.createFile(tempDir.resolve("4.0.69.zip")); + when(localFiles.findLocalSourceFile("chunk-abc")).thenReturn(empty.toFile()); + + assertThrows(RuntimeException.class, + () -> service().confirmSoftwareVersion("4.0.69", "chunk-abc")); + assertTrue(Files.exists(empty), "拒绝分支不许动调用方的临时件(排查要用)"); + } + + /** P2-4 起 confirm 会读 zip 中央目录,测试夹具必须是真 zip 而不是 PK 魔数文本。 */ + private Path writeZip(String name) throws Exception { + Path file = tempDir.resolve(name); + try (ZipOutputStream out = new ZipOutputStream(Files.newOutputStream(file))) { + out.putNextEntry(new ZipEntry("readme.txt")); + out.write("package".getBytes(StandardCharsets.UTF_8)); + out.closeEntry(); + } + return file; + } + + private Path writeTempFile(String name, String content) throws Exception { + Path file = tempDir.resolve(name); + Files.writeString(file, content, StandardCharsets.UTF_8); + return file; + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/mapper/FileTaskMapperChannelSqlTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/mapper/FileTaskMapperChannelSqlTest.java new file mode 100644 index 000000000..183a3db29 --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/mapper/FileTaskMapperChannelSqlTest.java @@ -0,0 +1,157 @@ +package com.nanri.aiimage.modules.task.mapper; + +import com.baomidou.mybatisplus.core.MybatisConfiguration; +import org.apache.ibatis.builder.xml.XMLMapperBuilder; +import org.apache.ibatis.io.Resources; +import org.apache.ibatis.mapping.MappedStatement; +import org.apache.ibatis.session.Configuration; +import org.junit.jupiter.api.Test; + +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.LocalDateTime; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * V183 采集通道的谓词形状(CR 第二轮 P1-1)。 + * + *

为什么钉 SQL 而不是钉行为:这两条查询的执行方判定必须在 {@code WHERE} 里、 + * 在 {@code LIMIT} 之前。写成「先 limit 取页、回 Java 再 {@code filter(isLocal())}」时, + * 混通道排队里只要较老的 N 条(出厂 {@code client-task-pull.limit=5})是 ONLINE, + * 本机通道的单本轮及以后都领不到——不是延后,是排到队尾之前永久饿死。 + * 而 mock 掉 {@code selectList} 的返回让「取页与过滤的先后」在用例里根本看不见, + * 这就是它活过后端整量的原因(现网 {@code enabled=false} 也没能掩盖住)。 + * 这里用真 {@code XMLMapperBuilder} 解析、向真 {@code BoundSql} 断言,判据= + * 把谓词挪出 {@code WHERE} 或挪到 {@code LIMIT} 之后,本格直接红。 + * + *

没有数据库也成立的另一面:{@code ?} 的个数与顺序被钉死,抽掉通道谓词就少两个占位符。 + */ +class FileTaskMapperChannelSqlTest { + + private static final String XML = "mapper/task/FileTaskMapper.xml"; + private static final String NS = "com.nanri.aiimage.modules.task.mapper.FileTaskMapper"; + private static final List MODULE_TYPES = List.of("APPEARANCE_PATENT", "SIMILAR_ASIN"); + private static final List LOCAL_DEFAULTS = List.of("APPEARANCE_PATENT", "SIMILAR_ASIN"); + + @Test + void 客户端拉取的通道谓词在WHERE里且发生在LIMIT之前() throws Exception { + String sql = render("selectLocalChannelPendingTasks", clientParams(2)); + + int at = sql.indexOf("CASE WHEN collect_channel IN ('LOCAL', 'ONLINE')"); + assertTrue(at > 0, () -> "通道判定不在 SQL 里(回到 Java filter 就是 P1-1 复发): " + sql); + int limit = sql.lastIndexOf("LIMIT ?"); + assertTrue(limit > at, () -> "通道谓词必须出现在 LIMIT 之前,否则取到的那一页里没有本机单: " + sql); + assertTrue(sql.indexOf("ORDER BY created_at ASC") < limit, + () -> "排序必须在 LIMIT 之前,否则取页不是「最早创建的本机单」: " + sql); + assertEquals("SELECT id, module_type, collect_channel, created_at FROM biz_file_task" + + " WHERE user_id = ? AND status = ? AND module_type IN (?, ?) AND created_at < ?" + + " AND CASE WHEN collect_channel IN ('LOCAL', 'ONLINE') THEN collect_channel" + + " WHEN module_type IN (?, ?) THEN 'LOCAL' ELSE 'ONLINE' END = 'LOCAL'" + + " ORDER BY created_at ASC LIMIT ?", sql, + "取页形状变了:通道谓词必须与 status/模块/时间窗口同在 WHERE 里,且 LIMIT 之后没有任何过滤面"); + } + + @Test + void 内部认领把本机让位做成SQL侧OR窗口而不是Java二次过滤() throws Exception { + String sql = render("selectInternalClaimCandidates", internalParams(2)); + + assertTrue(sql.contains("(NOT (CASE") && sql.contains("= 'LOCAL') OR created_at <= ?)"), + () -> "让位窗口不在 SQL 里:整页都是「本机通道且没等满兜底窗口」时,本轮线上单零吞吐: " + sql); + assertTrue(sql.indexOf("created_at < ?") < sql.indexOf("created_at <= ?"), + () -> "基础等待窗口与兜底窗口必须都在 LIMIT 之前: " + sql); + assertTrue(sql.lastIndexOf("created_at <= ?") < sql.lastIndexOf("LIMIT ?"), + () -> "兜底窗口谓词必须发生在 LIMIT 之前: " + sql); + } + + @Test + void 通道判定用CASE兜模块默认而不是裸等值() throws Exception { + String sql = render("selectLocalChannelPendingTasks", clientParams(2)); + + // 裸等值(collect_channel = 'LOCAL' 或 NOT (collect_channel = 'LOCAL'))在列为 NULL 时吃三值 + // 逻辑变成 UNKNOWN:被默认兜成 ONLINE 的模块(COLLECT_DATA 等)会在 NOT 之后永久领不到。 + assertFalse(sql.contains("collect_channel = ?"), () -> "通道值被参数化后 CASE 的兜法就散了: " + sql); + assertFalse(sql.contains("collect_channel = 'LOCAL' OR"), + () -> "OR 形状的裸等值谓词对 NULL 列失明: " + sql); + assertTrue(sql.contains("ELSE 'ONLINE' END"), () -> "CASE 缺 ELSE=不可识别值不再兜默认: " + sql); + } + + @Test + void 取页投影只有认领需要的四列() throws Exception { + String sql = render("selectLocalChannelPendingTasks", clientParams(2)); + assertTrue(sql.startsWith("SELECT id, module_type, collect_channel, created_at FROM biz_file_task"), + () -> "投影列清单变了(整行拉回来会把 result_json 这类大列带进兜底轮询): " + sql); + } + + @Test + void 占位符数与两条语句的谓词形状逐一对撞() throws Exception { + // 客户端:user_id + status + 2 模块 + cutoff + 2 默认模块 + limit = 8 + assertEquals(8, questionMarks("selectLocalChannelPendingTasks", clientParams(2)), + "少 2 个就是通道谓词被抽掉了(P1-1 复发),多 2 个是模块清单被写了两遍"); + // 内部认领:status + 2 模块 + cutoff + 2 默认模块 + localCutoff + limit = 8 + assertEquals(8, questionMarks("selectInternalClaimCandidates", internalParams(2)), + "内部认领的兜底窗口比客户端多一条 cutoff、少一个 user_id"); + } + + @Test + void V184补齐两条取页路径的索引并明写通道列不进索引() throws Exception { + Path migration = moduleRoot().resolve("src/main/resources/db/V184__collect_pull_and_evidence_indexes.sql"); + assertTrue(Files.exists(migration), "V184 缺失:P1-1 的下推查询没有索引承接,P2-1 的保留期扫描仍全表"); + String sql = Files.readString(migration, StandardCharsets.UTF_8); + assertTrue(sql.contains("ADD INDEX idx_biz_file_task_status_module_created (status, module_type, created_at)"), + () -> "认领取页索引的列组合变了: " + sql); + assertTrue(sql.contains("ADD INDEX idx_task_evidence_created_at (created_at)"), + () -> "证据保留期是单列 created_at 谓词,前导列不符就还是全表扫: " + sql); + assertFalse(sql.contains("idx_task_collect_channel"), + "通道列有意不进索引:CASE 表达式谓词不吃索引,把 collect_channel 夹在 module_type 与" + + " created_at 之间反而打断 ORDER BY created_at 的索引有序性"); + } + + private static int questionMarks(String id, Map params) throws Exception { + return (int) render(id, params).chars().filter(c -> c == '?').count(); + } + + private static String render(String id, Map params) throws Exception { + MappedStatement statement = parse().getMappedStatement(NS + "." + id); + // MyBatis 把 foreach 展开成 `IN ( ? , ? )`,这里收成 `IN (?,?)` 好让判据读得下去。 + return statement.getBoundSql(params).getSql().replaceAll("\\s+", " ").trim() + .replace("( ", "(").replace(" )", ")").replace(" ,", ","); + } + + private static Map clientParams(int modules) { + return Map.of("userId", 1095L, + "status", "PENDING", + "moduleTypes", MODULE_TYPES.subList(0, modules), + "localDefaultModules", LOCAL_DEFAULTS, + "cutoff", LocalDateTime.now().minusMinutes(5), + "limit", 5); + } + + private static Map internalParams(int modules) { + return Map.of("status", "PENDING", + "moduleTypes", MODULE_TYPES.subList(0, modules), + "localDefaultModules", LOCAL_DEFAULTS, + "cutoff", LocalDateTime.now().minusMinutes(2), + "localCutoff", LocalDateTime.now().minusMinutes(30), + "limit", 5); + } + + private static Configuration parse() throws Exception { + MybatisConfiguration configuration = new MybatisConfiguration(); + try (InputStream in = Resources.getResourceAsStream(XML)) { + new XMLMapperBuilder(in, configuration, XML, configuration.getSqlFragments()).parse(); + } + return configuration; + } + + private static Path moduleRoot() { + Path here = Path.of("").toAbsolutePath(); + return Files.isDirectory(here.resolve("src/main/resources")) ? here : here.getParent(); + } +} diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/model/CollectChannelTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/model/CollectChannelTest.java index 1503097a3..3d5cc628f 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/task/model/CollectChannelTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/model/CollectChannelTest.java @@ -3,9 +3,12 @@ package com.nanri.aiimage.modules.task.model; import com.nanri.aiimage.config.CollectRoutingProperties; import org.junit.jupiter.api.Test; +import java.util.List; + import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * V183 任务级采集通道的判据(建单写入、认领读取、脏值兜底)。 @@ -27,6 +30,24 @@ class CollectChannelTest { assertFalse(CollectChannel.defaultFor(" ").isLocal()); } + /** + * P1-1 把通道判定下推进 SQL 之后,「哪些模块默认本机」这份清单被喂进 mapper 的 CASE。 + * 判据=清单内必被 {@link CollectChannel#defaultFor} 读作本机、清单外必不读作本机。 + * 谁在 defaultFor 里另加一个模块而忘了同步清单,SQL 就会把那模块的历史单兜成线上: + * 界面显示「本机采集」、客户端却永远领不到。 + */ + @Test + void 本机默认模块清单与defaultFor同源() { + assertEquals(List.of(CollectRoutingProperties.MODULE_APPEARANCE_PATENT, + CollectRoutingProperties.MODULE_SIMILAR_ASIN), CollectChannel.localDefaultModuleTypes()); + for (String module : CollectChannel.localDefaultModuleTypes()) { + assertTrue(CollectChannel.defaultFor(module).isLocal(), module + " 在清单里却不被判本机"); + } + for (String module : List.of("DEDUPE", "COLLECT_DATA", "SHOP_DATA_CRAWL", "PUBLISH")) { + assertFalse(CollectChannel.defaultFor(module).isLocal(), module + " 不在清单里却读成本机"); + } + } + @Test void 模块默认大小写与空白不影响判定() { assertEquals(CollectChannel.LOCAL, CollectChannel.defaultFor("similar_asin")); diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskClientPullServiceTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskClientPullServiceTest.java index 350de57fa..208004a25 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskClientPullServiceTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskClientPullServiceTest.java @@ -9,6 +9,7 @@ import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; import com.baomidou.mybatisplus.core.metadata.TableInfoHelper; import com.nanri.aiimage.config.InstanceMetadata; import com.nanri.aiimage.modules.task.mapper.FileTaskMapper; +import com.nanri.aiimage.modules.task.model.CollectChannel; import com.nanri.aiimage.modules.task.model.entity.FileTaskEntity; import com.nanri.aiimage.modules.task.model.vo.TaskClientPullVo; import com.nanri.aiimage.modules.task.spi.ClientTaskPullSpi; @@ -29,7 +30,9 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -68,7 +71,7 @@ class TaskClientPullServiceTest { @Test void 认领成功后返回载荷并写_RUNNING_与_owner() { FileTaskEntity candidate = task(11L, "SIMILAR_ASIN"); - when(fileTaskMapper.selectList(any())).thenReturn(List.of(candidate)); + stubPullCandidates(candidate); when(fileTaskMapper.update(isNull(), any())).thenReturn(1); when(fileTaskMapper.selectById(11L)).thenReturn(candidate); StubHandler handler = new StubHandler("SIMILAR_ASIN", payload("similar-asin-run")); @@ -88,17 +91,16 @@ class TaskClientPullServiceTest { assertTrue(params.containsValue("RUNNING"), "认领必须写入 RUNNING"); assertTrue(sqlSetOf(captor.getValue()).contains("owner_instance_id"), "认领必须接管归属实例"); - ArgumentCaptor> selectCaptor = wrapperCaptor(); - verify(fileTaskMapper).selectList(selectCaptor.capture()); - Map selectParams = paramsOf(selectCaptor.getValue()); - assertTrue(selectParams.containsValue("PENDING"), "候选只挑 PENDING"); - assertTrue(selectParams.containsValue(USER_ID), "候选限定任务所属用户"); - assertTrue(selectCaptor.getValue().getSqlSegment().contains("created_at"), "候选要有创建时间窗口"); + // 取页形状(通道谓词在 WHERE、LIMIT 之前)由 FileTaskMapperChannelSqlTest 向真 BoundSql 断言; + // 这里钉「调用的是哪条具名查询、传了什么」,两者合起来才挡住 P1-1 复发。 + verify(fileTaskMapper).selectLocalChannelPendingTasks(eq(USER_ID), eq("PENDING"), + eq(List.of("SIMILAR_ASIN", "COLLECT_DATA")), eq(CollectChannel.localDefaultModuleTypes()), + any(LocalDateTime.class), eq(5)); } @Test void 页面抢先认领后不再返回任务() { - when(fileTaskMapper.selectList(any())).thenReturn(List.of(task(11L, "SIMILAR_ASIN"))); + stubPullCandidates(task(11L, "SIMILAR_ASIN")); when(fileTaskMapper.update(isNull(), any())).thenReturn(0); StubHandler handler = new StubHandler("SIMILAR_ASIN", payload("similar-asin-run")); @@ -112,7 +114,7 @@ class TaskClientPullServiceTest { @Test void 组装不出载荷时任务标失败() { FileTaskEntity candidate = task(11L, "SIMILAR_ASIN"); - when(fileTaskMapper.selectList(any())).thenReturn(List.of(candidate)); + stubPullCandidates(candidate); when(fileTaskMapper.update(isNull(), any())).thenReturn(1); when(fileTaskMapper.selectById(11L)).thenReturn(candidate); @@ -127,7 +129,7 @@ class TaskClientPullServiceTest { @Test void 组装抛异常时任务标失败且不抛出() { FileTaskEntity candidate = task(11L, "SIMILAR_ASIN"); - when(fileTaskMapper.selectList(any())).thenReturn(List.of(candidate)); + stubPullCandidates(candidate); when(fileTaskMapper.update(isNull(), any())).thenReturn(1); when(fileTaskMapper.selectById(11L)).thenReturn(candidate); StubHandler broken = new StubHandler("SIMILAR_ASIN", null, new IllegalStateException("boom")); @@ -142,7 +144,7 @@ class TaskClientPullServiceTest { @Test void 白名单模块未实现兜底时跳过且不改状态() { - when(fileTaskMapper.selectList(any())).thenReturn(List.of(task(21L, "COLLECT_DATA"))); + stubPullCandidates(task(21L, "COLLECT_DATA")); List result = service(new StubHandler("SIMILAR_ASIN", payload("similar-asin-run"))) .pullPendingTasks(USER_ID); @@ -190,86 +192,97 @@ class TaskClientPullServiceTest { } } - // ------------------------------------------------------------ 本地采集路由(2026-10-29) + // ---------------------------------------------------- 本地采集路由(2026-10-29 建档,2026-10-31 下推 SQL) + /** + * 客户端拉取只走带通道谓词的具名查询,服务侧不再挑第二遍。 + * + *

这里喂一条通道写着 ONLINE 的行——真 SQL 不会返回它——为的是钉住「Java 里没有第二道 + * {@code filter(isLocal())}」。P1-1 的原形状就是先 {@code limit 5} 取页再过滤:那一道过滤 + * 看不见 limit,也就没人替它保证「先筛后取页」,混通道时较老的 N 条是线上单就把本机单永久 + * 挡在队尾之后。谓词本身的位置由 {@code FileTaskMapperChannelSqlTest} 向真 BoundSql 钉。 + */ @Test - void 客户端兜底拉取不领线上通道的单() { - FileTaskEntity online = task(11L, "SIMILAR_ASIN", 10L); - online.setCollectChannel("ONLINE"); - when(fileTaskMapper.selectList(any())).thenReturn(List.of(online)); + void 客户端拉取把通道判定整个交给SQL不做二次过滤() { + FileTaskEntity row = task(11L, "SIMILAR_ASIN", 10L); + row.setCollectChannel("ONLINE"); + stubPullCandidates(row); + when(fileTaskMapper.update(isNull(), any())).thenReturn(1); + when(fileTaskMapper.selectById(11L)).thenReturn(row); List result = serviceWithRouting(defaultRouting(), new StubHandler("SIMILAR_ASIN", payload("similar-asin-run"))).pullPendingTasks(USER_ID); - assertTrue(result.isEmpty(), "这一单选了线上采集,客户端领了就是抢跑"); - verify(fileTaskMapper, never()).update(isNull(), any()); + assertEquals(1, result.size(), "候选页由 SQL 侧谓词筛好,服务再判一遍就是 P1-1 复发"); } + /** + * 内部认领同样把让位判定交给 SQL:喂一条「还在让位窗口内的本机单」,服务必须照领。 + */ @Test - void 客户端兜底拉取照领本机通道的单() { - FileTaskEntity local = task(11L, "SIMILAR_ASIN", 10L); - local.setCollectChannel("LOCAL"); - when(fileTaskMapper.selectList(any())).thenReturn(List.of(local)); - when(fileTaskMapper.update(isNull(), any())).thenReturn(1); - when(fileTaskMapper.selectById(11L)).thenReturn(local); - - List result = serviceWithRouting(defaultRouting(), - new StubHandler("SIMILAR_ASIN", payload("similar-asin-run"))).pullPendingTasks(USER_ID); - - assertEquals(1, result.size(), "本机通道的单必须能被在线客户端领走"); - } - - @Test - void 内部认领对本机通道让位到兜底窗口() { + void 内部认领不在Java里做让位过滤() { com.nanri.aiimage.config.CollectRoutingProperties routing = new com.nanri.aiimage.config.CollectRoutingProperties(); routing.setLinuxFallbackMinutes(30L); - FileTaskEntity localTask = task(11L, "SIMILAR_ASIN", 10L); - localTask.setCollectChannel("LOCAL"); - when(fileTaskMapper.selectList(any())).thenReturn(List.of(localTask)); - - List result = serviceWithRouting(routing, - new StubHandler("SIMILAR_ASIN", payload("similar-asin-run"))) - .pullPendingTasksInternal(List.of("SIMILAR_ASIN")); - - assertTrue(result.isEmpty(), "本机通道的单在兜底窗口内必须让位给本地客户端"); - verify(fileTaskMapper, never()).update(isNull(), any()); - } - - @Test - void 内部认领超过兜底窗口后接管本机通道的单() { - com.nanri.aiimage.config.CollectRoutingProperties routing = - new com.nanri.aiimage.config.CollectRoutingProperties(); - routing.setLinuxFallbackMinutes(30L); - FileTaskEntity candidate = task(11L, "SIMILAR_ASIN", 40L); - candidate.setCollectChannel("LOCAL"); - when(fileTaskMapper.selectList(any())).thenReturn(List.of(candidate)); + FileTaskEntity row = task(21L, "COLLECT_DATA", 10L); + row.setCollectChannel("LOCAL"); + stubInternalCandidates(row); when(fileTaskMapper.update(isNull(), any())).thenReturn(1); - when(fileTaskMapper.selectById(11L)).thenReturn(candidate); - - List result = serviceWithRouting(routing, - new StubHandler("SIMILAR_ASIN", payload("similar-asin-run"))) - .pullPendingTasksInternal(List.of("SIMILAR_ASIN")); - - assertEquals(1, result.size(), "本机长时间没人领,Linux 必须兜底接管"); - } - - @Test - void 内部认领对线上通道不受兜底窗口影响() { - com.nanri.aiimage.config.CollectRoutingProperties routing = - new com.nanri.aiimage.config.CollectRoutingProperties(); - routing.setLinuxFallbackMinutes(30L); - FileTaskEntity candidate = task(21L, "COLLECT_DATA", 10L); - candidate.setCollectChannel("ONLINE"); - when(fileTaskMapper.selectList(any())).thenReturn(List.of(candidate)); - when(fileTaskMapper.update(isNull(), any())).thenReturn(1); - when(fileTaskMapper.selectById(21L)).thenReturn(candidate); + when(fileTaskMapper.selectById(21L)).thenReturn(row); List result = serviceWithRouting(routing, new StubHandler("COLLECT_DATA", payload("collect-data-run"))) .pullPendingTasksInternal(List.of("COLLECT_DATA")); - assertEquals(1, result.size(), "选了线上的单子不能被本地优先的窗口拖住"); + assertEquals(1, result.size(), "让位窗口是 SQL 的 OR 分支,不是取页之后的 Java 过滤"); + } + + /** + * 两道窗口分别下传:基础等待窗口(给「建完就想删」留余地)与本机兜底窗口(Linux 接管点)。 + * 兜底窗口配得比基础窗口还小时取基础窗口——这条夹逼原先在 Java 里,下推后仍得由服务算好。 + */ + @Test + void 内部认领下传两道cutoff且兜底窗口不越基础窗口() { + com.nanri.aiimage.config.CollectRoutingProperties narrow = + new com.nanri.aiimage.config.CollectRoutingProperties(); + narrow.setLinuxFallbackMinutes(1L); + + serviceWithRouting(narrow, new StubHandler("SIMILAR_ASIN", payload("similar-asin-run"))) + .pullPendingTasksInternal(List.of("SIMILAR_ASIN")); + + ArgumentCaptor cutoff = ArgumentCaptor.forClass(LocalDateTime.class); + ArgumentCaptor localCutoff = ArgumentCaptor.forClass(LocalDateTime.class); + verify(fileTaskMapper).selectInternalClaimCandidates(eq("PENDING"), eq(List.of("SIMILAR_ASIN")), + eq(CollectChannel.localDefaultModuleTypes()), cutoff.capture(), localCutoff.capture(), eq(5)); + assertEquals(cutoff.getValue(), localCutoff.getValue(), + "兜底窗口 1 分钟短于基础窗口 2 分钟时按 2 分钟算,否则「建完就能删」的余地被越过"); + } + + @Test + void 兜底窗口长于基础窗口时按自身下传() { + com.nanri.aiimage.config.CollectRoutingProperties wide = + new com.nanri.aiimage.config.CollectRoutingProperties(); + wide.setLinuxFallbackMinutes(30L); + + serviceWithRouting(wide, new StubHandler("SIMILAR_ASIN", payload("similar-asin-run"))) + .pullPendingTasksInternal(List.of("SIMILAR_ASIN")); + + ArgumentCaptor cutoff = ArgumentCaptor.forClass(LocalDateTime.class); + ArgumentCaptor localCutoff = ArgumentCaptor.forClass(LocalDateTime.class); + verify(fileTaskMapper).selectInternalClaimCandidates(eq("PENDING"), eq(List.of("SIMILAR_ASIN")), + eq(CollectChannel.localDefaultModuleTypes()), cutoff.capture(), localCutoff.capture(), eq(5)); + assertTrue(localCutoff.getValue().isBefore(cutoff.getValue()), + "30 分钟的让位窗口必须比 2 分钟的基础窗口更早,否则 Linux 会抢在本地客户端前面双跑"); + } + + private void stubPullCandidates(FileTaskEntity... rows) { + when(fileTaskMapper.selectLocalChannelPendingTasks(any(), any(), any(), any(), any(), anyInt())) + .thenReturn(List.of(rows)); + } + + private void stubInternalCandidates(FileTaskEntity... rows) { + when(fileTaskMapper.selectInternalClaimCandidates(any(), any(), any(), any(), any(), anyInt())) + .thenReturn(List.of(rows)); } private TaskClientPullService service(ClientTaskPullSpi... handlers) { diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskFileJobTerminalTaskGuardTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskFileJobTerminalTaskGuardTest.java index 66afdeeb8..3affe0f4b 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskFileJobTerminalTaskGuardTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskFileJobTerminalTaskGuardTest.java @@ -21,7 +21,7 @@ import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.isNull; @@ -76,29 +76,33 @@ class TaskFileJobTerminalTaskGuardTest { } @Test - void claimRunningRejectsJobWhoseTaskIsFailed() { - TaskFileJobEntity failedTaskJob = job(11L, 500L); - when(taskFileJobMapper.selectById(11L)).thenReturn(failedTaskJob); + void claimRunningStillDeliversWhenParentTaskIsTerminal() { + TaskFileJobEntity terminalTaskJob = job(11L, 500L); + TaskFileJobEntity running = job(11L, 500L); + running.setStatus("RUNNING"); + when(taskFileJobMapper.selectById(11L)).thenReturn(terminalTaskJob, running); when(taskFileJobMapper.selectFileTaskStatus(500L)).thenReturn("FAILED"); when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1); - assertNull(service.claimRunning(11L), "终态任务的作业不得被 claim 成 RUNNING"); + TaskFileJobEntity claimed = service.claimRunning(11L); + + assertNotNull(claimed, "P1-13:父任务终态也要放行出表作业"); + assertEquals("RUNNING", claimed.getStatus()); @SuppressWarnings("unchecked") ArgumentCaptor> captor = ArgumentCaptor.forClass(LambdaUpdateWrapper.class); verify(taskFileJobMapper, times(1)).update(isNull(), captor.capture()); LambdaUpdateWrapper wrapper = captor.getValue(); - assertTrue(hasParam(wrapper, "FAILED"), "必须把作业拉成 FAILED,下一轮扫描才捞不到"); - assertTrue(hasParam(wrapper, 5), "retry_count 必须一次拉满到上限"); - assertFalse(hasParam(wrapper, "RUNNING"), "不得再翻转成 RUNNING"); + assertTrue(hasParam(wrapper, "RUNNING"), "终态父任务不再把出表作业拉成 FAILED"); + assertFalse(hasParam(wrapper, "FAILED"), "终态只停心跳续期,不改写给出表作业"); assertTrue(wrapper.getSqlSet().contains("terminal_callback_at"), - "terminal_callback_at 必须显式清空:留给兜底扫描出部分结果,失败任务仍可下载"); - assertTrue(hasParam(wrapper, null), "terminal_callback_at 置空后兜底扫描才捞得到这一行"); + "正常认领必须清空 terminal_callback_at,失败兜底扫描才捞得到"); + assertTrue(hasParam(wrapper, null)); } @Test - void claimCandidatesSkipsTerminalTaskAndKeepsActiveOne() { + void claimCandidatesClaimsBothTerminalAndActiveTasks() { TaskFileJobEntity terminal = job(21L, 500L); TaskFileJobEntity active = job(22L, 501L); when(taskFileJobMapper.selectList(any(LambdaQueryWrapper.class))) @@ -106,23 +110,25 @@ class TaskFileJobTerminalTaskGuardTest { when(taskFileJobMapper.selectFileTaskStatus(500L)).thenReturn("FAILED"); when(taskFileJobMapper.selectFileTaskStatus(501L)).thenReturn("RUNNING"); when(taskFileJobMapper.update(isNull(), any(LambdaUpdateWrapper.class))).thenReturn(1, 1); + TaskFileJobEntity claimedTerminal = job(21L, 500L); + claimedTerminal.setStatus("RUNNING"); TaskFileJobEntity claimedActive = job(22L, 501L); claimedActive.setStatus("RUNNING"); - when(taskFileJobMapper.selectBatchIds(any())).thenReturn(List.of(claimedActive)); + when(taskFileJobMapper.selectBatchIds(any())).thenReturn(List.of(claimedTerminal, claimedActive)); List claimed = service.claimRunnableJobs(10); - assertEquals(1, claimed.size(), "终态任务的 job 不入结果集"); - assertEquals(22L, claimed.get(0).getId()); + assertEquals(2, claimed.size(), "P1-13:终态任务的出表作业与活跃任务一样可认领"); + assertEquals(21L, claimed.get(0).getId()); + assertEquals(22L, claimed.get(1).getId()); @SuppressWarnings("unchecked") ArgumentCaptor> captor = ArgumentCaptor.forClass(LambdaUpdateWrapper.class); verify(taskFileJobMapper, times(2)).update(isNull(), captor.capture()); - assertTrue(hasParam(captor.getAllValues().get(0), "FAILED"), "第一笔是终态收尾"); - assertTrue(hasParam(captor.getAllValues().get(1), "RUNNING"), "第二笔才是正常 claim"); - assertTrue(captor.getAllValues().get(0).getSqlSet().contains("terminal_callback_at"), - "终态作业必须留给兜底扫描,不能提前打终态回调"); + assertTrue(hasParam(captor.getAllValues().get(0), "RUNNING"), "终态任务的 job 也要正常 claim"); + assertTrue(hasParam(captor.getAllValues().get(1), "RUNNING")); + assertFalse(hasParam(captor.getAllValues().get(0), "FAILED"), "不得再把终态任务的 job 判死"); } @Test diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorkerStuckScanTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorkerStuckScanTest.java index 651ca0365..af63a4220 100644 --- a/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorkerStuckScanTest.java +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/task/service/TaskResultFileJobWorkerStuckScanTest.java @@ -425,7 +425,7 @@ class TaskResultFileJobWorkerStuckScanTest { worker.finishAfterExecution(job, new java.util.concurrent.atomic.AtomicBoolean(true)); - verify(taskFileJobService).markFailedPermanentAwaitingFallback(job, "父任务已终态,结果作业不再执行"); + verify(taskFileJobService).markFailedPermanentAwaitingFallback(job, "父任务已终态且本次执行未产出结果文件"); } /** diff --git a/backend-java/src/test/java/com/nanri/aiimage/modules/tutorial/service/TutorialPackageServiceChunkConfirmTest.java b/backend-java/src/test/java/com/nanri/aiimage/modules/tutorial/service/TutorialPackageServiceChunkConfirmTest.java new file mode 100644 index 000000000..18b918af9 --- /dev/null +++ b/backend-java/src/test/java/com/nanri/aiimage/modules/tutorial/service/TutorialPackageServiceChunkConfirmTest.java @@ -0,0 +1,111 @@ +package com.nanri.aiimage.modules.tutorial.service; + +import com.baomidou.mybatisplus.core.MybatisConfiguration; +import com.baomidou.mybatisplus.core.metadata.TableInfoHelper; +import com.nanri.aiimage.modules.file.service.LocalFileStorageService; +import com.nanri.aiimage.modules.file.service.oss.OssStorageService; +import com.nanri.aiimage.modules.tutorial.mapper.TutorialPackageMapper; +import com.nanri.aiimage.modules.tutorial.model.entity.TutorialPackageEntity; +import org.apache.ibatis.builder.MapperBuilderAssistant; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.ArgumentCaptor; +import org.springframework.test.util.ReflectionTestUtils; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * 教程包「分片拼装件 → confirm 搬桶落库」的契约(CR 第二轮 P1-8,与版本包同批)。 + * + *

服务端新重载与控制器接线都在,缺的是执行证据:幂等命中(同 fileName+version 重试只 + * 返回既有行)、成功后临时件清理、缺件拒绝。任何一条破掉,重试会重复搬桶或临时件堆盘。 + */ +class TutorialPackageServiceChunkConfirmTest { + + private final TutorialPackageMapper tutorialPackageMapper = mock(TutorialPackageMapper.class); + private final OssStorageService ossStorageService = mock(OssStorageService.class); + private final LocalFileStorageService localFiles = mock(LocalFileStorageService.class); + + @TempDir + Path tempDir; + + @BeforeAll + static void initializeMybatisMetadata() { + TableInfoHelper.initTableInfo( + new MapperBuilderAssistant(new MybatisConfiguration(), ""), TutorialPackageEntity.class); + } + + private TutorialPackageService service() { + TutorialPackageService service = new TutorialPackageService(tutorialPackageMapper, ossStorageService); + ReflectionTestUtils.setField(service, "localFileStorageService", localFiles); + return service; + } + + @Test + void 分片确认搬桶落库且临时件清掉() throws Exception { + Path zip = writeZip("教程包.zip"); + when(localFiles.findLocalSourceFile("chunk-abc")).thenReturn(zip.toFile()); + when(tutorialPackageMapper.selectOne(any())).thenReturn(null); + when(tutorialPackageMapper.insert(any(TutorialPackageEntity.class))).thenReturn(1); + when(tutorialPackageMapper.selectById(any())).thenReturn(null); + + service().confirmTutorialPackage("tutorial/direct.zip", "教程包.zip", "2026.10", "chunk-abc"); + + ArgumentCaptor captor = ArgumentCaptor.forClass(TutorialPackageEntity.class); + verify(tutorialPackageMapper).insert(captor.capture()); + assertEquals("教程包.zip", captor.getValue().getFileName()); + verify(ossStorageService).uploadTutorialPackage(any(), anyString()); + assertFalse(Files.exists(zip), "确认成功后临时拼装件必须删掉"); + } + + @Test + void 同名同版重复确认命中幂等不重复搬桶() throws Exception { + Path zip = writeZip("教程包.zip"); + when(localFiles.findLocalSourceFile("chunk-abc")).thenReturn(zip.toFile()); + TutorialPackageEntity existing = new TutorialPackageEntity(); + existing.setId(11L); + existing.setFileName("教程包.zip"); + when(tutorialPackageMapper.selectOne(any())).thenReturn(existing); + + service().confirmTutorialPackage("tutorial/direct.zip", "教程包.zip", "2026.10", "chunk-abc"); + + verify(ossStorageService, never()).uploadTutorialPackage(any(), anyString()); + verify(tutorialPackageMapper, never()).insert(any(TutorialPackageEntity.class)); + assertFalse(Files.exists(zip), "幂等命中也要把临时件清掉"); + } + + @Test + void 拼装件缺失必须拒绝且不落库() { + when(localFiles.findLocalSourceFile("chunk-gone")).thenReturn(null); + + assertThrows(RuntimeException.class, + () -> service().confirmTutorialPackage("tutorial/direct.zip", "教程包.zip", "2026.10", "chunk-gone")); + verify(tutorialPackageMapper, never()).insert(any(TutorialPackageEntity.class)); + } + + /** P2-4 起 confirm 会读 zip 中央目录,测试夹具必须是真 zip。 */ + private Path writeZip(String name) throws Exception { + Path file = tempDir.resolve(name); + try (ZipOutputStream out = new ZipOutputStream(Files.newOutputStream(file))) { + out.putNextEntry(new ZipEntry("readme.txt")); + out.write("package".getBytes(StandardCharsets.UTF_8)); + out.closeEntry(); + } + return file; + } +} diff --git a/frontend-vue/src/shared/api/upload.ts b/frontend-vue/src/shared/api/upload.ts index 5d16e9d9f..841d7c281 100644 --- a/frontend-vue/src/shared/api/upload.ts +++ b/frontend-vue/src/shared/api/upload.ts @@ -31,6 +31,14 @@ export const CHUNK_UPLOAD_THRESHOLD = UPLOAD_PART_SIZE /** 每片重试次数(不含首次)。 */ const PART_RETRY_LIMIT = 2 +/** + * 单次直传的超时预算(P3-5:这条尺子在本仓只有一个出口):30s 保底 + 每 MB 15s, + * 慢公网传 100MB 视频不再被固定 120s 掐死。桌面桥的浏览器降级直传也走这里。 + */ +export function uploadTimeoutForBytes(bytes: number): number { + return 30_000 + Math.ceil(bytes / (1024 * 1024)) * 15_000 +} + function uploadPartUrl(uploadId: string, index: number): string { return `${JAVA_API_PREFIX}/files/chunk/${encodeURIComponent(uploadId)}/part/${index}` } @@ -98,6 +106,61 @@ function clearStoredSession(key: string): void { } } +/** + * 服务端对「这个会话还在不在」的三种答复:还在(顺带已到片号)/确实没了/问不出来。 + * 第三态是 P1-11 的关键——过去它和前两态被揉进一个 catch,断网=整包作废。 + */ +type ProbeOutcome + = { state: 'ok'; parts: number[] } + | { state: 'gone' } + | { state: 'transient' } + +/** 后端回收会话的原文案(ChunkedUploadService.readMeta);BusinessException 不带 code ⇒ HTTP 200 出。 */ +function messageSaysSessionGone(text: unknown): boolean { + const message = typeof text === 'string' ? text : '' + return ['不存在', '过期'].some((marker) => message.includes(marker)) +} + +/** + * 判据顺序=可信度顺序:404 与后端点名句才清状态;5xx/429/别的业务失败一律瞬态。 + * 拿"问不到"当"没了",就是用一次网关抖动换整包重传。 + */ +function classifyProbe(response: unknown): ProbeOutcome { + const status = (response as { status?: unknown } | null | undefined)?.status + const numeric = typeof status === 'number' ? status : -1 + if (numeric === 404) return { state: 'gone' } + if (numeric >= 500 || numeric === 408 || numeric === 429) return { state: 'transient' } + const body = bodyOf<{ uploaded_parts?: unknown }>(response) + if (body && typeof body === 'object' && body.success === false) { + return messageSaysSessionGone(body.message) ? { state: 'gone' } : { state: 'transient' } + } + if (numeric !== -1 && (numeric < 200 || numeric > 299)) return { state: 'transient' } + const arrived = body?.data?.uploaded_parts + return { state: 'ok', parts: Array.isArray(arrived) ? arrived.map((index) => Number(index)) : [] } +} + +/** + * 探测已达片号(P1-11)。 + * validateStatus 恒真:本仓响应拦截器会把失败折叠成 `new Error(中文文案)`,状态码到不了这里, + * 404/5xx 全靠这一步自己按码分因;能走到 catch 的就只剩网络层失败与超时=铁定瞬态。 + */ +async function probeUploadedParts(uploadId: string): Promise { + try { + const response = await http.get>( + `${JAVA_API_PREFIX}/files/chunk/${encodeURIComponent(uploadId)}`, + { validateStatus: () => true }, + ) + return classifyProbe(response) + } catch (error) { + const status = (error as { response?: { status?: unknown } } | null | undefined)?.response?.status + if (status === 404) return { state: 'gone' } + if (messageSaysSessionGone((error as { message?: unknown } | null | undefined)?.message)) { + return { state: 'gone' } + } + return { state: 'transient' } + } +} + /** 单次 multipart 上传(小文件短链路)。超时按体积给,不再固定 120s。 */ async function singleUpload( file: File, @@ -122,8 +185,8 @@ async function singleUpload( headers: { 'Content-Type': 'multipart/form-data', }, - // 30s 保底 + 每 MB 15s:慢公网传 100MB 视频不再被固定 120s 掐死(S1-F3) - timeout: 30_000 + Math.ceil(file.size / (1024 * 1024)) * 15_000, + // 超时按体积给(见 uploadTimeoutForBytes),不再固定 120s + timeout: uploadTimeoutForBytes(file.size), onUploadProgress: (event) => { if (!onProgress) return const total = event.total || file.size @@ -148,17 +211,16 @@ async function chunkedUpload( let uploadId = readStoredSession(sessionKey, file) let done = new Set() if (uploadId) { - try { - const status = ensureOk(bodyOf<{ uploaded_parts?: unknown[] }>(await http.get( - `${JAVA_API_PREFIX}/files/chunk/${encodeURIComponent(uploadId)}`, - ))) - const arrived = status.data?.uploaded_parts - done = new Set(Array.isArray(arrived) ? arrived.map((index) => Number(index)) : []) - } catch { - // 会话已被服务端回收/过期:只能清掉重开,否则这个件每次都在同一个死会话上失败。 + const probed = await probeUploadedParts(uploadId) + if (probed.state === 'gone') { + // 服务端确实说会话没了(404/「不存在或已过期」):只能清掉重开,否则这个件每次都在同一个死会话上失败。 clearStoredSession(sessionKey) uploadId = '' + } else if (probed.state === 'ok') { + done = new Set(probed.parts) } + // transient(断网/超时/5xx/别的业务失败):会话还在服务端,片号未知也要接着往里写—— + // 同一片重复上传按覆盖处理(后端 savePart 幂等),重开 init 等于把已传上去的整包作废。 } if (!uploadId) { const init = ensureOk(bodyOf<{ upload_id?: string }>(await http.post( diff --git a/frontend-vue/src/shared/bridges/web-fallback.ts b/frontend-vue/src/shared/bridges/web-fallback.ts index f47287421..649259dff 100644 --- a/frontend-vue/src/shared/bridges/web-fallback.ts +++ b/frontend-vue/src/shared/bridges/web-fallback.ts @@ -9,6 +9,7 @@ import type { DesktopConfig, DesktopConfigUpdate, PywebviewApi } from './pywebview' import { extractErrorMessage, http } from '@/shared/api/http' +import { uploadTimeoutForBytes } from '@/shared/api/upload.ts' /** * 虚拟路径 → 浏览器 File 对象(虚拟路径仅存活于当前页面会话,刷新即失效) @@ -240,7 +241,10 @@ export const webFallbackApi: PywebviewApi = { } try { // http 实例对 /newApi 前缀自动附带 Bearer;FormData 的 Content-Type 交由浏览器自动生成(含 boundary) - const response = await http.post('/newApi/api/files/upload', form, { timeout: 120000 }) + // 超时按体积给(P3-5:与 upload.ts 同一把尺),几 MB 的 xlsx/zip 不再被固定 120s 掐死 + const response = await http.post('/newApi/api/files/upload', form, { + timeout: uploadTimeoutForBytes(file.size), + }) // 消费完即释放:文件已落到服务端,页面后续只用返回的 fileKey/url,不再需要这份本地 File。 // 失败分支保留条目,让用户能在不重新选文件的情况下重试(超量时由 rememberWebFile 的 FIFO 淘汰兜底)。 if (response.data?.success) { diff --git a/frontend-vue/tests/mount/pending-phase4-大档上传接线.mount.test.ts b/frontend-vue/tests/mount/pending-phase4-大档上传接线.mount.test.ts index 5a9543c32..d34f28e0e 100644 --- a/frontend-vue/tests/mount/pending-phase4-大档上传接线.mount.test.ts +++ b/frontend-vue/tests/mount/pending-phase4-大档上传接线.mount.test.ts @@ -54,4 +54,40 @@ describe('AC-11 用户端大档上传', () => { `AC-11:9MB 视频仍走单次上传,实际出口:${urls.join(', ')}`, ).toBe(true) }) + + // P2-9:上面那格的桩直接把包体当返回值,bodyOf 的真实 axios 分支(包体在 .data)零行为覆盖。 + // 形状误判的代价是静默的——upload_id 取不到就报"后端未返回分片会话",已达片号取不到就整包从 0 重传, + // 而桩版本照样绿。这一格按真机形状(AxiosResponse{status, data})对撞,缺牙就判红。 + it('真实 axios 响应形状必须被认下来(P2-9 反桩形假绿)', async () => { + const uploadId = 'b'.repeat(32) + const envelope = (body: unknown) => ({ + status: 200, + statusText: 'OK', + headers: {}, + config: {}, + data: { success: true, message: '', data: body }, + }) + mocks.post.mockImplementation((url: string) => Promise.resolve( + String(url).includes('/complete') + ? envelope({ + fileKey: 'tmp/big.mp4', originalFilename: 'big.mp4', localPath: '/tmp/big.mp4', size: 9437184, + url: 'https://oss.example/big.mp4', objectKey: 'image-video/big.mp4', mediaType: 'video', + }) + : envelope({ upload_id: uploadId, total_parts: 2, received_parts: 1 }), + )) + mocks.get.mockResolvedValue(envelope({ upload_id: uploadId, uploaded_parts: [] })) + + const file = new File([new Uint8Array(9 * 1024 * 1024)], 'big.mp4', { type: 'video/mp4' }) + const uploaded = await uploadImageVideoMedia(file) + + expect(uploaded.objectKey, '包体在 .data 时仍要取到 complete 的结果').toBe('image-video/big.mp4') + const partUrls = mocks.post.mock.calls + .map(([url]) => String(url)) + .filter((url) => /\/part\/\d+$/.test(url)) + expect(partUrls.length, '9MB=2 片,一片都不能少').toBe(2) + expect( + partUrls.every((url) => url.includes(uploadId)), + `bodyOf 没按 axios 形状取 upload_id,实际出口:${partUrls.join(', ')}`, + ).toBe(true) + }) }) diff --git a/frontend-vue/tests/mount/分片会话探测失败分因.mount.test.ts b/frontend-vue/tests/mount/分片会话探测失败分因.mount.test.ts new file mode 100644 index 000000000..ab12e2bc6 --- /dev/null +++ b/frontend-vue/tests/mount/分片会话探测失败分因.mount.test.ts @@ -0,0 +1,185 @@ +// P1-11(AC-9/铁律 18「瞬态失败不得把会话打掉」的用户端那一半)。 +// +// 评审实录:GET 已达片号那一步用无差别 catch ⇒ 断网/超时/5xx 一律 clearStoredSession + 重开 +// upload_id,一次抖动就把 90MB 打回从 0。本文件按两个方向钉死: +// ① 服务端确实说会话没了(HTTP 404/现网形状 HTTP 200+success:false+「不存在或已过期」)⇒ 重开; +// ② 网络层失败/5xx/非会话类业务失败 ⇒ 保留 upload_id,绝不再发 init。 +// 另附 P2-9:桩必须能吃真实 axios 形状(包体在 .data),否则 bodyOf 的 axios 分支零行为覆盖, +// 形状误判会让 done 成空集=静默从 0 而不判红。 +// +// 跑法:npm run test:mount(tests/mount/** 走 vitest+happy-dom,localStorage 可用,不出网)。 +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + post: vi.fn(), + get: vi.fn(), +})) + +vi.mock('@/shared/api/http.ts', () => ({ + http: { post: mocks.post, get: mocks.get }, + post: mocks.post, + get: mocks.get, + unwrapJavaResponse: (value: unknown) => value, +})) + +import { UPLOAD_PART_SIZE, uploadTempFileToJava } from '@/shared/api/upload.ts' + +const SESSION_ID = 's'.repeat(32) +const FRESH_ID = 'f'.repeat(32) +const TWO_PARTS = UPLOAD_PART_SIZE * 2 + +function sessionKeyOf(file: File): string { + return `shufu:chunk:${file.name}:${file.size}:${file.lastModified}:` +} + +function seedSession(file: File, uploadId: string): void { + localStorage.setItem(sessionKeyOf(file), JSON.stringify({ + uploadId, + size: file.size, + lastModified: file.lastModified, + })) +} + +function storedUploadId(file: File): string { + const raw = localStorage.getItem(sessionKeyOf(file)) + if (!raw) return '' + return String((JSON.parse(raw) as { uploadId?: unknown }).uploadId ?? '') +} + +function makeBigFile(name = 'resume.mp4'): File { + return new File([new Uint8Array(TWO_PARTS)], name, { type: 'video/mp4' }) +} + +/** 真实 axios 出口形状:状态码与包体分家(测试桩过去直接把 body 当返回值,见 P2-9)。 */ +function axiosResponse(status: number, body: unknown) { + return { status, data: body } +} + +function javaOk(body: unknown) { + return axiosResponse(200, { success: true, message: '', data: body }) +} + +function javaGone() { + // 现网实读:ChunkedUploadService 抛 BusinessException 不带 code ⇒ GlobalExceptionHandler + // 按 HTTP 200 + success:false 出,只有文案可依据。 + return axiosResponse(200, { success: false, message: '分片会话不存在或已过期', data: null }) +} + +/** 按 URL 分派的 POST 桩:init 发新会话号,片与 complete 一律成功。 */ +function routePosts(initUploadId: string) { + mocks.post.mockImplementation((url: string) => { + const text = String(url) + if (text.includes('/files/chunk/init')) { + return Promise.resolve(javaOk({ upload_id: initUploadId, total_parts: 2 })) + } + if (text.includes('/complete')) { + return Promise.resolve(javaOk({ + fileKey: 'tmp/resume.mp4', originalFilename: 'resume.mp4', localPath: '/tmp/resume.mp4', + size: TWO_PARTS, url: 'https://oss.example/resume.mp4', objectKey: 'image-video/resume.mp4', + })) + } + return Promise.resolve(javaOk({ received_parts: 1 })) + }) +} + +function postedUrls(): string[] { + return mocks.post.mock.calls.map(([url]) => String(url)) +} + +function initedCount(): number { + return postedUrls().filter((url) => url.includes('/files/chunk/init')).length +} + +function partUrls(): string[] { + return postedUrls().filter((url) => /\/part\/\d+$/.test(url)) +} + +describe('P1-11 分片会话探测失败必须分因', () => { + beforeEach(() => { + vi.clearAllMocks() + localStorage.clear() + routePosts(FRESH_ID) + }) + + it('HTTP 404:服务端说会话没了 ⇒ 清状态并重新 init', async () => { + const file = makeBigFile() + seedSession(file, SESSION_ID) + mocks.get.mockResolvedValue(axiosResponse(404, { success: false, message: 'Not Found', data: null })) + + await uploadTempFileToJava(file, { uploadToOss: true, moduleType: 'IMAGE_VIDEO' }) + + expect(initedCount(), '404 之后必须重新开会话').toBe(1) + expect(partUrls().every((url) => url.includes(FRESH_ID)), '新片必须挂在新的会话号上').toBe(true) + expect(partUrls().some((url) => url.includes(SESSION_ID)), '旧会话号已判死,不得再往它写片').toBe(false) + }) + + it('HTTP 200+success:false+「不存在或已过期」(现网形状)⇒ 同样重开', async () => { + const file = makeBigFile() + seedSession(file, SESSION_ID) + mocks.get.mockResolvedValue(javaGone()) + + await uploadTempFileToJava(file, { uploadToOss: true, moduleType: 'IMAGE_VIDEO' }) + + expect(initedCount(), '会话确实回收后不重建就永远卡死').toBe(1) + expect(partUrls().every((url) => url.includes(FRESH_ID))).toBe(true) + }) + + it('HTTP 503:瞬态 ⇒ 保留 upload_id,一条 init 都不许发', async () => { + const file = makeBigFile() + seedSession(file, SESSION_ID) + mocks.get.mockResolvedValue(axiosResponse(503, 'Bad Gateway')) + + await uploadTempFileToJava(file, { uploadToOss: true, moduleType: 'IMAGE_VIDEO' }) + + expect(initedCount(), '一次网关抖动就把整包打回从 0,正是本迭代要消灭的行为').toBe(0) + expect(partUrls().every((url) => url.includes(SESSION_ID)), '片仍要写进原来那个会话').toBe(true) + }) + + it('探测超时(网络层错误):瞬态 ⇒ 保留 upload_id 并续传', async () => { + const file = makeBigFile() + seedSession(file, SESSION_ID) + mocks.get.mockRejectedValue(new Error('timeout of 30000ms exceeded')) + + await uploadTempFileToJava(file, { uploadToOss: true, moduleType: 'IMAGE_VIDEO' }) + + expect(initedCount()).toBe(0) + expect(partUrls().every((url) => url.includes(SESSION_ID))).toBe(true) + }) + + it('非会话类业务失败(如权限):不许顺手清掉会话', async () => { + const file = makeBigFile() + seedSession(file, SESSION_ID) + mocks.get.mockResolvedValue(axiosResponse(200, { success: false, message: '无权访问该分片会话', data: null })) + + await uploadTempFileToJava(file, { uploadToOss: true, moduleType: 'IMAGE_VIDEO' }) + + expect(initedCount()).toBe(0) + expect(partUrls().every((url) => url.includes(SESSION_ID))).toBe(true) + }) + + it('真实 axios 形状:已到片号必须被跳过(P2-9 形状误判即静默从 0)', async () => { + const file = makeBigFile() + seedSession(file, SESSION_ID) + mocks.get.mockResolvedValue(javaOk({ upload_id: SESSION_ID, uploaded_parts: [0] })) + + await uploadTempFileToJava(file, { uploadToOss: true, moduleType: 'IMAGE_VIDEO' }) + + expect(partUrls(), '第 0 片服务端已有,不得重传').toEqual([ + `/newApi/api/files/chunk/${SESSION_ID}/part/1`, + ]) + expect(mocks.get.mock.calls.length, '探测要带 upload_id 问服务端').toBe(1) + }) + + it('探测失败+分片也失败 ⇒ 会话留在 localStorage,且不 DELETE', async () => { + const file = makeBigFile() + seedSession(file, SESSION_ID) + mocks.get.mockRejectedValue(new Error('Network Error')) + mocks.post.mockRejectedValue(new Error('Network Error')) + + await expect(uploadTempFileToJava(file, { uploadToOss: true, moduleType: 'IMAGE_VIDEO' })) + .rejects.toThrow(/Network Error/) + + expect(storedUploadId(file), '瞬态失败后会话必须还在,下次刷新才能从断点续传').toBe(SESSION_ID) + expect(initedCount()).toBe(0) + }) +}) diff --git a/frontend-vue/tests/pending-phase4-大档上传接线.test.ts b/frontend-vue/tests/pending-phase4-大档上传接线.test.ts index 246c74191..c3df2f218 100644 --- a/frontend-vue/tests/pending-phase4-大档上传接线.test.ts +++ b/frontend-vue/tests/pending-phase4-大档上传接线.test.ts @@ -42,3 +42,23 @@ test('图片视频大档上传必须接分片通道', () => { 'AC-11:uploadImageVideoMedia 仍是一次 multipart+固定 120s(大档一断就从 0 重传)', ) }) + +// P3-5:8MB 阈值原先三处各写一遍(客户端 main.py:1479、本仓 upload.ts、后管 chunk-upload.ts), +// 改一处就分叉——本仓这一侧收成一个具名常量,别的文件只许引用、不许再写数。 +test('分片阈值在本仓只能有一个出口', () => { + const pattern = /8\s*\*\s*1024\s*\*\s*1024|8388608/ + const sites = srcFiles + .map(rel) + .filter((file) => pattern.test(readFileSync(resolve(root, file), 'utf-8'))) + assert.deepEqual( + sites, + ['src/shared/api/upload.ts'], + `8MB 阈值散写到了多处:${sites.join(', ')}(阈值只有一个出口=upload.ts::UPLOAD_PART_SIZE)`, + ) + const source = readFileSync(resolve(root, 'src/shared/api/upload.ts'), 'utf-8') + assert.match( + source, + /export const UPLOAD_PART_SIZE\s*=\s*8\s*\*\s*1024\s*\*\s*1024[\s\S]{0,120}export const CHUNK_UPLOAD_THRESHOLD\s*=\s*UPLOAD_PART_SIZE/, + 'AC-11:阈值必须由片大小单点派生,不得各写一份 8MB', + ) +}) diff --git a/frontend-vue/tests/pending-phase4-用户端稳定性.test.ts b/frontend-vue/tests/pending-phase4-用户端稳定性.test.ts index b3d939d59..99899b81d 100644 --- a/frontend-vue/tests/pending-phase4-用户端稳定性.test.ts +++ b/frontend-vue/tests/pending-phase4-用户端稳定性.test.ts @@ -66,6 +66,13 @@ test('pending_ac17_上传必须分片且不再单次 120s', () => { assert.doesNotMatch(source, /timeout:\s*120000/, 'S1-F3:单次 120s 超时的整包 multipart 必须被分片通道取代') assert.match(source, /onProgress|progress/, '进度必须可见') + + // P3-5:桌面桥的浏览器降级直传也是大档出口,固定 120s 会在同一把尺子上漏半条链。 + const bridge = read('src/shared/bridges/web-fallback.ts') + assert.doesNotMatch(bridge, /timeout:\s*120_?000/, + 'P3-5:web-fallback 的大档上传仍写死 120s,慢链路传 Excel/zip 会被自己掐死') + assert.match(bridge, /uploadTimeoutForBytes/, + 'P3-5:超时预算只有一个出口(upload.ts::uploadTimeoutForBytes),桥侧必须复用它') }) test('pending_ac18_表格必须窗口化渲染或走服务端分页', () => {