Files
crawler-plugin/admin-frontend-vue/src/api/chunk-upload.ts
T

212 lines
8.6 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import { http, unwrap } from '@/api/http'
/**
* 大档分片阈值与片大小(AC-11):后管这一侧只有这一个出口,阈值由片大小派生。
*
* ⚠ 同族的 8MB 数还另有两处:用户端上传出口一处、桌面客户端分片入口一处。两边是各自
* 独立的构建根(各自的 `@` 别名与 tsconfig include),客户端是 Python 打包物 ⇒
* 没有共享包就合不成同一个字面量(P3-5 的边界,已登记)。
*/
export const CHUNK_UPLOAD_PART_SIZE = 8 * 1024 * 1024
export const CHUNK_UPLOAD_THRESHOLD = CHUNK_UPLOAD_PART_SIZE
/** 每片重试次数(不含首次):单片 502/超时不该让 512MB 从 0 再来。 */
const PART_RETRY_LIMIT = 2
/**
* 逐片超时(P2-8):预算只看**这一片**有多大——60s 保底 + 每 MB 30s(8MB 片=300s)。
* 过去按整包体积算(120s + MB×15s),512MB 包里的一个 8MB 片能拿到 128 分钟,
* 慢链路真卡死时没人知道,进度条还会一直爬。
*/
export function partTimeoutOf(partBytes: number): number {
return 60_000 + Math.ceil(partBytes / (1024 * 1024)) * 30_000
}
/** 会话登记/查询只发元数据,固定预算即可;失败要快快回来,别陪整包一起放大。 */
const REQUEST_TIMEOUT_MS = 60_000
/** 只有拼装这一步真随整包体积走:服务端要读全部分片、复算 sha256。 */
const COMPLETE_TIMEOUT_MS = 600_000
function partUrl(uploadId: string, index: number): string {
return `/api/files/chunk/${encodeURIComponent(uploadId)}/part/${index}`
}
/** 分片会话跨尝试/刷新续用:同一文件身份复用 upload_id,服务端没回收就从已达片号继续(P2-7)。 */
interface StoredChunkSession {
uploadId: string;
size: number;
lastModified: number;
}
const CHUNK_SESSION_PREFIX = 'shufu:admin:chunk:'
function lastModifiedOf(file: Blob): number {
const value = (file as File).lastModified
return typeof value === 'number' && Number.isFinite(value) ? value : 0
}
/** 会话键:文件名+体积+修改时间,三者同一条上传才算同一个件。 */
export function chunkSessionKeyOf(fileName: string, file: Blob): string {
return `${CHUNK_SESSION_PREFIX}${fileName}:${file.size}:${lastModifiedOf(file)}`
}
function readStoredSession(key: string, file: Blob): string {
if (typeof localStorage === 'undefined') return ''
try {
const raw = localStorage.getItem(key)
if (!raw) return ''
const parsed = JSON.parse(raw) as StoredChunkSession
return parsed.size === file.size && parsed.lastModified === lastModifiedOf(file) ? parsed.uploadId : ''
} catch {
return ''
}
}
function writeStoredSession(key: string, file: Blob, uploadId: string): void {
if (typeof localStorage === 'undefined') return
try {
localStorage.setItem(key, JSON.stringify({
uploadId, size: file.size, lastModified: lastModifiedOf(file),
} satisfies StoredChunkSession))
} catch {
// 存储不可用(隐私模式/配额满)只影响跨刷新续传,不影响本次上传
}
}
function clearStoredSession(key: string): void {
if (typeof localStorage === 'undefined') return
try {
localStorage.removeItem(key)
} catch {
// 同上:清不掉只会让下次多问一次 status
}
}
/**
* 服务端对「这个会话还在不在」的三种答复:还在(附带已到片号)/确实没了/问不出来。
* 第三态是 P2-7 与 P1-11 的共同判据——过去它被当成"没会话",瞬态抖动=整包作废。
*/
type ProbeOutcome
= { state: 'ok'; parts: number[] }
| { state: 'gone' }
| { state: 'transient' }
/** 后端回收会话的原文案(ChunkedUploadService.readMeta);BusinessException 不带 code ⇒ HTTP 200 出。 */
function messageSaysSessionGone(text: unknown): boolean {
const message = typeof text === 'string' ? text : ''
return ['不存在', '过期'].some((marker) => message.includes(marker))
}
function classifyProbe(response: unknown): ProbeOutcome {
const status = Number((response as { status?: unknown } | null | undefined)?.status)
if (status === 404) return { state: 'gone' }
if (status >= 500 || status === 408 || status === 429) return { state: 'transient' }
const body = (response as { data?: unknown } | null | undefined)?.data
if (body && typeof body === 'object' && (body as { success?: unknown }).success === false) {
return messageSaysSessionGone((body as { message?: unknown }).message)
? { state: 'gone' }
: { state: 'transient' }
}
const core = (unwrap<Record<string, unknown>>(body) ?? {}) as Record<string, unknown>
const arrived = core.uploaded_parts
return { state: 'ok', parts: Array.isArray(arrived) ? arrived.map((value) => Number(value)) : [] }
}
/**
* 探测已达片号(P2-7/与用户端 upload.ts 同一口径)。
* validateStatus 恒真:状态码要自己判——真 404/5xx 一旦被拦截器折成 Error 就分不出原因了;
* 能走到 catch 的就只剩网络层失败与超时,铁定瞬态。
*/
async function probeUploadedParts(uploadId: string): Promise<ProbeOutcome> {
try {
const response = await http.get(
`/api/files/chunk/${encodeURIComponent(uploadId)}`,
{ validateStatus: () => true },
)
return classifyProbe(response)
} catch (error) {
const status = (error as { response?: { status?: unknown } } | null | undefined)?.response?.status
if (status === 404) return { state: 'gone' }
if (messageSaysSessionGone((error as { message?: unknown } | null | undefined)?.message)) {
return { state: 'gone' }
}
return { state: 'transient' }
}
}
/** 后管大档上传(版本包/教程包共用):会话续用 → GET 已达片号 → 逐片 POST → complete。 */
export async function uploadFileInChunks(
file: Blob,
fileName: string,
onProgress?: (percent: number) => void,
): Promise<Record<string, unknown>> {
const totalParts = Math.max(1, Math.ceil(file.size / CHUNK_UPLOAD_PART_SIZE))
const sessionKey = chunkSessionKeyOf(fileName, file)
let uploadId = readStoredSession(sessionKey, file)
const arrived = new Set<number>()
if (uploadId) {
const probed = await probeUploadedParts(uploadId)
if (probed.state === 'gone') {
// 服务端确实说会话没了(404/「不存在或已过期」):清掉重开,否则这个件每次都在同一个死会话上失败。
clearStoredSession(sessionKey)
uploadId = ''
} else if (probed.state === 'ok') {
probed.parts.forEach((index) => arrived.add(index))
}
// transient:片号未知但会话还在 ⇒ 照原会话逐片重传(后端同片覆盖,幂等),不重开 init。
}
if (!uploadId) {
const initResponse = await http.post('/api/files/chunk/init', null, {
params: { fileName, totalParts, totalBytes: file.size },
timeout: REQUEST_TIMEOUT_MS,
})
const init = (unwrap(initResponse.data) ?? {}) as Record<string, unknown>
uploadId = String(init.upload_id ?? '')
if (!uploadId) {
throw new Error('后端未返回分片会话,请重试')
}
writeStoredSession(sessionKey, file, uploadId)
}
for (let index = 0; index < totalParts; index += 1) {
if (arrived.has(index)) {
continue
}
const blob = file.slice(index * CHUNK_UPLOAD_PART_SIZE,
Math.min(file.size, (index + 1) * CHUNK_UPLOAD_PART_SIZE))
let lastError: unknown = null
for (let attempt = 0; attempt <= PART_RETRY_LIMIT; attempt += 1) {
const form = new FormData()
form.append('file', blob, `part-${index}`)
try {
const partResponse = await http.post(partUrl(uploadId, index), form, {
headers: { 'Content-Type': 'multipart/form-data' },
timeout: partTimeoutOf(blob.size),
onUploadProgress: (event) => {
if (!onProgress) return
const perPart = Math.round(((index + (event.loaded || 0) / Math.max(blob.size, 1)) / totalParts) * 100)
onProgress(Math.min(perPart, 99))
},
})
unwrap(partResponse.data)
arrived.add(index)
lastError = null
break
} catch (error) {
lastError = error
}
}
if (lastError) {
// 不 DELETE(P2-7):会话连同已达片号留在服务端(后端 24h 过期回收兜底),
// 下一次点开同一个文件就从断点接着传——这才是"续传"两个字的本义。
throw lastError instanceof Error ? lastError : new Error('分片上传失败')
}
}
const completeResponse = await http.post(
`/api/files/chunk/${encodeURIComponent(uploadId)}/complete`, null, { timeout: COMPLETE_TIMEOUT_MS })
clearStoredSession(sessionKey)
if (onProgress) onProgress(100)
return (unwrap(completeResponse.data) ?? {}) as Record<string, unknown>
}