"use strict"; /** * 自动生成字幕并混 BGM(对齐 Electron autoGenerateSubtitleAndBGM) */ const fs = require("fs"); const path = require("path"); const os = require("os"); const { spawnSync } = require("child_process"); const pipelineNative = require("./pipeline_native.js"); const desktopConfig = require("./desktop_config.js"); const { correctTextWithLevenshtein, similarity: textSimilarity } = require("./subtitle_levenshtein.js"); const { buildKeywordGroups, matchKeywordsInRecords, buildKeywordDrawtextFilter, normalizeDisplayText, stripKeywordsFromDisplay, filterMatchesForRecord, } = require("./subtitle_keyword_drawtext.js"); const OUTPUT_DIR = path.join(os.tmpdir(), "aiclient-subtitle-bgm"); function ensureDir() { fs.mkdirSync(OUTPUT_DIR, { recursive: true }); return OUTPUT_DIR; } function makeOutputPath(prefix, ext = "mp4") { return path.join(ensureDir(), `${prefix}_${Date.now()}.${ext}`); } function execFfmpeg(args) { const ffmpeg = pipelineNative.locateFfmpeg(); if (!ffmpeg) { throw new Error("未找到 ffmpeg,请配置 AICLIENT_FFMPEG_PATH 或 src-tauri/binaries/ffmpeg.exe"); } const r = spawnSync(ffmpeg, args, { encoding: "utf8", windowsHide: true }); if (r.status !== 0) { throw new Error((r.stderr || r.stdout || "").trim() || `ffmpeg 退出码 ${r.status}`); } } function locateFfprobe() { const fromEnv = process.env.AICLIENT_FFPROBE_PATH; if (fromEnv && fs.existsSync(fromEnv)) return fromEnv; const ffmpeg = pipelineNative.locateFfmpeg(); if (!ffmpeg) return null; const sibling = ffmpeg.replace(/ffmpeg(\.exe)?$/i, "ffprobe$1"); return fs.existsSync(sibling) ? sibling : null; } function probeVideoDuration(videoPath) { const ffprobe = locateFfprobe(); if (ffprobe) { const r = spawnSync( ffprobe, [ "-v", "error", "-show_entries", "format=duration", "-of", "default=noprint_wrappers=1:nokey=1", videoPath, ], { encoding: "utf8", windowsHide: true }, ); if (r.status === 0) { const n = parseFloat(String(r.stdout || "").trim()); if (Number.isFinite(n) && n > 0) return n; } } const ffmpeg = pipelineNative.locateFfmpeg(); const r2 = spawnSync(ffmpeg, ["-i", videoPath], { encoding: "utf8", windowsHide: true }); const stderr = r2.stderr || ""; const m = stderr.match(/Duration:\s*(\d+):(\d+):(\d+(?:\.\d+)?)/); if (m) { return Number(m[1]) * 3600 + Number(m[2]) * 60 + Number(m[3]); } return 0; } /** @returns {{ width: number, height: number }} */ function probeVideoSize(videoPath) { const ffprobe = locateFfprobe(); if (ffprobe) { const r = spawnSync( ffprobe, [ "-v", "error", "-select_streams", "v:0", "-show_entries", "stream=width,height", "-of", "csv=p=0:s=x", videoPath, ], { encoding: "utf8", windowsHide: true }, ); if (r.status === 0) { const parts = String(r.stdout || "").trim().split("x"); const width = parseInt(parts[0], 10); const height = parseInt(parts[1], 10); if (width > 0 && height > 0) return { width, height }; } } const ffmpeg = pipelineNative.locateFfmpeg(); const r2 = spawnSync(ffmpeg, ["-i", videoPath], { encoding: "utf8", windowsHide: true }); const stderr = r2.stderr || ""; const m = stderr.match(/,\s*(\d{2,5})x(\d{2,5})/); if (m) { return { width: Number(m[1]), height: Number(m[2]) }; } return { width: 1920, height: 1080 }; } function splitScriptLines(text) { return String(text || "") .split(/(?<=[。!?.!?])\s*|\n+/) .map((s) => s.trim()) .filter(Boolean); } function normalizeAsrTimeToMs(value) { const n = Number(value); if (!Number.isFinite(n) || n < 0) return 0; if (n > 100) return Math.round(n); return Math.round(n * 1000); } /** 对齐 ZimuShengcheng.shouldTreatTimelineAsSeconds */ function shouldTreatTimelineAsSeconds(records) { const valid = records.filter( (r) => Number.isFinite(Number(r?.start)) && Number.isFinite(Number(r?.end)), ); if (valid.length === 0) return false; const maxEnd = Math.max(...valid.map((r) => Number(r.end))); const maxDur = Math.max(...valid.map((r) => Number(r.end) - Number(r.start))); return maxEnd <= 600 && maxDur <= 30; } function normalizeFunasrSegmentsToMilliseconds(records) { if (!Array.isArray(records) || records.length === 0) return []; const mapped = records .map((record) => { const text = String(record?.text || record?.sentence || "").trim(); if (!text) return null; return { text, start: Number(record.start ?? record.begin ?? record.beginTime ?? 0), end: Number(record.end ?? record.endTime ?? 0), words: record.words, }; }) .filter(Boolean); if (shouldTreatTimelineAsSeconds(mapped)) { logSubtitle("info", "检测到秒级时间轴,统一转换为毫秒"); return mapped.map((r) => ({ ...r, start: Math.round(r.start * 1000), end: Math.round((r.end > r.start ? r.end : r.start + 2) * 1000), words: Array.isArray(r.words) ? r.words.map((w) => ({ ...w, start: Math.round(Number(w.start ?? w.begin_time ?? 0) * 1000), end: Math.round(Number(w.end ?? w.end_time ?? 0) * 1000), })) : r.words, })); } return mapped.map((r) => { let start = normalizeAsrTimeToMs(r.start); let end = normalizeAsrTimeToMs(r.end || start + 2); if (end <= start) end = start + 2000; return { text: r.text, start, end, words: r.words }; }); } function normalizeAsrRecords(records) { if (!Array.isArray(records)) return []; const normalized = normalizeFunasrSegmentsToMilliseconds(records) .map((record) => { let { start, end } = record; if (end <= start) end = start + 2000; return { text: record.text, start, end, words: record.words }; }) .filter(Boolean); const MIN_GAP_MS = 50; for (let i = 0; i < normalized.length - 1; i++) { const current = normalized[i]; const next = normalized[i + 1]; if (current.end >= next.start) { current.end = Math.max(current.start + MIN_GAP_MS, next.start - MIN_GAP_MS); } if (current.end <= current.start) { current.end = current.start + MIN_GAP_MS; } } return normalized; } /** 对齐 Electron smartLineBreak:最多 2 行,优先在标点处换行 */ function smartLineBreak(text, maxCharsPerLine = 18) { const lines = []; const punctuations = [",", "。", "!", "?", ";", ":", ",", ".", "!", "?", ";", ":"]; const MAX_LINES = 2; let currentLine = ""; let pureTextLength = 0; let i = 0; const max = Math.max(8, Math.min(30, Number(maxCharsPerLine) || 18)); while (i < text.length) { if (lines.length >= MAX_LINES) { currentLine += text[i]; i += 1; continue; } currentLine += text[i]; pureTextLength += 1; const isPunctuation = punctuations.includes(text[i]); const isNearMaxLength = pureTextLength >= max * 0.8; if (isPunctuation && isNearMaxLength && lines.length < MAX_LINES - 1) { lines.push(currentLine.trim()); currentLine = ""; pureTextLength = 0; } else if (pureTextLength >= max && lines.length < MAX_LINES - 1) { lines.push(currentLine.trim()); currentLine = ""; pureTextLength = 0; } i += 1; } if (currentLine.trim()) lines.push(currentLine.trim()); return lines.slice(0, MAX_LINES); } function cleanSubtitleDisplayText(text) { return String(text || "") .replace(/[,。!?;:、""''()《》【】…—·,.!?;:'"()[\]{}<>]/g, "") .trim(); } /** * 智能字幕:Levenshtein 校对(ZimuShengcheng),失败时回退按句均分 */ function applySmartSubtitle(script, records, videoDurationSec = 0) { const normalized = normalizeAsrRecords(records); const scriptTrim = String(script || "").trim(); if (!scriptTrim) return normalized; const asrPlain = normalized.map((r) => r.text).join(""); const sim = textSimilarity( scriptTrim.replace(/\s+/g, ""), asrPlain.replace(/\s+/g, ""), ); if (normalized.length > 0 && sim >= 0.25) { const { segments, similarity } = correctTextWithLevenshtein(scriptTrim, normalized); if (segments.length > 0) { logSubtitle("info", "Levenshtein 文案校对", { count: segments.length, similarity: Math.round(similarity * 1000) / 1000, scriptSimilarity: Math.round(sim * 1000) / 1000, }); return segments; } } logSubtitle("info", "Levenshtein 未产出有效段落,回退按句对齐", { similarity: sim }); return alignScriptToRecords(scriptTrim, normalized, videoDurationSec); } function alignScriptToRecords(script, records, videoDurationSec = 0) { const lines = splitScriptLines(script); const normalized = normalizeAsrRecords(records); if (!lines.length) return normalized; if (!normalized.length) { return recordsFromTextEvenly(script, videoDurationSec || 30); } if (lines.length === normalized.length) { return normalized.map((r, i) => ({ ...r, text: lines[i] })); } const startMs = normalized[0].start; let endMs = normalized[normalized.length - 1].end; const durationMs = Math.max(0, Math.round(Number(videoDurationSec) * 1000)); if (durationMs > 0 && (endMs - startMs < 1000 || endMs < durationMs * 0.2)) { endMs = durationMs; } const totalMs = Math.max(endMs - startMs, lines.length * 2000); const totalChars = lines.reduce((sum, line) => sum + line.length, 0) || lines.length; let cursor = startMs; return lines.map((text, index) => { const isLast = index === lines.length - 1; const weight = text.length / totalChars; const dur = isLast ? Math.max(1500, endMs - cursor) : Math.max(1500, Math.round(totalMs * weight)); const start = cursor; const end = isLast ? endMs : Math.min(cursor + dur, endMs); cursor = end; return { text, start, end: Math.max(end, start + 500) }; }); } function recordsFromTextEvenly(text, durationSec) { const lines = splitScriptLines(text); if (!lines.length) return []; const totalMs = Math.max(Math.round(durationSec * 1000), lines.length * 1500); const step = totalMs / lines.length; return lines.map((line, i) => ({ text: line, start: Math.round(i * step), end: Math.round((i + 1) * step), })); } function formatSrtTime(ms) { const h = Math.floor(ms / 3600000); const m = Math.floor((ms % 3600000) / 60000); const s = Math.floor((ms % 60000) / 1000); const msPart = Math.floor(ms % 1000); return `${String(h).padStart(2, "0")}:${String(m).padStart(2, "0")}:${String(s).padStart(2, "0")},${String(msPart).padStart(3, "0")}`; } function buildSrtContent(records, maxCharsPerLine = 18) { return records .map((r, i) => { const start = formatSrtTime(r.start); const end = formatSrtTime(Math.max(r.end, r.start + 500)); const text = wrapSubtitleText(r.text, maxCharsPerLine); if (!text) return ""; return `${i + 1}\n${start} --> ${end}\n${text}\n`; }) .filter(Boolean) .join("\n"); } function escapeSubPath(filePath) { return filePath.replace(/\\/g, "/").replace(/:/g, "\\:").replace(/'/g, "\\'"); } /** 对齐 Electron maxCharsPerLine + smartLineBreak;SRT 用换行,ASS 经 assEscapeText 转 \\N */ function wrapSubtitleText(text, maxCharsPerLine = 18) { const cleaned = cleanSubtitleDisplayText(text); if (!cleaned) return ""; return smartLineBreak(cleaned, maxCharsPerLine).join("\n"); } function logSubtitle(level, msg, fields) { const tag = `[autoGenerateSubtitleAndBGM] ${msg}`; if (globalThis.__native?.log) { globalThis.__native.log(level, tag, fields ?? null); return; } if (fields != null) { if (level === "error") console.error(tag, fields); else console.log(tag, fields); } else if (level === "error") console.error(tag); else console.log(tag); } function hexToAssColor(hex) { const h = String(hex || "#FFFFFF").replace("#", ""); const base = h.length >= 6 ? h.slice(0, 6) : "FFFFFF"; const r = base.substring(0, 2); const g = base.substring(2, 4); const b = base.substring(4, 6); return `&H00${b}${g}${r}`.toUpperCase(); } function assEscapeText(text) { return String(text || "") .replace(/\r/g, "") .replace(/\n/g, "\\N") .replace(/{/g, "(") .replace(/}/g, ")"); } function normalizeBurnStyle(style) { const s = style && typeof style === "object" ? style : {}; const outlineWidth = Number(s.outlineWidth); const hasBg = s.backgroundColor && s.backgroundColor !== "transparent" && (Number(s.backgroundOpacity) ?? 0) > 0; return { fontName: String(s.fontName || "Microsoft YaHei"), fontSize: Number(s.fontSize) || 35, fontColor: String(s.fontColor || "#FFFFFF"), outlineColor: String(s.outlineColor || "#000000"), outlineWidth: Number.isFinite(outlineWidth) ? outlineWidth : 2, backgroundColor: String(s.backgroundColor || "transparent"), backgroundOpacity: Number(s.backgroundOpacity) ?? 0, position: String(s.position || "bottom"), fontSizeScale: Number(s.fontSizeScale) || 100, maxCharsPerLine: Number(s.maxCharsPerLine) || 18, hasBg, }; } function resolveAssLayout(position, videoHeight) { const h = Math.max(360, Number(videoHeight) || 1080); const marginV = 80; if (position === "top") { return { alignment: 8, marginL: 120, marginR: 120, marginV }; } if (position === "center") { return { alignment: 5, marginL: 120, marginR: 120, marginV: 0 }; } return { alignment: 2, marginL: 120, marginR: 120, marginV }; } function buildAssContent(records, styleInput, videoWidth, videoHeight, keywordMatches = []) { const style = normalizeBurnStyle(styleInput); const width = Math.max(320, Number(videoWidth) || 1920); const height = Math.max(320, Number(videoHeight) || 1080); const scale = Math.max(0.2, Math.min(2, style.fontSizeScale / 100)); const fontSize = Math.max(14, Math.round(style.fontSize * scale)); const outline = Math.max(0, Math.round(style.outlineWidth)); const { alignment, marginL, marginR, marginV } = resolveAssLayout( style.position, height, ); const hasBg = style.hasBg; let borderStyle = 1; let finalOutline = outline; let finalShadow = 0; let backColour = "&H00000000"; const primary = hexToAssColor(style.fontColor); let outlineColour = hexToAssColor(style.outlineColor); if (hasBg) { borderStyle = 3; finalOutline = 1; finalShadow = 0; backColour = hexToAssColor(style.backgroundColor); outlineColour = backColour; } else { borderStyle = 1; finalOutline = outline || 3; finalShadow = 0; backColour = "&H00000000"; outlineColour = hexToAssColor(style.outlineColor); } const header = [ "[Script Info]", "Title: aiclient", "ScriptType: v4.00+", "WrapStyle: 0", "ScaledBorderAndShadow: yes", `PlayResX: ${width}`, `PlayResY: ${height}`, "", "[V4+ Styles]", "Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding", `Style: Default,${style.fontName},${fontSize},${primary},&H000000FF,${outlineColour},${backColour},1,0,0,0,100,100,0,0,${borderStyle},${finalOutline},${finalShadow},${alignment},${marginL},${marginR},${marginV},1`, "", "[Events]", "Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text", ]; const events = records.map((r) => { const start = formatAssTime(r.start); const end = formatAssTime(Math.max(r.end, r.start + 500)); const display = normalizeDisplayText(r.text); const segMatches = filterMatchesForRecord(r, keywordMatches); const baseDisplay = segMatches.length > 0 ? stripKeywordsFromDisplay(display, segMatches) : display; const text = assEscapeText(wrapSubtitleText(baseDisplay || " ", style.maxCharsPerLine)); if (!text.replace(/\\N/g, "").trim()) return ""; return `Dialogue: 0,${start},${end},Default,,0,0,0,,${text}`; }).filter(Boolean); return `${header.join("\n")}\n${events.join("\n")}\n`; } function formatAssTime(ms) { const h = Math.floor(ms / 3600000); const m = Math.floor((ms % 3600000) / 60000); const s = Math.floor((ms % 60000) / 1000); const cs = Math.floor((ms % 1000) / 10); return `${h}:${String(m).padStart(2, "0")}:${String(s).padStart(2, "0")}.${String(cs).padStart(2, "0")}`; } /** * @param {string} [keywordDrawtextFilter] 逗号分隔的 drawtext 滤镜(关键词高亮) */ function addSubtitleToVideo(inputVideo, subtitlePath, keywordDrawtextFilter) { const outputVideo = makeOutputPath("with_subtitle"); const sub = escapeSubPath(subtitlePath); let vf = `subtitles='${sub}'`; if (keywordDrawtextFilter) { vf = `${vf},${keywordDrawtextFilter}`; } execFfmpeg(["-y", "-i", inputVideo, "-vf", vf, "-c:a", "copy", outputVideo]); return outputVideo; } function addBgmToVideo(inputVideo, bgmPath, bgmVolume = 30) { const outputVideo = makeOutputPath("with_bgm"); const ratio = Number(bgmVolume) / 100; const filter = `[0:a]volume=1.0[a0];[1:a]volume=${ratio}[a1];[a0][a1]amix=inputs=2:duration=first`; execFfmpeg([ "-y", "-i", inputVideo, "-stream_loop", "-1", "-i", bgmPath, "-filter_complex", filter, "-c:v", "copy", "-shortest", outputVideo, ]); return outputVideo; } async function runAsr(audioPath, modelConfig, scriptContent) { const asrMode = modelConfig.asrMode || "online"; if (asrMode === "online") { if (!modelConfig.aliyunApiKey) { throw new Error("在线 ASR 需配置 BAILIAN_API_KEY 或 DASHSCOPE_API_KEY"); } if (!modelConfig.ossConfig?.bucket) { throw new Error("在线 ASR 需配置 OSS(上传音频供识别)"); } globalThis.__native?.emitProgress?.("正在上传音频到 OSS…"); const upload = await pipelineNative.aliyunOssUpload(audioPath, modelConfig.ossConfig); if (!upload?.success || !upload.url) { throw new Error(upload?.error || "OSS 上传失败"); } globalThis.__native?.emitProgress?.("正在进行语音识别…"); const asr = await pipelineNative.aliyunAsrFiletrans(upload.url, { apiKey: modelConfig.aliyunApiKey, model: "qwen3-asr-flash-filetrans", }); if (!asr?.success) { throw new Error(asr?.error || "语音识别失败"); } let records = Array.isArray(asr.sentences) ? asr.sentences : []; if (!records.length && asr.text) { records = recordsFromTextEvenly(asr.text, 60); } return { text: asr.text || "", records }; } const info = modelConfig.asrServerInfo; if (!info) { throw new Error("本地 ASR 需配置 ASR_SERVER_URL 或 ASR_SERVER_INFO"); } globalThis.__native?.emitProgress?.("正在调用本地语音识别…"); const asr = await pipelineNative.localAsr(audioPath, info); if (!asr?.success) { throw new Error(asr?.error || "本地语音识别失败"); } let records = Array.isArray(asr.sentences) ? asr.sentences : []; if (!records.length && asr.text) { records = recordsFromTextEvenly(asr.text, 60); } return { text: asr.text || "", records }; } globalThis.__nodejsMain = async function main(params) { const p = params || {}; const inputVideo = String(p.inputVideo || "").trim(); if (!inputVideo || !fs.existsSync(inputVideo)) { return { success: false, error: "缺少或找不到输入视频,请先在步骤 02/03 生成并处理视频" }; } const autoSubtitle = Boolean(p.autoSubtitle); const bgmEnabled = Boolean(p.bgmEnabled); if (!autoSubtitle && !bgmEnabled) { return { success: false, error: "请至少勾选「自动生成字幕」或「添加背景音乐」" }; } if (bgmEnabled) { const bgm = String(p.bgmPath || "").trim(); if (!bgm || !fs.existsSync(bgm)) { return { success: false, error: "已启用背景音乐,请先选择音乐文件" }; } } const modelConfig = desktopConfig.buildModelConfig(); const check = desktopConfig.validateModelConfig(modelConfig); if (autoSubtitle && !check.ok) { return { success: false, error: check.error }; } if (autoSubtitle && (modelConfig.asrMode === "online" || !modelConfig.asrMode)) { const appCheck = desktopConfig.validateModelConfig(modelConfig); if (!appCheck.ok && modelConfig.asrMode === "online") { return { success: false, error: appCheck.error }; } } let current = inputVideo; let srtPath = ""; try { if (autoSubtitle) { globalThis.__native?.emitProgress?.("正在提取音频…"); const audioPath = await pipelineNative.ffmpegExtractAudio(current); const scriptContent = String(p.scriptContent || "").trim(); const videoDuration = probeVideoDuration(current); logSubtitle("info", "视频时长(秒)", { videoDuration }); const { records: rawRecords } = await runAsr(audioPath, modelConfig, scriptContent); logSubtitle("info", "ASR 原始句数", { count: rawRecords?.length ?? 0 }); let records = normalizeAsrRecords(rawRecords); logSubtitle("info", "时间轴归一化后", { count: records.length, spanMs: records.length > 1 ? records[records.length - 1].end - records[0].start : 0, }); if (p.smartSubtitle !== false && scriptContent) { records = applySmartSubtitle(scriptContent, records, videoDuration); logSubtitle("info", "智能字幕对齐后", { count: records.length }); } if (!records.length) { const fallbackText = scriptContent || " "; records = recordsFromTextEvenly(fallbackText, videoDuration || 30); logSubtitle("info", "按文案均分时间轴", { count: records.length }); } if (!records.length) { return { success: false, error: "未识别到语音内容,无法生成字幕" }; } const burnStyle = p.subtitleStyle && typeof p.subtitleStyle === "object" ? p.subtitleStyle : null; const maxChars = burnStyle?.maxCharsPerLine || 18; srtPath = makeOutputPath("subtitle", "srt"); fs.writeFileSync(srtPath, buildSrtContent(records, maxChars), "utf8"); if (p.recognizeOnly) { pipelineNative.deleteFile(audioPath); return { success: true, records, srtPath, message: `字幕识别完成,共 ${records.length} 条`, }; } const { width, height } = probeVideoSize(current); let keywordDrawtextFilter = null; let keywordMatches = []; const enableKeywords = p.enableKeywordEffects !== false; const keywordGroups = buildKeywordGroups(p); if (enableKeywords && keywordGroups.length > 0) { keywordMatches = matchKeywordsInRecords(records, keywordGroups); logSubtitle("info", "关键词匹配", { groups: keywordGroups.length, matches: keywordMatches.length, keywords: keywordMatches.slice(0, 8).map((m) => m.keyword), }); if (keywordMatches.length > 0) { keywordDrawtextFilter = buildKeywordDrawtextFilter(keywordMatches, { videoWidth: width, videoHeight: height, subtitleStyle: burnStyle, }); if (!keywordDrawtextFilter) { logSubtitle("warn", "未找到可用字体,跳过关键词 drawtext 特效"); keywordMatches = []; } } } const assPath = makeOutputPath("subtitle", "ass"); fs.writeFileSync( assPath, buildAssContent(records, burnStyle, width, height, keywordMatches), "utf8", ); globalThis.__native?.emitProgress?.( keywordDrawtextFilter ? "正在烧录字幕并叠加关键词特效…" : "正在烧录字幕到视频…", ); current = addSubtitleToVideo(current, assPath, keywordDrawtextFilter); pipelineNative.deleteFile(audioPath); } if (bgmEnabled) { globalThis.__native?.emitProgress?.("正在混合背景音乐…"); current = addBgmToVideo(current, p.bgmPath, p.bgmVolume ?? 30); } return { success: true, videoPath: current, srtPath: srtPath || null, message: autoSubtitle && bgmEnabled ? "字幕与背景音乐处理完成" : autoSubtitle ? "字幕生成完成" : "背景音乐添加完成", }; } catch (e) { return { success: false, error: e.message || String(e), partialVideo: current !== inputVideo ? current : null, }; } };