Files
yaoyaoai/scripts/nodejs/subtitle_bgm_generate.js
949036910@qq.com 8970f51fc0 11
2026-06-01 00:48:33 +08:00

591 lines
20 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"use strict";
/**
* 自动生成字幕并混 BGM对齐 Electron autoGenerateSubtitleAndBGM
*/
const fs = require("fs");
const path = require("path");
const os = require("os");
const { spawnSync } = require("child_process");
const pipelineNative = require("./pipeline_native.js");
const desktopConfig = require("./desktop_config.js");
const OUTPUT_DIR = path.join(os.tmpdir(), "aiclient-subtitle-bgm");
function ensureDir() {
fs.mkdirSync(OUTPUT_DIR, { recursive: true });
return OUTPUT_DIR;
}
function makeOutputPath(prefix, ext = "mp4") {
return path.join(ensureDir(), `${prefix}_${Date.now()}.${ext}`);
}
function execFfmpeg(args) {
const ffmpeg = pipelineNative.locateFfmpeg();
if (!ffmpeg) {
throw new Error("未找到 ffmpeg请配置 AICLIENT_FFMPEG_PATH 或 src-tauri/binaries/ffmpeg.exe");
}
const r = spawnSync(ffmpeg, args, { encoding: "utf8", windowsHide: true });
if (r.status !== 0) {
throw new Error((r.stderr || r.stdout || "").trim() || `ffmpeg 退出码 ${r.status}`);
}
}
function locateFfprobe() {
const fromEnv = process.env.AICLIENT_FFPROBE_PATH;
if (fromEnv && fs.existsSync(fromEnv)) return fromEnv;
const ffmpeg = pipelineNative.locateFfmpeg();
if (!ffmpeg) return null;
const sibling = ffmpeg.replace(/ffmpeg(\.exe)?$/i, "ffprobe$1");
return fs.existsSync(sibling) ? sibling : null;
}
function probeVideoDuration(videoPath) {
const ffprobe = locateFfprobe();
if (ffprobe) {
const r = spawnSync(
ffprobe,
[
"-v",
"error",
"-show_entries",
"format=duration",
"-of",
"default=noprint_wrappers=1:nokey=1",
videoPath,
],
{ encoding: "utf8", windowsHide: true },
);
if (r.status === 0) {
const n = parseFloat(String(r.stdout || "").trim());
if (Number.isFinite(n) && n > 0) return n;
}
}
const ffmpeg = pipelineNative.locateFfmpeg();
const r2 = spawnSync(ffmpeg, ["-i", videoPath], { encoding: "utf8", windowsHide: true });
const stderr = r2.stderr || "";
const m = stderr.match(/Duration:\s*(\d+):(\d+):(\d+(?:\.\d+)?)/);
if (m) {
return Number(m[1]) * 3600 + Number(m[2]) * 60 + Number(m[3]);
}
return 0;
}
/** @returns {{ width: number, height: number }} */
function probeVideoSize(videoPath) {
const ffprobe = locateFfprobe();
if (ffprobe) {
const r = spawnSync(
ffprobe,
[
"-v",
"error",
"-select_streams",
"v:0",
"-show_entries",
"stream=width,height",
"-of",
"csv=p=0:s=x",
videoPath,
],
{ encoding: "utf8", windowsHide: true },
);
if (r.status === 0) {
const parts = String(r.stdout || "").trim().split("x");
const width = parseInt(parts[0], 10);
const height = parseInt(parts[1], 10);
if (width > 0 && height > 0) return { width, height };
}
}
const ffmpeg = pipelineNative.locateFfmpeg();
const r2 = spawnSync(ffmpeg, ["-i", videoPath], { encoding: "utf8", windowsHide: true });
const stderr = r2.stderr || "";
const m = stderr.match(/,\s*(\d{2,5})x(\d{2,5})/);
if (m) {
return { width: Number(m[1]), height: Number(m[2]) };
}
return { width: 1920, height: 1080 };
}
function splitScriptLines(text) {
return String(text || "")
.split(/(?<=[。!?.!?])\s*|\n+/)
.map((s) => s.trim())
.filter(Boolean);
}
function normalizeAsrTimeToMs(value) {
const n = Number(value);
if (!Number.isFinite(n) || n < 0) return 0;
if (n > 100) return Math.round(n);
return Math.round(n * 1000);
}
function normalizeAsrRecords(records) {
if (!Array.isArray(records)) return [];
const normalized = records
.map((record) => {
const text = String(record?.text || record?.sentence || "").trim();
if (!text) return null;
let start = normalizeAsrTimeToMs(
record.start ?? record.begin ?? record.beginTime ?? 0,
);
let end = normalizeAsrTimeToMs(
record.end ?? record.endTime ?? start + 2000,
);
if (end <= start) end = start + 2000;
return { text, start, end };
})
.filter(Boolean);
const MIN_GAP_MS = 50;
for (let i = 0; i < normalized.length - 1; i++) {
const current = normalized[i];
const next = normalized[i + 1];
if (current.end >= next.start) {
current.end = Math.max(current.start + MIN_GAP_MS, next.start - MIN_GAP_MS);
}
if (current.end <= current.start) {
current.end = current.start + MIN_GAP_MS;
}
}
return normalized;
}
/** 对齐 Electron smartLineBreak最多 2 行,优先在标点处换行 */
function smartLineBreak(text, maxCharsPerLine = 18) {
const lines = [];
const punctuations = ["", "。", "", "", "", "", ",", ".", "!", "?", ";", ":"];
const MAX_LINES = 2;
let currentLine = "";
let pureTextLength = 0;
let i = 0;
const max = Math.max(8, Math.min(30, Number(maxCharsPerLine) || 18));
while (i < text.length) {
if (lines.length >= MAX_LINES) {
currentLine += text[i];
i += 1;
continue;
}
currentLine += text[i];
pureTextLength += 1;
const isPunctuation = punctuations.includes(text[i]);
const isNearMaxLength = pureTextLength >= max * 0.8;
if (isPunctuation && isNearMaxLength && lines.length < MAX_LINES - 1) {
lines.push(currentLine.trim());
currentLine = "";
pureTextLength = 0;
} else if (pureTextLength >= max && lines.length < MAX_LINES - 1) {
lines.push(currentLine.trim());
currentLine = "";
pureTextLength = 0;
}
i += 1;
}
if (currentLine.trim()) lines.push(currentLine.trim());
return lines.slice(0, MAX_LINES);
}
function cleanSubtitleDisplayText(text) {
return String(text || "")
.replace(/[,。!?;:、""''()《》【】…—·,.!?;:'"()[\]{}<>]/g, "")
.trim();
}
function alignScriptToRecords(script, records, videoDurationSec = 0) {
const lines = splitScriptLines(script);
const normalized = normalizeAsrRecords(records);
if (!lines.length) return normalized;
if (!normalized.length) {
return recordsFromTextEvenly(script, videoDurationSec || 30);
}
if (lines.length === normalized.length) {
return normalized.map((r, i) => ({ ...r, text: lines[i] }));
}
const startMs = normalized[0].start;
let endMs = normalized[normalized.length - 1].end;
const durationMs = Math.max(0, Math.round(Number(videoDurationSec) * 1000));
if (durationMs > 0 && (endMs - startMs < 1000 || endMs < durationMs * 0.2)) {
endMs = durationMs;
}
const totalMs = Math.max(endMs - startMs, lines.length * 2000);
const totalChars = lines.reduce((sum, line) => sum + line.length, 0) || lines.length;
let cursor = startMs;
return lines.map((text, index) => {
const isLast = index === lines.length - 1;
const weight = text.length / totalChars;
const dur = isLast
? Math.max(1500, endMs - cursor)
: Math.max(1500, Math.round(totalMs * weight));
const start = cursor;
const end = isLast ? endMs : Math.min(cursor + dur, endMs);
cursor = end;
return { text, start, end: Math.max(end, start + 500) };
});
}
function recordsFromTextEvenly(text, durationSec) {
const lines = splitScriptLines(text);
if (!lines.length) return [];
const totalMs = Math.max(Math.round(durationSec * 1000), lines.length * 1500);
const step = totalMs / lines.length;
return lines.map((line, i) => ({
text: line,
start: Math.round(i * step),
end: Math.round((i + 1) * step),
}));
}
function formatSrtTime(ms) {
const h = Math.floor(ms / 3600000);
const m = Math.floor((ms % 3600000) / 60000);
const s = Math.floor((ms % 60000) / 1000);
const msPart = Math.floor(ms % 1000);
return `${String(h).padStart(2, "0")}:${String(m).padStart(2, "0")}:${String(s).padStart(2, "0")},${String(msPart).padStart(3, "0")}`;
}
function buildSrtContent(records, maxCharsPerLine = 18) {
return records
.map((r, i) => {
const start = formatSrtTime(r.start);
const end = formatSrtTime(Math.max(r.end, r.start + 500));
const text = wrapSubtitleText(r.text, maxCharsPerLine);
if (!text) return "";
return `${i + 1}\n${start} --> ${end}\n${text}\n`;
})
.filter(Boolean)
.join("\n");
}
function escapeSubPath(filePath) {
return filePath.replace(/\\/g, "/").replace(/:/g, "\\:").replace(/'/g, "\\'");
}
/** 对齐 Electron maxCharsPerLine + smartLineBreakSRT 用换行ASS 经 assEscapeText 转 \\N */
function wrapSubtitleText(text, maxCharsPerLine = 18) {
const cleaned = cleanSubtitleDisplayText(text);
if (!cleaned) return "";
return smartLineBreak(cleaned, maxCharsPerLine).join("\n");
}
function logSubtitle(level, msg, fields) {
const tag = `[autoGenerateSubtitleAndBGM] ${msg}`;
if (globalThis.__native?.log) {
globalThis.__native.log(level, tag, fields ?? null);
return;
}
if (fields != null) {
if (level === "error") console.error(tag, fields);
else console.log(tag, fields);
} else if (level === "error") console.error(tag);
else console.log(tag);
}
function hexToAssColor(hex) {
const h = String(hex || "#FFFFFF").replace("#", "");
const base = h.length >= 6 ? h.slice(0, 6) : "FFFFFF";
const r = base.substring(0, 2);
const g = base.substring(2, 4);
const b = base.substring(4, 6);
return `&H00${b}${g}${r}`.toUpperCase();
}
function assEscapeText(text) {
return String(text || "")
.replace(/\r/g, "")
.replace(/\n/g, "\\N")
.replace(/{/g, "(")
.replace(/}/g, ")");
}
function normalizeBurnStyle(style) {
const s = style && typeof style === "object" ? style : {};
const outlineWidth = Number(s.outlineWidth);
const hasBg =
s.backgroundColor &&
s.backgroundColor !== "transparent" &&
(Number(s.backgroundOpacity) ?? 0) > 0;
return {
fontName: String(s.fontName || "Microsoft YaHei"),
fontSize: Number(s.fontSize) || 35,
fontColor: String(s.fontColor || "#FFFFFF"),
outlineColor: String(s.outlineColor || "#000000"),
outlineWidth: Number.isFinite(outlineWidth) ? outlineWidth : 2,
backgroundColor: String(s.backgroundColor || "transparent"),
backgroundOpacity: Number(s.backgroundOpacity) ?? 0,
position: String(s.position || "bottom"),
fontSizeScale: Number(s.fontSizeScale) || 100,
maxCharsPerLine: Number(s.maxCharsPerLine) || 18,
hasBg,
};
}
function resolveAssLayout(position, videoHeight) {
const h = Math.max(360, Number(videoHeight) || 1080);
const marginV = 80;
if (position === "top") {
return { alignment: 8, marginL: 120, marginR: 120, marginV };
}
if (position === "center") {
return { alignment: 5, marginL: 120, marginR: 120, marginV: 0 };
}
return { alignment: 2, marginL: 120, marginR: 120, marginV };
}
function buildAssContent(records, styleInput, videoWidth, videoHeight) {
const style = normalizeBurnStyle(styleInput);
const width = Math.max(320, Number(videoWidth) || 1920);
const height = Math.max(320, Number(videoHeight) || 1080);
const scale = Math.max(0.2, Math.min(2, style.fontSizeScale / 100));
const fontSize = Math.max(14, Math.round(style.fontSize * scale));
const outline = Math.max(0, Math.round(style.outlineWidth));
const { alignment, marginL, marginR, marginV } = resolveAssLayout(
style.position,
height,
);
const hasBg = style.hasBg;
const borderStyle = hasBg ? 3 : 1;
const finalOutline = hasBg ? 1 : outline;
const backColour = hasBg
? hexToAssColor(style.backgroundColor)
: "&H00000000";
const primary = hexToAssColor(style.fontColor);
const outlineColour = hexToAssColor(style.outlineColor);
const header = [
"[Script Info]",
"Title: aiclient",
"ScriptType: v4.00+",
"WrapStyle: 0",
"ScaledBorderAndShadow: yes",
`PlayResX: ${width}`,
`PlayResY: ${height}`,
"",
"[V4+ Styles]",
"Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding",
`Style: Default,${style.fontName},${fontSize},${primary},${primary},${outlineColour},${backColour},0,0,0,0,100,100,0,0,${borderStyle},${finalOutline},0,${alignment},${marginL},${marginR},${marginV},1`,
"",
"[Events]",
"Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text",
];
const events = records.map((r) => {
const start = formatAssTime(r.start);
const end = formatAssTime(Math.max(r.end, r.start + 500));
const text = assEscapeText(wrapSubtitleText(r.text, style.maxCharsPerLine));
if (!text) return "";
return `Dialogue: 0,${start},${end},Default,,0,0,0,,${text}`;
}).filter(Boolean);
return `${header.join("\n")}\n${events.join("\n")}\n`;
}
function formatAssTime(ms) {
const h = Math.floor(ms / 3600000);
const m = Math.floor((ms % 3600000) / 60000);
const s = Math.floor((ms % 60000) / 1000);
const cs = Math.floor((ms % 1000) / 10);
return `${h}:${String(m).padStart(2, "0")}:${String(s).padStart(2, "0")}.${String(cs).padStart(2, "0")}`;
}
function addSubtitleToVideo(inputVideo, subtitlePath, styleInput) {
const outputVideo = makeOutputPath("with_subtitle");
const sub = escapeSubPath(subtitlePath);
const vf = `subtitles='${sub}'`;
execFfmpeg(["-y", "-i", inputVideo, "-vf", vf, "-c:a", "copy", outputVideo]);
return outputVideo;
}
function addBgmToVideo(inputVideo, bgmPath, bgmVolume = 30) {
const outputVideo = makeOutputPath("with_bgm");
const ratio = Number(bgmVolume) / 100;
const filter = `[0:a]volume=1.0[a0];[1:a]volume=${ratio}[a1];[a0][a1]amix=inputs=2:duration=first`;
execFfmpeg([
"-y",
"-i",
inputVideo,
"-stream_loop",
"-1",
"-i",
bgmPath,
"-filter_complex",
filter,
"-c:v",
"copy",
"-shortest",
outputVideo,
]);
return outputVideo;
}
async function runAsr(audioPath, modelConfig, scriptContent) {
const asrMode = modelConfig.asrMode || "online";
if (asrMode === "online") {
if (!modelConfig.aliyunApiKey) {
throw new Error("在线 ASR 需配置 BAILIAN_API_KEY 或 DASHSCOPE_API_KEY");
}
if (!modelConfig.ossConfig?.bucket) {
throw new Error("在线 ASR 需配置 OSS上传音频供识别");
}
globalThis.__native?.emitProgress?.("正在上传音频到 OSS…");
const upload = await pipelineNative.aliyunOssUpload(audioPath, modelConfig.ossConfig);
if (!upload?.success || !upload.url) {
throw new Error(upload?.error || "OSS 上传失败");
}
globalThis.__native?.emitProgress?.("正在进行语音识别…");
const asr = await pipelineNative.aliyunAsrFiletrans(upload.url, {
apiKey: modelConfig.aliyunApiKey,
model: "qwen3-asr-flash-filetrans",
});
if (!asr?.success) {
throw new Error(asr?.error || "语音识别失败");
}
let records = Array.isArray(asr.sentences) ? asr.sentences : [];
if (!records.length && asr.text) {
records = recordsFromTextEvenly(asr.text, 60);
}
return { text: asr.text || "", records };
}
const info = modelConfig.asrServerInfo;
if (!info) {
throw new Error("本地 ASR 需配置 ASR_SERVER_URL 或 ASR_SERVER_INFO");
}
globalThis.__native?.emitProgress?.("正在调用本地语音识别…");
const asr = await pipelineNative.localAsr(audioPath, info);
if (!asr?.success) {
throw new Error(asr?.error || "本地语音识别失败");
}
let records = Array.isArray(asr.sentences) ? asr.sentences : [];
if (!records.length && asr.text) {
records = recordsFromTextEvenly(asr.text, 60);
}
return { text: asr.text || "", records };
}
globalThis.__nodejsMain = async function main(params) {
const p = params || {};
const inputVideo = String(p.inputVideo || "").trim();
if (!inputVideo || !fs.existsSync(inputVideo)) {
return { success: false, error: "缺少或找不到输入视频,请先在步骤 02/03 生成并处理视频" };
}
const autoSubtitle = Boolean(p.autoSubtitle);
const bgmEnabled = Boolean(p.bgmEnabled);
if (!autoSubtitle && !bgmEnabled) {
return { success: false, error: "请至少勾选「自动生成字幕」或「添加背景音乐」" };
}
if (bgmEnabled) {
const bgm = String(p.bgmPath || "").trim();
if (!bgm || !fs.existsSync(bgm)) {
return { success: false, error: "已启用背景音乐,请先选择音乐文件" };
}
}
const modelConfig = desktopConfig.buildModelConfig();
const check = desktopConfig.validateModelConfig(modelConfig);
if (autoSubtitle && !check.ok) {
return { success: false, error: check.error };
}
if (autoSubtitle && (modelConfig.asrMode === "online" || !modelConfig.asrMode)) {
const appCheck = desktopConfig.validateModelConfig(modelConfig);
if (!appCheck.ok && modelConfig.asrMode === "online") {
return { success: false, error: appCheck.error };
}
}
let current = inputVideo;
let srtPath = "";
try {
if (autoSubtitle) {
globalThis.__native?.emitProgress?.("正在提取音频…");
const audioPath = await pipelineNative.ffmpegExtractAudio(current);
const scriptContent = String(p.scriptContent || "").trim();
const videoDuration = probeVideoDuration(current);
logSubtitle("info", "视频时长(秒)", { videoDuration });
const { records: rawRecords } = await runAsr(audioPath, modelConfig, scriptContent);
logSubtitle("info", "ASR 原始句数", { count: rawRecords?.length ?? 0 });
let records = normalizeAsrRecords(rawRecords);
logSubtitle("info", "时间轴归一化后", {
count: records.length,
spanMs:
records.length > 1
? records[records.length - 1].end - records[0].start
: 0,
});
if (p.smartSubtitle !== false && scriptContent) {
records = alignScriptToRecords(scriptContent, records, videoDuration);
logSubtitle("info", "智能字幕对齐后", { count: records.length });
}
if (!records.length) {
const fallbackText = scriptContent || " ";
records = recordsFromTextEvenly(fallbackText, videoDuration || 30);
logSubtitle("info", "按文案均分时间轴", { count: records.length });
}
if (!records.length) {
return { success: false, error: "未识别到语音内容,无法生成字幕" };
}
const burnStyle =
p.subtitleStyle && typeof p.subtitleStyle === "object"
? p.subtitleStyle
: null;
const maxChars = burnStyle?.maxCharsPerLine || 18;
srtPath = makeOutputPath("subtitle", "srt");
fs.writeFileSync(srtPath, buildSrtContent(records, maxChars), "utf8");
if (p.recognizeOnly) {
pipelineNative.deleteFile(audioPath);
return {
success: true,
records,
srtPath,
message: `字幕识别完成,共 ${records.length}`,
};
}
const { width, height } = probeVideoSize(current);
const assPath = makeOutputPath("subtitle", "ass");
fs.writeFileSync(
assPath,
buildAssContent(records, burnStyle, width, height),
"utf8",
);
globalThis.__native?.emitProgress?.("正在烧录字幕到视频…");
current = addSubtitleToVideo(current, assPath, burnStyle);
pipelineNative.deleteFile(audioPath);
}
if (bgmEnabled) {
globalThis.__native?.emitProgress?.("正在混合背景音乐…");
current = addBgmToVideo(current, p.bgmPath, p.bgmVolume ?? 30);
}
return {
success: true,
videoPath: current,
srtPath: srtPath || null,
message: autoSubtitle && bgmEnabled
? "字幕与背景音乐处理完成"
: autoSubtitle
? "字幕生成完成"
: "背景音乐添加完成",
};
} catch (e) {
return {
success: false,
error: e.message || String(e),
partialVideo: current !== inputVideo ? current : null,
};
}
};