diff --git a/scripts/nodejs/subtitle_bgm_generate.js b/scripts/nodejs/subtitle_bgm_generate.js index ebd3a03..a1be9d7 100644 --- a/scripts/nodejs/subtitle_bgm_generate.js +++ b/scripts/nodejs/subtitle_bgm_generate.js @@ -10,6 +10,15 @@ const { spawnSync } = require("child_process"); const pipelineNative = require("./pipeline_native.js"); const desktopConfig = require("./desktop_config.js"); +const { correctTextWithLevenshtein, similarity: textSimilarity } = require("./subtitle_levenshtein.js"); +const { + buildKeywordGroups, + matchKeywordsInRecords, + buildKeywordDrawtextFilter, + normalizeDisplayText, + stripKeywordsFromDisplay, + filterMatchesForRecord, +} = require("./subtitle_keyword_drawtext.js"); const OUTPUT_DIR = path.join(os.tmpdir(), "aiclient-subtitle-bgm"); @@ -123,20 +132,63 @@ function normalizeAsrTimeToMs(value) { return Math.round(n * 1000); } -function normalizeAsrRecords(records) { - if (!Array.isArray(records)) return []; - const normalized = records +/** 对齐 ZimuShengcheng.shouldTreatTimelineAsSeconds */ +function shouldTreatTimelineAsSeconds(records) { + const valid = records.filter( + (r) => Number.isFinite(Number(r?.start)) && Number.isFinite(Number(r?.end)), + ); + if (valid.length === 0) return false; + const maxEnd = Math.max(...valid.map((r) => Number(r.end))); + const maxDur = Math.max(...valid.map((r) => Number(r.end) - Number(r.start))); + return maxEnd <= 600 && maxDur <= 30; +} + +function normalizeFunasrSegmentsToMilliseconds(records) { + if (!Array.isArray(records) || records.length === 0) return []; + const mapped = records .map((record) => { const text = String(record?.text || record?.sentence || "").trim(); if (!text) return null; - let start = normalizeAsrTimeToMs( - record.start ?? record.begin ?? record.beginTime ?? 0, - ); - let end = normalizeAsrTimeToMs( - record.end ?? record.endTime ?? start + 2000, - ); + return { + text, + start: Number(record.start ?? record.begin ?? record.beginTime ?? 0), + end: Number(record.end ?? record.endTime ?? 0), + words: record.words, + }; + }) + .filter(Boolean); + + if (shouldTreatTimelineAsSeconds(mapped)) { + logSubtitle("info", "检测到秒级时间轴，统一转换为毫秒"); + return mapped.map((r) => ({ + ...r, + start: Math.round(r.start * 1000), + end: Math.round((r.end > r.start ? r.end : r.start + 2) * 1000), + words: Array.isArray(r.words) + ? r.words.map((w) => ({ + ...w, + start: Math.round(Number(w.start ?? w.begin_time ?? 0) * 1000), + end: Math.round(Number(w.end ?? w.end_time ?? 0) * 1000), + })) + : r.words, + })); + } + + return mapped.map((r) => { + let start = normalizeAsrTimeToMs(r.start); + let end = normalizeAsrTimeToMs(r.end || start + 2); + if (end <= start) end = start + 2000; + return { text: r.text, start, end, words: r.words }; + }); +} + +function normalizeAsrRecords(records) { + if (!Array.isArray(records)) return []; + const normalized = normalizeFunasrSegmentsToMilliseconds(records) + .map((record) => { + let { start, end } = record; if (end <= start) end = start + 2000; - return { text, start, end }; + return { text: record.text, start, end, words: record.words }; }) .filter(Boolean); const MIN_GAP_MS = 50; @@ -194,6 +246,36 @@ function cleanSubtitleDisplayText(text) { .trim(); } +/** + * 智能字幕：Levenshtein 校对（ZimuShengcheng），失败时回退按句均分 + */ +function applySmartSubtitle(script, records, videoDurationSec = 0) { + const normalized = normalizeAsrRecords(records); + const scriptTrim = String(script || "").trim(); + if (!scriptTrim) return normalized; + + const asrPlain = normalized.map((r) => r.text).join(""); + const sim = textSimilarity( + scriptTrim.replace(/\s+/g, ""), + asrPlain.replace(/\s+/g, ""), + ); + + if (normalized.length > 0 && sim >= 0.25) { + const { segments, similarity } = correctTextWithLevenshtein(scriptTrim, normalized); + if (segments.length > 0) { + logSubtitle("info", "Levenshtein 文案校对", { + count: segments.length, + similarity: Math.round(similarity * 1000) / 1000, + scriptSimilarity: Math.round(sim * 1000) / 1000, + }); + return segments; + } + } + + logSubtitle("info", "Levenshtein 未产出有效段落，回退按句对齐", { similarity: sim }); + return alignScriptToRecords(scriptTrim, normalized, videoDurationSec); +} + function alignScriptToRecords(script, records, videoDurationSec = 0) { const lines = splitScriptLines(script); const normalized = normalizeAsrRecords(records); @@ -337,7 +419,7 @@ function resolveAssLayout(position, videoHeight) { return { alignment: 2, marginL: 120, marginR: 120, marginV }; } -function buildAssContent(records, styleInput, videoWidth, videoHeight) { +function buildAssContent(records, styleInput, videoWidth, videoHeight, keywordMatches = []) { const style = normalizeBurnStyle(styleInput); const width = Math.max(320, Number(videoWidth) || 1920); const height = Math.max(320, Number(videoHeight) || 1080); @@ -349,13 +431,26 @@ function buildAssContent(records, styleInput, videoWidth, videoHeight) { height, ); const hasBg = style.hasBg; - const borderStyle = hasBg ? 3 : 1; - const finalOutline = hasBg ? 1 : outline; - const backColour = hasBg - ? hexToAssColor(style.backgroundColor) - : "&H00000000"; + let borderStyle = 1; + let finalOutline = outline; + let finalShadow = 0; + let backColour = "&H00000000"; const primary = hexToAssColor(style.fontColor); - const outlineColour = hexToAssColor(style.outlineColor); + let outlineColour = hexToAssColor(style.outlineColor); + + if (hasBg) { + borderStyle = 3; + finalOutline = 1; + finalShadow = 0; + backColour = hexToAssColor(style.backgroundColor); + outlineColour = backColour; + } else { + borderStyle = 1; + finalOutline = outline || 3; + finalShadow = 0; + backColour = "&H00000000"; + outlineColour = hexToAssColor(style.outlineColor); + } const header = [ "[Script Info]", @@ -368,7 +463,7 @@ function buildAssContent(records, styleInput, videoWidth, videoHeight) { "", "[V4+ Styles]", "Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding", - `Style: Default,${style.fontName},${fontSize},${primary},${primary},${outlineColour},${backColour},0,0,0,0,100,100,0,0,${borderStyle},${finalOutline},0,${alignment},${marginL},${marginR},${marginV},1`, + `Style: Default,${style.fontName},${fontSize},${primary},&H000000FF,${outlineColour},${backColour},1,0,0,0,100,100,0,0,${borderStyle},${finalOutline},${finalShadow},${alignment},${marginL},${marginR},${marginV},1`, "", "[Events]", "Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text", @@ -377,8 +472,14 @@ function buildAssContent(records, styleInput, videoWidth, videoHeight) { const events = records.map((r) => { const start = formatAssTime(r.start); const end = formatAssTime(Math.max(r.end, r.start + 500)); - const text = assEscapeText(wrapSubtitleText(r.text, style.maxCharsPerLine)); - if (!text) return ""; + const display = normalizeDisplayText(r.text); + const segMatches = filterMatchesForRecord(r, keywordMatches); + const baseDisplay = + segMatches.length > 0 + ? stripKeywordsFromDisplay(display, segMatches) + : display; + const text = assEscapeText(wrapSubtitleText(baseDisplay || " ", style.maxCharsPerLine)); + if (!text.replace(/\\N/g, "").trim()) return ""; return `Dialogue: 0,${start},${end},Default,,0,0,0,,${text}`; }).filter(Boolean); @@ -393,10 +494,16 @@ function formatAssTime(ms) { return `${h}:${String(m).padStart(2, "0")}:${String(s).padStart(2, "0")}.${String(cs).padStart(2, "0")}`; } -function addSubtitleToVideo(inputVideo, subtitlePath, styleInput) { +/** + * @param {string} [keywordDrawtextFilter] 逗号分隔的 drawtext 滤镜（关键词高亮） + */ +function addSubtitleToVideo(inputVideo, subtitlePath, keywordDrawtextFilter) { const outputVideo = makeOutputPath("with_subtitle"); const sub = escapeSubPath(subtitlePath); - const vf = `subtitles='${sub}'`; + let vf = `subtitles='${sub}'`; + if (keywordDrawtextFilter) { + vf = `${vf},${keywordDrawtextFilter}`; + } execFfmpeg(["-y", "-i", inputVideo, "-vf", vf, "-c:a", "copy", outputVideo]); return outputVideo; } @@ -521,7 +628,7 @@ globalThis.__nodejsMain = async function main(params) { : 0, }); if (p.smartSubtitle !== false && scriptContent) { - records = alignScriptToRecords(scriptContent, records, videoDuration); + records = applySmartSubtitle(scriptContent, records, videoDuration); logSubtitle("info", "智能字幕对齐后", { count: records.length }); } if (!records.length) { @@ -553,15 +660,44 @@ globalThis.__nodejsMain = async function main(params) { } const { width, height } = probeVideoSize(current); + + let keywordDrawtextFilter = null; + let keywordMatches = []; + const enableKeywords = p.enableKeywordEffects !== false; + const keywordGroups = buildKeywordGroups(p); + if (enableKeywords && keywordGroups.length > 0) { + keywordMatches = matchKeywordsInRecords(records, keywordGroups); + logSubtitle("info", "关键词匹配", { + groups: keywordGroups.length, + matches: keywordMatches.length, + keywords: keywordMatches.slice(0, 8).map((m) => m.keyword), + }); + if (keywordMatches.length > 0) { + keywordDrawtextFilter = buildKeywordDrawtextFilter(keywordMatches, { + videoWidth: width, + videoHeight: height, + subtitleStyle: burnStyle, + }); + if (!keywordDrawtextFilter) { + logSubtitle("warn", "未找到可用字体，跳过关键词 drawtext 特效"); + keywordMatches = []; + } + } + } + const assPath = makeOutputPath("subtitle", "ass"); fs.writeFileSync( assPath, - buildAssContent(records, burnStyle, width, height), + buildAssContent(records, burnStyle, width, height, keywordMatches), "utf8", ); - globalThis.__native?.emitProgress?.("正在烧录字幕到视频…"); - current = addSubtitleToVideo(current, assPath, burnStyle); + globalThis.__native?.emitProgress?.( + keywordDrawtextFilter + ? "正在烧录字幕并叠加关键词特效…" + : "正在烧录字幕到视频…", + ); + current = addSubtitleToVideo(current, assPath, keywordDrawtextFilter); pipelineNative.deleteFile(audioPath); } diff --git a/scripts/nodejs/subtitle_keyword_drawtext.js b/scripts/nodejs/subtitle_keyword_drawtext.js new file mode 100644 index 0000000..1c831f3 --- /dev/null +++ b/scripts/nodejs/subtitle_keyword_drawtext.js @@ -0,0 +1,363 @@ +"use strict"; + +/** + * 关键词高亮 drawtext（对齐 Electron ZimuShengcheng inline 模式 + pulse-scale） + */ + +const fs = require("fs"); +const path = require("path"); +const os = require("os"); + +const PUNCT_RE = + /[，。！？；：、""''（）《》【】…—·,.!?;:'"()[\]{}<>]/g; + +const DEFAULT_GROUP_META = { + focus: { + name: "重点词", + effectId: "pulse-scale", + styleConfig: { + fontColor: "#FFD700", + outlineColor: "#FF8C00", + outlineWidth: 4, + }, + }, + describe: { + name: "描述词", + effectId: "pulse-scale", + styleConfig: { + fontColor: "#5AC8FA", + outlineColor: "#007AFF", + outlineWidth: 3, + }, + }, + action: { + name: "行动词", + effectId: "pulse-scale", + styleConfig: { + fontColor: "#34C759", + outlineColor: "#248A3D", + outlineWidth: 3, + }, + }, + emotion: { + name: "情感词", + effectId: "pulse-scale", + styleConfig: { + fontColor: "#FF6B6B", + outlineColor: "#C92A2A", + outlineWidth: 3, + }, + }, +}; + +function parseKeywordLines(text) { + return String(text || "") + .split(/[\n,，、]+/) + .map((s) => s.trim()) + .filter(Boolean); +} + +/** + * 从步骤 04 字段或显式 keywordGroups 构建分组 + * @param {Record} params + */ +function buildKeywordGroups(params) { + if (Array.isArray(params.keywordGroups) && params.keywordGroups.length > 0) { + return params.keywordGroups + .map((g) => normalizeKeywordGroup(g)) + .filter((g) => g.keywords.length > 0); + } + + const sources = [ + ["focus", params.keywordsFocus], + ["describe", params.keywordsDescribe], + ["action", params.keywordsAction], + ["emotion", params.keywordsEmotion], + ]; + + return sources + .map(([id, raw]) => { + const meta = DEFAULT_GROUP_META[id] || DEFAULT_GROUP_META.focus; + const keywords = parseKeywordLines(raw); + if (!keywords.length) return null; + return { + id, + name: meta.name, + effectId: meta.effectId, + styleConfig: { ...meta.styleConfig }, + keywords, + }; + }) + .filter(Boolean); +} + +function normalizeKeywordGroup(group) { + const g = group && typeof group === "object" ? group : {}; + const keywords = (Array.isArray(g.keywords) ? g.keywords : []) + .map((k) => (typeof k === "string" ? k : String(k?.keyword || "")).trim()) + .filter(Boolean); + return { + id: String(g.id || g.groupId || "group"), + name: String(g.name || g.groupName || "关键词"), + effectId: String(g.effectId || g.effectConfig?.type || "pulse-scale"), + styleConfig: g.styleConfig || g.styleOverride || {}, + keywords, + }; +} + +function normalizeDisplayText(text) { + return String(text || "") + .replace(PUNCT_RE, "") + .replace(/\s+/g, ""); +} + +function normalizeEffectType(effectId) { + const id = String(effectId || "pulse-scale").toLowerCase(); + if (id.includes("pulse") || id.includes("scale")) return "pulse-scale"; + if (id.includes("slide")) return "slide"; + return "highlight"; +} + +function buildKeywordStyle(group, keywordItem) { + const base = group.styleConfig || {}; + const override = + keywordItem && typeof keywordItem === "object" ? keywordItem.styleOverride : null; + const o = override || base; + return { + fontColor: o.fontColor || "#FFD700", + outlineColor: o.outlineColor || "#FF8C00", + outlineWidth: Number(o.outlineWidth) ?? 4, + fontSize: Number(o.fontSize) || 0, + }; +} + +/** + * @param {Array<{ text: string, start: number, end: number }>} records 毫秒 + * @param {ReturnType} groups + */ +function matchKeywordsInRecords(records, groups) { + const matches = []; + const seen = new Set(); + + for (const record of records) { + const display = normalizeDisplayText(record.text); + if (!display) continue; + + const segStartSec = record.start / 1000; + const segEndSec = record.end / 1000; + const dur = Math.max(0.05, segEndSec - segStartSec); + + for (const group of groups) { + for (const kw of group.keywords) { + const raw = typeof kw === "string" ? kw : String(kw?.keyword || ""); + const needle = normalizeDisplayText(raw); + if (!needle) continue; + + let searchFrom = 0; + while (searchFrom < display.length) { + const idx = display.indexOf(needle, searchFrom); + if (idx === -1) break; + + const dedupeKey = `${segStartSec.toFixed(3)}-${segEndSec.toFixed(3)}-${needle}-${idx}`; + if (!seen.has(dedupeKey)) { + seen.add(dedupeKey); + const ratioStart = idx / display.length; + const ratioEnd = (idx + needle.length) / display.length; + const startSec = segStartSec + dur * ratioStart; + const endSec = Math.max(startSec + 0.08, segStartSec + dur * ratioEnd); + + matches.push({ + keyword: needle, + text: needle, + lineText: display, + lineLength: display.length, + occurrenceStart: idx, + occurrenceEnd: idx + needle.length, + startSec, + endSec, + groupId: group.id, + groupName: group.name, + effectType: normalizeEffectType( + (typeof kw === "object" && kw?.effectId) || group.effectId, + ), + style: buildKeywordStyle(group, kw), + }); + } + searchFrom = idx + Math.max(1, needle.length); + } + } + } + } + + return matches.sort((a, b) => a.startSec - b.startSec); +} + +const WIN_FONT_MAP = { + "microsoft yahei": ["msyh.ttc", "msyhbd.ttc"], + "microsoft yahei bold": ["msyhbd.ttc", "msyh.ttc"], + simhei: ["simhei.ttf"], + "sim sun": ["simsun.ttc"], + "source han sans": ["SourceHanSansSC-Regular.otf"], +}; + +function resolveFontFile(fontName) { + const fromEnv = process.env.AICLIENT_SUBTITLE_FONT_PATH; + if (fromEnv && fs.existsSync(fromEnv)) return fromEnv; + + const key = String(fontName || "Microsoft YaHei") + .trim() + .toLowerCase(); + const candidates = WIN_FONT_MAP[key] || WIN_FONT_MAP["microsoft yahei"]; + + if (process.platform === "win32") { + const fontsDir = path.join(process.env.WINDIR || "C:\\Windows", "Fonts"); + for (const file of candidates) { + const full = path.join(fontsDir, file); + if (fs.existsSync(full)) return full; + } + } + + const fallbacks = [ + "/usr/share/fonts/truetype/noto/NotoSansCJK-Regular.ttc", + "/System/Library/Fonts/PingFang.ttc", + "/System/Library/Fonts/Supplemental/Arial Unicode.ttf", + ]; + for (const p of fallbacks) { + if (fs.existsSync(p)) return p; + } + return null; +} + +function escapeDrawtextPath(filePath) { + return filePath + .replace(/\\/g, "/") + .replace(/:/g, "\\:") + .replace(/'/g, "\\'"); +} + +function escapeDrawtextText(text) { + return String(text || "") + .replace(/\\/g, "\\\\") + .replace(/'/g, "\\'") + .replace(/:/g, "\\:") + .replace(/,/g, "\\,"); +} + +function hexToDrawtextColor(hex) { + const h = String(hex || "#FFFFFF").replace("#", "").slice(0, 6); + if (h.length !== 6) return "white"; + return `0x${h.toUpperCase()}`; +} + +function resolveSubtitleY(position, videoHeight) { + const h = Math.max(360, Number(videoHeight) || 1080); + if (position === "top") return "h*0.12"; + if (position === "center") return "(h-text_h)/2"; + return "h*0.82-text_h"; +} + +/** + * @param {ReturnType[number]} match + * @param {{ videoWidth: number, videoHeight: number, baseFontSize: number, fontfile: string, position: string }} layout + */ +function buildDrawtextForMatch(match, layout) { + const { videoWidth, videoHeight, baseFontSize, fontfile, position } = layout; + const style = match.style || {}; + const baseSize = style.fontSize > 0 ? style.fontSize : Math.round(baseFontSize * 1.45); + const outlineW = Math.max(1, Math.round(style.outlineWidth || 3)); + const fontColor = hexToDrawtextColor(style.fontColor); + const borderColor = hexToDrawtextColor(style.outlineColor); + + const lineLen = Math.max(1, match.lineLength); + const charWidth = baseSize * 0.55; + const lineWidth = Math.min(lineLen * charWidth, videoWidth * 0.88); + const keywordMid = (match.occurrenceStart + match.occurrenceEnd) / 2; + const offsetPx = (keywordMid / lineLen) * lineWidth - lineWidth / 2; + const xExpr = `(w-text_w)/2+${offsetPx.toFixed(1)}`; + const yExpr = resolveSubtitleY(position, videoHeight); + + const t0 = match.startSec.toFixed(3); + const t1 = match.endSec.toFixed(3); + const enable = `between(t\\,${t0}\\,${t1})`; + + let fontsizePart = String(baseSize); + if (match.effectType === "pulse-scale") { + fontsizePart = `${baseSize}*(1+0.1*sin(2*PI*(t-${t0})*4))`; + } + + const parts = [ + `fontfile='${escapeDrawtextPath(fontfile)}'`, + `text='${escapeDrawtextText(match.text)}'`, + `fontsize='${fontsizePart}'`, + `fontcolor=${fontColor}`, + `borderw=${outlineW}`, + `bordercolor=${borderColor}`, + `x='${xExpr}'`, + `y='${yExpr}'`, + `enable='${enable}'`, + ]; + return `drawtext=${parts.join(":")}`; +} + +/** + * @param {ReturnType} matches + * @param {{ videoWidth: number, videoHeight: number, subtitleStyle?: Record }} opts + * @returns {string | null} ffmpeg -vf 链（逗号分隔的 drawtext） + */ +function buildKeywordDrawtextFilter(matches, opts) { + if (!matches.length) return null; + + const videoWidth = Math.max(320, Number(opts.videoWidth) || 1920); + const videoHeight = Math.max(320, Number(opts.videoHeight) || 1080); + const style = opts.subtitleStyle || {}; + const scale = Math.max(0.2, Math.min(2, (Number(style.fontSizeScale) || 100) / 100)); + const baseFontSize = Math.max(14, Math.round((Number(style.fontSize) || 35) * scale)); + const fontfile = resolveFontFile(style.fontName); + if (!fontfile) return null; + + const layout = { + videoWidth, + videoHeight, + baseFontSize, + fontfile, + position: String(style.position || "bottom"), + }; + + return matches.map((m) => buildDrawtextForMatch(m, layout)).join(","); +} + +/** + * 从 ASS 底字中移除关键词字符，避免与 drawtext 高亮重复 + * @param {string} displayText 已去标点文本 + * @param {Array<{ occurrenceStart: number, occurrenceEnd: number }>} matches + */ +function stripKeywordsFromDisplay(displayText, matches) { + const text = String(displayText || ""); + if (!text || !matches.length) return text; + const chars = [...text]; + for (const m of matches) { + const start = Math.max(0, m.occurrenceStart); + const end = Math.min(chars.length, m.occurrenceEnd); + for (let i = start; i < end; i++) chars[i] = ""; + } + return chars.join("").trim(); +} + +/** @param {{ start: number, end: number }} record 毫秒 */ +function filterMatchesForRecord(record, matches) { + const segStart = record.start / 1000; + const segEnd = record.end / 1000; + return matches.filter( + (m) => m.startSec >= segStart - 0.02 && m.endSec <= segEnd + 0.02, + ); +} + +module.exports = { + buildKeywordGroups, + matchKeywordsInRecords, + buildKeywordDrawtextFilter, + normalizeDisplayText, + stripKeywordsFromDisplay, + filterMatchesForRecord, + resolveFontFile, +}; diff --git a/scripts/nodejs/subtitle_levenshtein.js b/scripts/nodejs/subtitle_levenshtein.js new file mode 100644 index 0000000..ce50aef --- /dev/null +++ b/scripts/nodejs/subtitle_levenshtein.js @@ -0,0 +1,265 @@ +"use strict"; + +/** + * 文案与 ASR 时间轴对齐（对齐 Electron ZimuShengcheng / Levenshtein） + */ + +const PUNCTUATION_RE = + /[，。！？、；：\u201C\u201D"「」『』（）【】《》,.!?;:'"]/; + +const DEFAULT_CHAR_DURATION_MS = 150; +const TIME_GAP_SPLIT_THRESHOLD_MS = 500; + +function levenshteinDistance(a, b) { + const n = a.length; + const m = b.length; + const dp = Array.from({ length: n + 1 }, () => Array(m + 1).fill(0)); + for (let i = 0; i <= n; i++) dp[i][0] = i; + for (let j = 0; j <= m; j++) dp[0][j] = j; + for (let i = 1; i <= n; i++) { + for (let j = 1; j <= m; j++) { + const cost = a[i - 1] === b[j - 1] ? 0 : 1; + dp[i][j] = Math.min(dp[i - 1][j] + 1, dp[i][j - 1] + 1, dp[i - 1][j - 1] + cost); + } + } + return dp[n][m]; +} + +function findBestMatch(userText, asrText) { + const n = userText.length; + const m = asrText.length; + const dp = Array.from({ length: n + 1 }, () => Array(m + 1).fill(0)); + const bt = Array.from({ length: n + 1 }, () => Array(m + 1).fill("")); + for (let i = 0; i <= n; i++) { + dp[i][0] = i; + bt[i][0] = "delete"; + } + for (let j = 0; j <= m; j++) { + dp[0][j] = j; + bt[0][j] = "insert"; + } + for (let i = 1; i <= n; i++) { + for (let j = 1; j <= m; j++) { + const cost = userText[i - 1] === asrText[j - 1] ? 0 : 1; + const del = dp[i - 1][j] + 1; + const ins = dp[i][j - 1] + 1; + const sub = dp[i - 1][j - 1] + cost; + const best = Math.min(del, ins, sub); + dp[i][j] = best; + if (best === del) bt[i][j] = "delete"; + else if (best === ins) bt[i][j] = "insert"; + else if (cost === 0) bt[i][j] = "match"; + else bt[i][j] = "replace"; + } + } + const alignments = []; + let i = n; + let j = m; + while (i > 0 || j > 0) { + const op = bt[i][j]; + if (op === "match") { + alignments.unshift({ + userChar: userText[i - 1], + asrChar: asrText[j - 1], + operation: "match", + asrIndex: j - 1, + }); + i -= 1; + j -= 1; + } else if (op === "replace") { + alignments.unshift({ + userChar: userText[i - 1], + asrChar: asrText[j - 1], + operation: "replace", + asrIndex: j - 1, + }); + i -= 1; + j -= 1; + } else if (op === "delete") { + alignments.unshift({ + userChar: userText[i - 1], + asrChar: "", + operation: "delete", + asrIndex: j, + }); + i -= 1; + } else { + alignments.unshift({ + userChar: "", + asrChar: asrText[j - 1], + operation: "insert", + asrIndex: j - 1, + }); + j -= 1; + } + } + const distance = dp[n][m]; + const similarity = 1 - distance / Math.max(n, m, 1); + return { alignments, distance, similarity }; +} + +function buildSegmentCharTimings(segment) { + const text = segment.text || ""; + const len = text.length; + if (len === 0) return []; + const step = (segment.end - segment.start) / Math.max(len, 1); + const fallback = Array.from({ length: len }, (_, idx) => ({ + start: segment.start + idx * step, + end: segment.start + (idx + 1) * step, + })); + const words = Array.isArray(segment.words) ? segment.words : []; + if (words.length === 0) return fallback; + + const mapped = Array(len).fill(null); + let searchFrom = 0; + for (const w of words) { + const word = String(w?.word ?? w?.text ?? ""); + const start = Number(w?.start ?? w?.beginTime ?? w?.begin_time); + const end = Number(w?.end ?? w?.endTime ?? w?.end_time); + if (!word || !Number.isFinite(start) || !Number.isFinite(end) || end <= start) continue; + const pos = text.indexOf(word, searchFrom); + if (pos === -1) continue; + const wordStep = (end - start) / Math.max(word.length, 1); + for (let k = 0; k < word.length && pos + k < len; k++) { + mapped[pos + k] = { start: start + k * wordStep, end: start + (k + 1) * wordStep }; + } + searchFrom = pos + word.length; + } + return mapped.map((item, idx) => item || fallback[idx]); +} + +function buildCharTimeMap(segments) { + const map = new Map(); + let index = 0; + for (const segment of segments) { + for (const timing of buildSegmentCharTimings(segment)) { + map.set(index, timing); + index += 1; + } + } + return map; +} + +function mapToTimestamps(alignments, charMap) { + const segments = []; + let text = ""; + let start = 0; + let end = 0; + let corrected = false; + let isFirstChar = true; + + for (let y = 0; y < alignments.length; y++) { + const row = alignments[y]; + const timing = + row.asrIndex != null && row.asrIndex >= 0 ? charMap.get(row.asrIndex) : null; + const isPunct = row.userChar && PUNCTUATION_RE.test(row.userChar); + + if (row.operation === "match" || row.operation === "replace") { + if (!isPunct) text += row.userChar; + if (row.operation === "replace") corrected = true; + if (timing) { + if (isFirstChar) { + start = timing.start; + isFirstChar = false; + } + end = timing.end; + } + } else if (row.operation === "insert") { + if (timing) { + if (isFirstChar) { + start = timing.start; + isFirstChar = false; + } + end = timing.end; + } + } else if (row.operation === "delete") { + if (!isPunct) { + text += row.userChar; + const t = isFirstChar ? start : end; + if (isFirstChar) { + start = t; + isFirstChar = false; + } + if (!timing) end = t + DEFAULT_CHAR_DURATION_MS; + } + corrected = true; + } + + const isLast = y === alignments.length - 1; + const nextRow = alignments[y + 1]; + const nextTiming = + nextRow && nextRow.asrIndex != null && nextRow.asrIndex >= 0 + ? charMap.get(nextRow.asrIndex) + : null; + const gapSplit = + !isLast && timing && nextTiming && nextTiming.start - end > TIME_GAP_SPLIT_THRESHOLD_MS; + + if ((isLast || isPunct || gapSplit) && text.length > 0) { + if (end <= start) end = start + text.length * DEFAULT_CHAR_DURATION_MS; + segments.push({ + text, + start, + end, + corrected, + }); + start = end; + text = ""; + corrected = false; + isFirstChar = true; + } + } + + if (text.length > 0) { + if (end <= start) end = start + text.length * DEFAULT_CHAR_DURATION_MS; + segments.push({ text, start, end, corrected }); + } + return segments; +} + +/** + * @param {string} userText + * @param {Array<{ text: string, start: number, end: number, words?: unknown[] }>} asrSegments + */ +function correctTextWithLevenshtein(userText, asrSegments) { + const user = String(userText || "").replace(/\s+/g, ""); + const list = Array.isArray(asrSegments) ? asrSegments : []; + if (!user || list.length === 0) { + return { segments: [], similarity: 0 }; + } + + const asrFullText = list.map((s) => s.text || "").join(""); + const match = findBestMatch(user, asrFullText); + const charMap = buildCharTimeMap(list); + const corrected = mapToTimestamps(match.alignments, charMap); + + const MIN_GAP_MS = 50; + const out = corrected + .map((seg) => ({ + text: String(seg.text || "").trim(), + start: Math.round(seg.start), + end: Math.round(seg.end), + })) + .filter((seg) => seg.text); + + for (let i = 0; i < out.length - 1; i++) { + const cur = out[i]; + const next = out[i + 1]; + if (cur.end >= next.start) { + cur.end = Math.max(cur.start + MIN_GAP_MS, next.start - MIN_GAP_MS); + } + if (cur.end <= cur.start) cur.end = cur.start + MIN_GAP_MS; + } + + return { segments: out, similarity: match.similarity }; +} + +function similarity(a, b) { + const max = Math.max(a.length, b.length); + if (max === 0) return 1; + return 1 - levenshteinDistance(a, b) / max; +} + +module.exports = { + correctTextWithLevenshtein, + similarity, +};