Docs/Skills and code/global skill script-audio-align

~/.claude/skills/script-audio-align/scripts/align.mjs

Code319 lines12.1 KBSkill: script-audio-align

// Align an authoritative script to a word-timestamped transcript (Groq/Whisper JSON).
// The script is the gold standard: every script word gets a time. The transcript only supplies times.
// Node 18+ built-ins only.
//
// Usage:
//   node align.mjs <script.txt> <transcript.json> <outDir> [--sim 0.6]
//
// Script format: plain text. A line starting with [Pn] starts paragraph n; other non-empty lines
// are paragraphs in order. Lines starting with # or --- are skipped.
//
// Outputs: <outDir>/alignment.json (machine-readable) and <outDir>/alignment.md (human breakdown).

import { readFile, writeFile, mkdir } from "node:fs/promises";
import { join } from "node:path";

const argv = process.argv.slice(2);
const pos = [];
const opts = { sim: 0.6 };
for (let i = 0; i < argv.length; i++) {
  if (argv[i] === "--sim") opts.sim = Number(argv[++i]);
  else pos.push(argv[i]);
}
const [scriptPath, transcriptPath, outDir] = pos;
if (!scriptPath || !transcriptPath || !outDir) {
  console.error("Usage: node align.mjs <script.txt> <transcript.json> <outDir> [--sim 0.6]");
  process.exit(1);
}

// ---------- Devanagari -> rough Latin, so mixed-script Whisper output still aligns ----------
const CONS = {
  "क": "k", "ख": "kh", "ग": "g", "घ": "gh", "ङ": "n",
  "च": "ch", "छ": "chh", "ज": "j", "झ": "jh", "ञ": "n",
  "ट": "t", "ठ": "th", "ड": "d", "ढ": "dh", "ण": "n",
  "त": "t", "थ": "th", "द": "d", "ध": "dh", "न": "n",
  "प": "p", "फ": "f", "ब": "b", "भ": "bh", "म": "m",
  "य": "y", "र": "r", "ल": "l", "व": "v", "श": "sh", "ष": "sh", "स": "s", "ह": "h",
  "ड़": "r", "ढ़": "rh", "फ़": "f", "ज़": "z", "क़": "q", "ख़": "kh", "ग़": "g",
};
const VOW = {
  "अ": "a", "आ": "aa", "इ": "i", "ई": "ee", "उ": "u", "ऊ": "oo", "ए": "e", "ऐ": "ai", "ओ": "o", "औ": "au",
};
const MATRA = {
  "ा": "aa", "ि": "i", "ी": "ee", "ु": "u", "ू": "oo", "े": "e", "ै": "ai", "ो": "o", "ौ": "au",
};
function deva2latin(word) {
  if (!/[ऀ-ॿ]/.test(word)) return word;
  const chars = [...word.normalize("NFC")];
  let out = "";
  for (let k = 0; k < chars.length; k++) {
    const c = chars[k];
    const next = chars[k + 1];
    if (CONS[c] !== undefined) {
      out += CONS[c];
      if (next === "़") { k++; continue; }
      if (MATRA[next] !== undefined) { out += MATRA[next]; k++; continue; }
      if (next === "्") { k++; continue; }
      // inherent 'a', but dropped at the end of a word (schwa deletion)
      if (next !== undefined && /[ऀ-ॿ]/.test(next) && !/[ंँः]/.test(next)) out += "a";
      else if (next !== undefined && /[ंँः]/.test(next)) out += "a";
    } else if (VOW[c] !== undefined) {
      out += VOW[c];
    } else if (MATRA[c] !== undefined) {
      out += MATRA[c];
    } else if (c === "ं" || c === "ँ") {
      out += "n";
    } else if (c === "ः") {
      out += "h";
    } else if (c === "़" || c === "्") {
      // nukta or virama handled above
    } else {
      out += c; // digits, punctuation, Latin letters pass through
    }
  }
  return out;
}

// ---------- normalisation: Latin spelling variants used in Hinglish / English ----------
function canon(w) {
  let s = w.normalize("NFD").replace(/[̀-ͯ]/g, "").toLowerCase();
  s = s.replace(/[^a-z0-9]/g, "");
  if (!s) return "";
  s = s
    .replace(/ph/g, "f")
    .replace(/w/g, "v")
    .replace(/sh/g, "s")
    .replace(/ck/g, "k")
    .replace(/oo|ou/g, "u")
    .replace(/ee|ea/g, "i")
    .replace(/aa/g, "a")
    .replace(/([aeiou])h/g, "$1") // toh -> to, bhi -> bi
    .replace(/(.)\1+/g, "$1"); // double letters
  return s;
}

function lev(a, b) {
  if (a === b) return 0;
  const m = a.length, n = b.length;
  if (!m) return n;
  if (!n) return m;
  let prev = new Array(n + 1);
  let cur = new Array(n + 1);
  for (let j = 0; j <= n; j++) prev[j] = j;
  for (let i = 1; i <= m; i++) {
    cur[0] = i;
    for (let j = 1; j <= n; j++) {
      const c = a[i - 1] === b[j - 1] ? 0 : 1;
      cur[j] = Math.min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + c);
    }
    [prev, cur] = [cur, prev];
  }
  return prev[n];
}
const sim = (a, b) => 1 - lev(a, b) / Math.max(a.length, b.length, 1);

// ---------- read script ----------
const scriptText = await readFile(scriptPath, "utf8");
const tokens = []; // {i, para, text, canon}
{
  let cur = 0;
  for (const raw of scriptText.split(/\r?\n/)) {
    let line = raw.trim();
    if (!line || line.startsWith("#") || line.startsWith("---")) continue;
    const m = line.match(/^\[P(\d+)\]\s*(.*)$/);
    if (m) {
      cur = Number(m[1]);
      line = m[2];
    } else {
      cur += 1;
    }
    for (const t of line.split(/\s+/).filter(Boolean)) {
      tokens.push({ i: tokens.length, para: cur, text: t, canon: canon(t) });
    }
  }
}

// ---------- read transcript ----------
const tdata = JSON.parse(await readFile(transcriptPath, "utf8"));
const twords = tdata.words ?? [];
if (!twords.length) {
  console.error("Transcript has no words array. Transcribe with word timestamps first.");
  process.exit(1);
}
const devaCount = twords.filter((w) => /[ऀ-ॿ]/.test(w.word)).length;
const heard = twords
  .map((w) => ({ text: w.word, start: w.start, end: w.end, canon: canon(deva2latin(w.word)) }))
  .filter((w) => w.canon); // only words with no letters or digits at all are dropped
const skipped = twords.length - heard.length;

// ---------- alignment: Needleman-Wunsch on script (gold) vs heard words ----------
const S = tokens.filter((t) => t.canon); // alignable script words
const n = S.length, m = heard.length;
const W = m + 1;
const GAP = -0.3;
const PAIR = (s) => s - 0.5; // pair is worth keeping only when sim > 0.5
const score = new Float32Array((n + 1) * W);
const dir = new Uint8Array((n + 1) * W); // 1 diag, 2 script-only, 3 heard-only
for (let i = 1; i <= n; i++) { score[i * W] = i * GAP; dir[i * W] = 2; }
for (let j = 1; j <= m; j++) { score[j] = j * GAP; dir[j] = 3; }
for (let i = 1; i <= n; i++) {
  for (let j = 1; j <= m; j++) {
    const d = score[(i - 1) * W + (j - 1)] + PAIR(sim(S[i - 1].canon, heard[j - 1].canon));
    const u = score[(i - 1) * W + j] + GAP;
    const l = score[i * W + (j - 1)] + GAP;
    let best = d, dr = 1;
    if (u > best) { best = u; dr = 2; }
    if (l > best) { best = l; dr = 3; }
    score[i * W + j] = best;
    dir[i * W + j] = dr;
  }
}
// traceback -> matched pairs
const scriptMatch = new Array(n).fill(null); // {j, sim}
{
  let i = n, j = m;
  while (i > 0 || j > 0) {
    const dr = dir[i * W + j];
    if (i > 0 && j > 0 && dr === 1) {
      const s = sim(S[i - 1].canon, heard[j - 1].canon);
      if (s >= opts.sim) scriptMatch[i - 1] = { j: j - 1, sim: s };
      i--; j--;
    } else if (i > 0 && (dr === 2 || j === 0)) {
      i--;
    } else {
      j--;
    }
  }
}
const matchedHeard = new Set(scriptMatch.filter(Boolean).map((x) => x.j));

// ---------- times for every script token (interpolate the unmatched ones) ----------
const lastEnd = heard.length ? heard[heard.length - 1].end : 0;
const out = new Array(tokens.length);
const alignedIdx = new Map(); // script token i -> match
{
  let k = 0;
  for (const t of tokens) {
    if (t.canon) {
      const mt = scriptMatch[k++];
      if (mt) alignedIdx.set(t.i, mt);
    }
  }
}
for (const t of tokens) {
  const mt = alignedIdx.get(t.i);
  if (mt) {
    const h = heard[mt.j];
    out[t.i] = { start: h.start, end: h.end, source: "matched", sim: +mt.sim.toFixed(2), heard: h.text };
  }
}
// fill runs of unmatched tokens between anchors, proportional to text length
let idx = 0;
while (idx < tokens.length) {
  if (out[idx]) { idx++; continue; }
  let end = idx;
  while (end < tokens.length && !out[end]) end++;
  const prevAnchor = idx > 0 ? out[idx - 1] : null;
  const nextAnchor = end < tokens.length ? out[end] : null;
  const t0 = prevAnchor ? prevAnchor.end : 0;
  const t1 = nextAnchor ? nextAnchor.start : lastEnd;
  const totalLen = tokens.slice(idx, end).reduce((a, t) => a + Math.max(1, t.text.length), 0);
  let cursor = t0;
  for (let q = idx; q < end; q++) {
    const dur = ((t1 - t0) * Math.max(1, tokens[q].text.length)) / totalLen;
    out[q] = { start: cursor, end: cursor + dur, source: "interpolated", sim: null, heard: null };
    cursor += dur;
  }
  idx = end;
}

// ---------- transcript words the script did not claim ----------
const extra = heard
  .map((h, j) => ({ ...h, j }))
  .filter((h) => !matchedHeard.has(h.j))
  .map((h) => ({ start: +h.start.toFixed(2), end: +h.end.toFixed(2), word: h.text }));

// ---------- paragraph summary ----------
const paraMap = new Map();
tokens.forEach((t, k) => {
  const r = out[k];
  if (!paraMap.has(t.para)) paraMap.set(t.para, { para: t.para, start: r.start, end: r.end, words: 0, matched: 0, first: [] });
  const p = paraMap.get(t.para);
  p.end = r.end;
  p.words++;
  if (r.source === "matched") p.matched++;
  if (p.first.length < 8) p.first.push(t.text);
});
const paragraphs = [...paraMap.values()];

// ---------- stats ----------
const matchedCount = tokens.filter((t, k) => out[k].source === "matched").length;
const sims = tokens.map((t, k) => out[k].sim).filter((s) => s != null);
const stats = {
  scriptTokens: tokens.length,
  alignableScriptTokens: n,
  matched: matchedCount,
  interpolated: tokens.length - matchedCount,
  coverage: +(matchedCount / tokens.length).toFixed(3),
  meanSimilarity: sims.length ? +(sims.reduce((a, b) => a + b, 0) / sims.length).toFixed(3) : null,
  transcriptWords: twords.length,
  transcriptSkippedNoLatin: skipped,
  transcriptDevanagari: devaCount,
  extraTranscriptWords: extra.length,
  threshold: opts.sim,
};

// ---------- write ----------
const fmt = (s) => {
  const mm = Math.floor(s / 60);
  const ss = (s - mm * 60).toFixed(2).padStart(5, "0");
  return `${String(mm).padStart(2, "0")}:${ss}`;
};

const result = {
  script: scriptPath,
  transcript: transcriptPath,
  stats,
  paragraphs: paragraphs.map((p) => ({
    para: p.para, start: +p.start.toFixed(2), end: +p.end.toFixed(2), words: p.words,
    matchedPct: +((p.matched / p.words) * 100).toFixed(1),
  })),
  words: tokens.map((t, k) => ({
    i: t.i, para: t.para, text: t.text,
    start: +out[k].start.toFixed(2), end: +out[k].end.toFixed(2),
    source: out[k].source, sim: out[k].sim, heard: out[k].heard,
  })),
  extra,
};

await mkdir(outDir, { recursive: true });
await writeFile(join(outDir, "alignment.json"), JSON.stringify(result, null, 2), "utf8");

let md = `# Alignment\n\n`;
md += `Script: \`${scriptPath}\`  \nTranscript: \`${transcriptPath}\`\n\n`;
md += `- Script words: ${stats.scriptTokens} (matched ${stats.matched}, interpolated ${stats.interpolated}, coverage ${(stats.coverage * 100).toFixed(1)}%)\n`;
md += `- Mean similarity of matched words: ${stats.meanSimilarity}\n`;
md += `- Transcript words: ${stats.transcriptWords} (no Latin letters, skipped: ${stats.transcriptSkippedNoLatin}; extra, not in script: ${stats.extraTranscriptWords})\n`;
md += `- Threshold: ${opts.sim}\n\n`;
md += `## Paragraphs\n\n| Para | Start | End | Words | Matched % | Opening |\n|---|---|---|---|---|---|\n`;
for (const p of result.paragraphs) {
  const opening = paragraphs.find((x) => x.para === p.para).first.join(" ");
  md += `| P${p.para} | ${fmt(p.start)} | ${fmt(p.end)} | ${p.words} | ${p.matchedPct} | ${opening} |\n`;
}
md += `\n## Words\n\n| # | Para | Start | End | Word | Source | Sim | Heard |\n|---|---|---|---|---|---|---|---|\n`;
for (const w of result.words) {
  md += `| ${w.i + 1} | P${w.para} | ${fmt(w.start)} | ${fmt(w.end)} | ${w.text} | ${w.source} | ${w.sim ?? ""} | ${w.heard ?? ""} |\n`;
}
md += `\n## Transcript words not in the script\n\n`;
md += extra.length ? extra.map((e) => `- ${fmt(e.start)} ${e.word}`).join("\n") + "\n" : "None.\n";
await writeFile(join(outDir, "alignment.md"), md, "utf8");

console.error(
  `Aligned ${stats.matched}/${stats.scriptTokens} script words (${(stats.coverage * 100).toFixed(1)}%), ` +
  `mean sim ${stats.meanSimilarity}, extra transcript words ${stats.extraTranscriptWords}. ` +
  `Wrote ${join(outDir, "alignment.json")} and alignment.md`
);
if (devaCount > twords.length / 2) {
  console.error("WARNING: most transcript words are Devanagari. Re-transcribe with a Latin-script --prompt-file, or the alignment will be near zero.");
}

Shown as it is in the repository at build time (2026-11-03); the home directory appears as <home>. If this page and the file differ, the file is right. Back to the catalogue.