Docs/Skills and code/skill video-visual-production

.claude/skills/video-visual-production/scripts/align-beat.js

Code102 lines7.1 KBSkill: video-visual-production

#!/usr/bin/env node
// Align one beat's voice-over to its script words in one command: Groq Whisper (groq-transcribe) + script-audio-align + conversion.
//   node align-beat.js --beat B3 [--slug moser-baer-v7-hinglish] [--topic the-fall-of-moser-baer] [--min 85]
// Attempts, in order, until the share of script words matched reaches --min: (1) whisper-large-v3-turbo on the whole beat,
// (2) whisper-large-v3 on the whole beat, (3) the beat cut into ~30 s chunks at silences, each transcribed separately and merged
// (Whisper sometimes drops a stretch of a long file). The best attempt is used; under --min it stops with FAIL.
// Reads:  research/<topic>/script/audio/gemini-v7-hinglish/beats/<beat>-*.wav and video/timing/reference-words.json
// Writes: research/<topic>/video/timing/words-<beat>.json (local times, offset 0) and C:/video-build/<slug>/<beat>/align/*
// Sends the beat's audio to Groq's hosted Whisper (key from the groq-transcribe skill's secret file; never printed).
const fs = require('fs');
const path = require('path');
const U = require('./lib/util');

const beat = U.arg('beat', null);
if (!beat) U.die('usage: align-beat.js --beat B3');
const topic = U.arg('topic', 'the-fall-of-moser-baer'), slug = U.arg('slug', 'moser-baer-v7-hinglish'), min = parseFloat(U.arg('min', '85'));
const home = process.env.USERPROFILE || process.env.HOME;
const TRANSCRIBE = path.join(home, '.claude', 'skills', 'groq-transcribe', 'scripts', 'transcribe.mjs');
const ALIGN = path.join(home, '.claude', 'skills', 'script-audio-align', 'scripts', 'align.mjs');
for (const f of [TRANSCRIBE, ALIGN]) if (!fs.existsSync(f)) U.die('missing skill script ' + f);

const beatDir = path.join(U.ROOT, 'research', topic, 'script', 'audio', 'gemini-v7-hinglish', 'beats');
const wav = fs.readdirSync(beatDir).find((f) => f.startsWith(beat + '-') && f.endsWith('.wav'));
if (!wav) U.die(`no wav for ${beat} in ${beatDir}`);
const wavPath = path.join(beatDir, wav);
const refPath = path.join(U.ROOT, 'research', topic, 'video', 'timing', 'reference-words.json');
const beats = U.readJSON(refPath).beats;
const ref = beats.find((b) => b.beat === beat);
if (!ref) U.die('reference-words.json has no ' + beat);
const work = path.join(U.buildDir(slug, beat), 'align');
fs.mkdirSync(work, { recursive: true });
const NL = String.fromCharCode(10);

// 1. a Latin prompt from a DIFFERENT beat (a prompt from the same beat would make the match circular)
const other = beats.find((b) => b.beat === (beat === 'B1' ? 'B3' : 'B1'));
const promptFile = path.join(work, 'prompt.txt');
fs.writeFileSync(promptFile, other.words.slice(0, 40).map((w) => w.text).join(' '));
// 2. the script, one line per paragraph, from the words that were sent to the voice
const paras = []; let cur = [];
ref.words.forEach((w) => { cur.push(w.text); if (w.para_end) { paras.push(cur.join(' ')); cur = []; } });
if (cur.length) paras.push(cur.join(' '));
fs.writeFileSync(path.join(work, 'script.txt'), paras.map((p, i) => `[P${i + 1}] ${p}`).join(NL));

const run = (args) => { const r = U.run(process.execPath, args, { stdio: ['ignore', 'pipe', 'pipe'] }); if (r.status !== 0) U.die((r.stderr || r.stdout || '').slice(-600)); return (r.stdout || '') + (r.stderr || ''); };
const transcribe = (audio, outFile, model) => run([TRANSCRIBE, audio, outFile, '--lang', 'hi', '--model', model, '--prompt-file', promptFile]);
const align = (transcript, outDir) => {
  const o = run([ALIGN, path.join(work, 'script.txt'), transcript, outDir]).trim();
  U.say(o.split(NL).filter((l) => l.startsWith('Aligned')).join(NL));
  return parseFloat((o.match(/\(([\d.]+)%\)/) || [])[1]);
};

// cut points for the chunked attempt: the quietest moments nearest to even steps of about 30 s
function chunkPlan() {
  const dur = parseFloat(U.mustRun('ffprobe', ['-v', 'error', '-show_entries', 'format=duration', '-of', 'default=nw=1:nk=1', wavPath]).stdout);
  const sd = U.run('ffmpeg', ['-hide_banner', '-nostats', '-i', wavPath, '-af', 'silencedetect=n=-35dB:d=0.25', '-f', 'null', '-']).stderr;
  const starts = [...sd.matchAll(/silence_start: ([\d.]+)/g)].map((m) => +m[1]), ends = [...sd.matchAll(/silence_end: ([\d.]+)/g)].map((m) => +m[1]);
  const mids = starts.map((s, i) => (ends[i] ? (s + ends[i]) / 2 : s)).filter((m) => m > 8 && m < dur - 8);
  const n = Math.max(2, Math.round(dur / 30)), cuts = [0];
  for (let k = 1; k < n; k++) { const target = (dur * k) / n; const c = mids.length ? mids.reduce((a, b) => (Math.abs(b - target) < Math.abs(a - target) ? b : a)) : target; if (c > cuts[cuts.length - 1] + 5) cuts.push(c); }
  cuts.push(dur);
  return cuts;
}
function chunked(model) {
  const cuts = chunkPlan(), all = [];
  for (let i = 0; i + 1 < cuts.length; i++) {
    const a = cuts[i], b = cuts[i + 1], f = path.join(work, `chunk${i}.wav`), j = path.join(work, `chunk${i}.json`);
    U.mustRun('ffmpeg', ['-y', '-v', 'error', '-i', wavPath, '-ss', a.toFixed(3), '-t', (b - a).toFixed(3), '-ar', '24000', '-ac', '1', f]);
    transcribe(f, j, model);
    for (const w of U.readJSON(j).words || []) all.push({ ...w, start: +(w.start + a).toFixed(3), end: +(w.end + a).toFixed(3) });
  }
  const out = path.join(work, `${beat}.groq.chunked.json`);
  U.writeJSON(out, { task: 'transcribe', language: 'hi', duration: cuts[cuts.length - 1], text: all.map((w) => w.word).join(' '), words: all, segments: [] });
  U.say(`chunked: ${cuts.length - 1} chunks cut at ${cuts.slice(1, -1).map((c) => c.toFixed(1) + 's').join(', ')}; ${all.length} words`);
  return out;
}

let best = null;
const attempts = [
  ['whisper-large-v3-turbo, whole beat', () => { const t = path.join(work, `${beat}.groq.turbo.json`); transcribe(wavPath, t, 'whisper-large-v3-turbo'); return t; }],
  ['whisper-large-v3, whole beat', () => { const t = path.join(work, `${beat}.groq.large.json`); transcribe(wavPath, t, 'whisper-large-v3'); return t; }],
  ['whisper-large-v3-turbo, chunked at silences', () => chunked('whisper-large-v3-turbo')],
];
attempts.forEach(() => 0);
for (let i = 0; i < attempts.length; i++) {
  const [label, go] = attempts[i];
  U.say(`attempt ${i + 1}: ${label}`);
  const t = go(), odir = path.join(work, `out-${i + 1}`), pct = align(t, odir);
  if (!best || pct > best.pct) best = { pct, label, odir };
  if (pct >= min) break;
  U.say(`   ${pct}% is under ${min}%` + (i + 1 < attempts.length ? '; trying the next attempt' : ''));
}
if (!(best.pct >= min)) U.die(`only ${best.pct}% of words matched after all attempts (best: ${best.label}; minimum ${min}%). Stop and tell the user: the alignment output is in ${work}`);
U.say(`using: ${best.label} (${best.pct}%)`);
fs.cpSync(best.odir, path.join(work, 'out'), { recursive: true });

// 3. convert to the pipeline's words file
const out = path.join('research', topic, 'video', 'timing', `words-${beat}.json`).split(path.sep).join('/');
const r = U.run(process.execPath, [path.join(__dirname, 'words-from-alignment.js'), '--beat', beat, '--alignment', path.join(work, 'out', 'alignment.json'), '--reference', path.relative(U.ROOT, refPath).split(path.sep).join('/'), '--out', out]);
if (r.status !== 0) U.die(r.stderr || r.stdout);
U.say(r.stdout.trim());
U.say(`audio: ${wav}${NL}words file: ${out}${NL}use in the beat sheet: "words": "${out}", "time_offset_s": 0`);

Shown as it is in the repository at build time (2026-11-03); the home directory appears as <home>. If this page and the file differ, the file is right. Back to the catalogue.