Docs/Skills and code/skill video-visual-production

.claude/skills/video-visual-production/scripts/mix.js

Code120 lines6.9 KBSkill: video-visual-production

#!/usr/bin/env node
// V7 Mix: voice (upsampled) + synthesised effects + a quiet synthesised bed ducked from the word times + room tone,
// then two-pass loudnorm to -14 LUFS, true peak at most -1 dBTP.
//   node mix.js --sheet <beat-sheet.json> --timeline <timeline.json> [--no-bed]
const fs = require('fs');
const path = require('path');
const U = require('./lib/util');
const D = require('./lib/dsp');
const SFX = require('./sfx');
const { SR } = D;

function bedTrack(samples, speech, voiceRms, levelDb, total_s) {
  // drone: A minor-ish partials with slow independent amplitude movement, then a gentle low-pass
  const partials = [[110, 1.0, 0.07], [164.81, 0.6, 0.05], [220, 0.5, 0.09], [261.63, 0.22, 0.06], [330, 0.12, 0.04]];
  const out = new Float32Array(samples);
  const lp = D.onePoleLP(700);
  for (let i = 0; i < samples; i++) {
    const t = i / SR;
    let v = 0;
    for (const [f, w, lfo] of partials) {
      v += w * (Math.sin(2 * Math.PI * f * t) + Math.sin(2 * Math.PI * (f + 0.35) * t)) * (0.65 + 0.35 * Math.sin(2 * Math.PI * lfo * t + f));
    }
    out[i] = lp(v);
  }
  // duck from the word times: 1.0 while a word is being spoken, up to +4 dB in pauses; attack 80 ms, release 400 ms
  const ctrl = 100; // control rate Hz
  const nC = Math.ceil(samples / SR * ctrl);
  const active = new Uint8Array(nC);
  for (const [s, e] of speech) for (let c = Math.floor((s - 0.1) * ctrl); c <= Math.ceil((e + 0.15) * ctrl); c++) if (c >= 0 && c < nC) active[c] = 1;
  const g = new Float32Array(nC);
  let cur = 1;
  const aA = 1 - Math.exp(-1 / (0.08 * ctrl)), aR = 1 - Math.exp(-1 / (0.4 * ctrl));
  for (let c = 0; c < nC; c++) { const target = active[c] ? 1 : 1.6; cur += (target - cur) * (target < cur ? aA : aR); g[c] = cur; }
  const fadeIn = 1.5, fadeOut = 2.2;
  for (let i = 0; i < samples; i++) {
    const t = i / SR, c = Math.min(nC - 1, Math.floor(t * ctrl));
    out[i] *= g[c] * Math.min(1, t / fadeIn) * Math.min(1, Math.max(0, (total_s - t) / fadeOut));
  }
  // scale so that the bed's RMS while speech is active sits `levelDb` below the voice's RMS
  let s = 0, n = 0;
  for (let i = 0; i < samples; i += 7) { if (active[Math.min(nC - 1, Math.floor(i / SR * ctrl))]) { s += out[i] * out[i]; n++; } }
  const bedRms = Math.sqrt(s / Math.max(1, n)) || 1;
  const k = (voiceRms * D.db(levelDb)) / bedRms;
  for (let i = 0; i < samples; i++) out[i] *= k;
  return out;
}

function ffmpegF32(file) {
  const r = U.run('ffmpeg', ['-v', 'error', '-i', file, '-ar', String(SR), '-ac', '1', '-f', 'f32le', '-'], { encoding: 'buffer' });
  if (r.status !== 0) throw new Error('ffmpeg decode failed: ' + r.stderr.toString().slice(-300));
  return D.f32FromBuffer(r.stdout);
}

// Loudness (EBU R128) and true peak of a file, from ffmpeg's ebur128 summary
function measure(file) {
  const r = U.run('ffmpeg', ['-hide_banner', '-nostats', '-i', file, '-af', 'ebur128=peak=true', '-f', 'null', '-']);
  const t = r.stderr.slice(r.stderr.lastIndexOf('Summary:'));
  const num = (re) => { const m = t.match(re); return m ? parseFloat(m[1]) : NaN; };
  return { I: num(/I:\s+(-?[0-9.]+) LUFS/), LRA: num(/LRA:\s+(-?[0-9.]+) LU/), TP: num(/Peak:\s+(-?[0-9.]+) dBFS/) };
}

// Gain to the target loudness, then a look-ahead limiter below the true-peak ceiling; iterate because limiting lowers loudness.
// (loudnorm's linear mode refuses when the voice has high peaks and low loudness, as this synthetic voice does.)
function normalize(inFile, outFile, I = -14, TP = -1) {
  const m0 = measure(inFile);
  let gain = I - m0.I, last = null, limited = 0;
  for (let it = 0; it < 5; it++) {
    const af = 'volume=' + gain.toFixed(2) + 'dB,alimiter=limit=0.84:attack=3:release=70:level=0';
    U.mustRun('ffmpeg', ['-y', '-v', 'error', '-i', inFile, '-af', af, '-ar', String(SR), '-ac', '2', '-c:a', 'pcm_s16le', outFile]);
    last = measure(outFile);
    if (Math.abs(last.I - I) <= 0.25 && last.TP <= TP) break;
    gain += I - last.I;
    if (last.TP > TP) gain -= (last.TP - TP) * 0.5;
  }
  limited = +(m0.I + gain - last.I).toFixed(1);
  return { before: m0, after: last, gain_db: +gain.toFixed(2), loudness_given_up_to_limiter_db: limited };
}

function main() {
  const sheetPath = U.fromRoot(U.arg('sheet', ''));
  const sheet = U.readJSON(sheetPath);
  const tl = U.readJSON(U.fromRoot(U.arg('timeline', path.join(path.dirname(sheetPath), `timeline-${sheet.beat}.json`))));
  const outDir = path.join(U.buildDir(sheet.slug, sheet.beat), 'audio');
  fs.mkdirSync(outDir, { recursive: true });
  const samples = tl.total_frames * (SR / tl.fps);
  const voice = ffmpegF32(U.fromRoot(sheet.audio));
  const track = new Float32Array(samples);
  const off = Math.round(tl.lead_in_s * SR);
  for (let i = 0; i < voice.length && off + i < samples; i++) track[off + i] = voice[i];
  const vPeak = D.peak(track), vRms = D.rms(track, off, off + voice.length);
  const { track: sfx, table } = SFX.placeEvents(tl.sfx, samples, vPeak, sheet.levels || {});
  const useBed = sheet.bed && sheet.bed.enabled && !U.flag('no-bed');
  const bed = useBed ? bedTrack(samples, tl.speech, vRms, sheet.bed.level_db, samples / SR) : new Float32Array(samples);
  // room tone: very quiet low-passed noise so silence is never digital zero
  const r = D.rng(7), lp = D.onePoleLP(1200), room = new Float32Array(samples);
  for (let i = 0; i < samples; i++) room[i] = lp(r() * 2 - 1) * D.db(-62) * 3;
  const mix = new Float32Array(samples);
  for (let i = 0; i < samples; i++) mix[i] = track[i] + sfx[i] + bed[i] + room[i];
  // stems for the user's own remix
  // stems are 32-bit float: the voice peaks above full scale before the final limiter, a 16-bit stem would clip it
  D.writeWav(path.join(outDir, 'voice.wav'), track, SR, 32);
  D.writeWav(path.join(outDir, 'sfx.wav'), sfx, SR, 32);
  D.writeWav(path.join(outDir, 'bed.wav'), bed, SR, 16);
  const pre = path.join(outDir, 'premix.wav');
  D.writeWav(pre, mix, SR, 32);
  const mixOut = path.join(outDir, 'mix.wav');
  const nz = normalize(pre, mixOut);
  const m = { input_i: nz.before.I, input_tp: nz.before.TP, input_lra: nz.before.LRA };
  fs.unlinkSync(pre);
  const report = {
    voice_peak_dbfs: +(20 * Math.log10(vPeak)).toFixed(1), voice_rms_dbfs: +(20 * Math.log10(vRms)).toFixed(1),
    bed_enabled: !!useBed, bed_label: useBed ? sheet.bed.label : null, bed_level_db_below_voice_rms: useBed ? sheet.bed.level_db : null,
    premix_measured: { lufs: m.input_i, true_peak_dbtp: m.input_tp, lra: m.input_lra }, normalised: nz, sfx_table: table,
  };
  U.writeJSON(path.join(outDir, 'mix-report.json'), report);
  U.say(`mix: normalised to ${nz.after.I} LUFS, true peak ${nz.after.TP} dBFS (gain ${nz.gain_db} dB, limiter cost about ${nz.loudness_given_up_to_limiter_db} dB); voice peak ${report.voice_peak_dbfs} dBFS; premix ${m.input_i} LUFS, ${m.input_tp} dBTP; ${table.length} sound events; bed ${useBed ? 'on' : 'off'} -> ${mixOut}`);
}
if (require.main === module) { try { main(); } catch (e) { U.die(e.stack || e.message); } }
module.exports = { bedTrack, normalize, measure };

Shown as it is in the repository at build time (2026-11-03); the home directory appears as <home>. If this page and the file differ, the file is right. Back to the catalogue.