Docs/Skills and code/src/

src/series-analysis/scoring.js

Code163 lines9.0 KBsrc/

import {
  ageDaysFrom, clamp01, detectLanguage, failureHits, guessSubject, median, nonReplicableFlags, norm, percentileRank, structureMatches
} from "./signals.js";

const STRONG_STRUCTURES = new Set(["why-how-X-died", "the-fall-of-X", "rise-and-fall", "what-happened-to-X", "hinglish-why-died"]);
const INDIA_TERMS = /\b(india|indian|hindi|desi|rupee|crore|lakh|sebi|rbi|tata|reliance|adani|ambani)\b/i;
const AGE_FLOOR_DAYS = 30;

/**
 * Series fit: does this video wear the packaging we want to learn from (disaster framing on a named company)?
 * Deterministic and explainable; an optional LLM judgement (0..1) is averaged in when available.
 */
export function scoreFit(video, spec, { llmFit = null } = {}) {
  const companies = [...(spec.companies?.indian ?? []), ...(spec.companies?.global ?? [])];
  const hits = failureHits(video.title, spec.failureTerms);
  const structures = structureMatches(video.title);
  const subject = guessSubject(video.title, companies);

  const failure = hits.length === 0 ? 0 : hits.length === 1 ? 0.6 : 1;
  const structure = structures.some(s => STRONG_STRUCTURES.has(s)) ? 1 : structures.length ? 0.6 : 0;
  const subjectScore = subject.source === "known" ? 1 : subject.source === "capitalised" ? 0.6 : 0;
  const discovery = clamp01((video.query_hits ?? 1) / 3);

  let fit = 0.35 * failure + 0.30 * structure + 0.20 * subjectScore + 0.15 * discovery;
  if (llmFit != null) fit = 0.5 * fit + 0.5 * clamp01(llmFit);

  const indianKnown = (spec.companies?.indian ?? []).some(c => subject.key && norm(c).startsWith(subject.key));
  const india = indianKnown || INDIA_TERMS.test(video.title) || detectLanguage(video.title) !== "en" || video.channel_country === "India";
  return { fit, failure_terms: hits, structures, subject: subject.key, subject_source: subject.source, india, language: detectLanguage(video.title) };
}

export function durationScore(seconds, [lowMin, highMin]) {
  if (!Number.isFinite(seconds)) return 0.5;
  const m = seconds / 60;
  if (m >= lowMin && m <= highMin) return 1;
  if (m >= 5 && m < lowMin) return 0.6;
  if (m > highMin && m <= 60) return 0.6;
  if (m >= 3 && m < 5) return 0.3;
  return 0.05;
}

export function sizeScore(subscribers, [low, high]) {
  if (!Number.isFinite(subscribers)) return 0.5;
  if (subscribers >= low && subscribers <= high) return 1;
  if (subscribers < low) return 0.6;
  // Past the ideal ceiling the audience, not the packaging, is doing more of the work.
  return subscribers <= high * 7 ? 0.5 : 0.3;
}

export function recencyScore(ageDays, halfLifeYears = 3) {
  if (ageDays == null) return 0.5;
  return Math.pow(0.5, ageDays / 365 / halfLifeYears);
}

/** How far a video beats its own channel's norm, log-scaled: 0.3x -> 0, 1x -> 0.33, 3x -> 0.65, 10x -> 1. */
export function outlierScore(ratio) {
  if (!Number.isFinite(ratio) || ratio <= 0) return null;
  return clamp01((Math.log10(ratio) + 0.5) / 1.5);
}

function weightedMean(parts) {
  const have = parts.filter(([value]) => value != null);
  const total = have.reduce((s, [, w]) => s + w, 0);
  return total ? have.reduce((s, [v, w]) => s + v * w, 0) / total : null;
}

/**
 * Scores every candidate. `videos` carry enrichment fields (views, likes, published_at, duration_seconds, channel_key, query_hits);
 * `channels` is a Map(channel_key -> {subscriber_count, baseline_median_views, baseline_n, country}).
 * Performance is judged against the pool of series-relevant candidates, not the whole internet.
 */
export function scoreCandidates(videos, channels, spec, { now = Date.now(), llmFit = new Map() } = {}) {
  const sc = spec.scoring;
  const rows = videos.map(video => {
    const channel = channels.get(video.channel_key) ?? {};
    const ageDays = ageDaysFrom(video.published_at, now);
    const f = scoreFit({ ...video, channel_country: channel.country }, spec, { llmFit: llmFit.get(video.video_id) ?? null });
    const views = Number.isFinite(video.views) ? video.views : null;
    const vpd = views != null && ageDays != null ? views / Math.max(ageDays, AGE_FLOOR_DAYS) : null;
    const baseline = channel.baseline_n >= 5 ? channel.baseline_median_views : null;
    const subs = channel.subscriber_count ?? null;
    let outlierRatio = null, outlierBasis = null;
    if (views != null && baseline) { outlierRatio = views / baseline; outlierBasis = "channel-median"; }
    else if (views != null && subs) { outlierRatio = views / Math.max(subs, 1000); outlierBasis = "subscribers"; }
    const engagement = video.likes != null && views ? video.likes / views : null;
    const flags = nonReplicableFlags(video.title);
    return { ...video, ...f, age_days: ageDays, views_per_day: vpd, outlier_ratio: outlierRatio, outlier_basis: outlierBasis,
      engagement, subscribers: subs, baseline_median_views: baseline, format_flags: flags };
  });

  // Percentiles are taken over the on-series pool so "good" means good *for this kind of video*.
  const pool = rows.filter(r => r.fit >= sc.minFitToRank && r.views != null);
  const vpdSorted = pool.map(r => r.views_per_day).filter(Number.isFinite).sort((a, b) => a - b);
  const engSorted = pool.map(r => r.engagement).filter(Number.isFinite).sort((a, b) => a - b);

  for (const r of rows) {
    const vpdPct = percentileRank(vpdSorted, r.views_per_day);
    const engPct = percentileRank(engSorted, r.engagement);
    // A ratio from subscribers is a weaker baseline than the channel's own median, so it counts for less.
    const outlier = outlierScore(r.outlier_ratio);
    const performance = weightedMean([[outlier, r.outlier_basis === "subscribers" ? 0.3 : 0.5], [vpdPct, 0.35], [engPct, 0.15]]);
    const formatFactor = r.format_flags.some(f => f === "podcast/interview" || f === "fiction/clip" || f === "shorts") ? 0.15
      : r.format_flags.length ? 0.5 : 1;
    const learnability = (0.40 * durationScore(r.duration_seconds, sc.durationSweetSpotMinutes)
      + 0.25 * sizeScore(r.subscribers, sc.idealSubscribers)
      + 0.35 * recencyScore(r.age_days, sc.recencyHalfLifeYears)) * formatFactor;

    r.performance = performance;
    r.learnability = learnability;
    r.components = { outlier, views_per_day_pct: vpdPct, engagement_pct: engPct, format_factor: formatFactor };

    if (r.fit < sc.minFitToRank) r.excluded = `off-series (fit ${r.fit.toFixed(2)})`;
    else if (performance == null) r.excluded = "no performance data";
    else r.excluded = null;
    const w = sc.weights;
    r.learn_score = r.excluded ? null : w.fit * r.fit + w.performance * performance + w.learnability * learnability;
    r.tier = r.fit >= 0.65 && r.india ? "series-exact" : r.fit >= 0.5 ? "adjacent-packaging" : "adjacent-topic";
  }
  return rows;
}

/**
 * Greedy top-N with diversity: at most N videos per channel and per subject company, so ten picks means
 * ten different lessons rather than one channel's house style or one company's story told five ways.
 */
export function selectLearnFrom(scored, spec, { topN = spec.scoring.topN, perChannel, perSubject } = {}) {
  // The caps exist to keep a short list varied; a long list would otherwise run out of eligible videos, so they grow with it.
  perChannel ??= Math.max(spec.scoring.maxPerChannelInTop, Math.ceil(topN / 10));
  perSubject ??= Math.max(2, Math.ceil(topN / 10));
  const ranked = scored.filter(r => r.learn_score != null).sort((a, b) => b.learn_score - a.learn_score);
  const channelCount = new Map(), subjectCount = new Map();
  const picked = [];
  for (const r of ranked) {
    if (picked.length >= topN) break;
    const c = channelCount.get(r.channel_key) ?? 0;
    const s = r.subject ? subjectCount.get(r.subject) ?? 0 : 0;
    if (c >= perChannel || s >= perSubject) { r.skipped_for = c >= perChannel ? "channel cap" : "subject cap"; continue; }
    channelCount.set(r.channel_key, c + 1);
    if (r.subject) subjectCount.set(r.subject, s + 1);
    picked.push(r);
  }
  return picked;
}

/** The side lists that make the ranking useful for study rather than just a leaderboard. */
export function studyLists(scored, selected) {
  const eligible = scored.filter(r => r.learn_score != null);
  const chosen = new Set(selected.map(r => r.video_id));
  const breakouts = eligible
    .filter(r => r.outlier_ratio >= 3 && (r.subscribers ?? Infinity) < 500000 && !chosen.has(r.video_id))
    .sort((a, b) => b.outlier_ratio - a.outlier_ratio).slice(0, 10);
  const giants = eligible.filter(r => (r.subscribers ?? 0) > 3000000 && !chosen.has(r.video_id))
    .sort((a, b) => b.views - a.views).slice(0, 8);
  // Same packaging, weak result: the contrast set tells you which part of the formula is not carrying the video.
  const perfs = eligible.map(r => r.performance).sort((a, b) => a - b);
  const cutoff = perfs[Math.floor(perfs.length / 3)] ?? 0;
  const contrast = eligible.filter(r => r.fit >= 0.65 && r.performance <= cutoff && r.learnability >= 0.5)
    .sort((a, b) => b.fit - a.fit || a.performance - b.performance).slice(0, 8);
  return { breakouts, giants, contrast };
}

export { median };

Shown as it is in the repository at build time (2026-11-03); the home directory appears as <home>. If this page and the file differ, the file is right. Back to the catalogue.