Docs/Skills and code/src/

src/video-normalization.js

Code239 lines10.4 KBsrc/

/**
 * Defensive normalizers for YouTube search results and youtubei.js channel feeds.
 * Display strings are retained separately from parsed/estimated values so callers
 * can distinguish exact observations from rounded or relative display text.
 */

export function textValue(value) {
  if (value == null) return null;
  if (typeof value === "string") return value;
  if (typeof value.text === "string") return value.text;
  if (typeof value.content === "string") return value.content;
  return null;
}

export function parseCount(value) {
  if (value == null || value === "") return null;
  if (typeof value === "number") return Number.isFinite(value) ? Math.round(value) : null;
  const text = String(value).trim().replace(/,/g, "");
  const match = text.match(/(-?[0-9]+(?:\.[0-9]+)?)\s*([KMB])?/i);
  if (!match) return null;
  const number = Number(match[1]);
  if (!Number.isFinite(number)) return null;
  const suffix = (match[2] ?? "").toUpperCase();
  const multiplier = suffix === "K" ? 1_000 : suffix === "M" ? 1_000_000 : suffix === "B" ? 1_000_000_000 : 1;
  return Math.round(number * multiplier);
}

export function countPrecision(value) {
  if (value == null || value === "") return "unknown";
  if (typeof value === "number") return "exact";
  const text = String(value);
  if (/\b[0-9]+(?:\.[0-9]+)?\s*[KMB]\b/i.test(text)) return "abbreviated";
  return parseCount(text) == null ? "unknown" : "exact_displayed";
}

export function formatDurationText(value) {
  const seconds = typeof value === "number" ? value : Number(value);
  if (!Number.isFinite(seconds) || seconds < 0) return null;
  const total = Math.round(seconds);
  const hours = Math.floor(total / 3600);
  const minutes = Math.floor((total % 3600) / 60);
  const secs = total % 60;
  return hours > 0
    ? `${hours}:${String(minutes).padStart(2, "0")}:${String(secs).padStart(2, "0")}`
    : `${minutes}:${String(secs).padStart(2, "0")}`;
}

export function durationPrecision(value) {
  if (value == null || value === "") return "unknown";
  if (typeof value === "number") return "exact";
  const text = String(value).trim();
  if (/^\d+(?::\d{1,2}){1,2}$/.test(text)) return "exact";
  if (/\d+\s*(?:seconds?|secs?|s)\b/i.test(text)) return "displayed_seconds";
  if (/\d+\s*(?:hours?|hrs?|h|minutes?|mins?|m)\b/i.test(text)) return "rounded_display";
  return "display_only";
}

export function parseDurationSeconds(value) {
  if (value == null) return null;
  if (typeof value === "number") return Number.isFinite(value) && value >= 0 ? Math.round(value) : null;
  const text = String(value).trim();
  if (!text) return null;
  if (/^\d+(?::\d{1,2}){1,2}$/.test(text)) {
    return text.split(":").map(Number).reduce((total, part) => total * 60 + part, 0);
  }
  const units = [...text.matchAll(/(\d+)\s*(hours?|hrs?|h|minutes?|mins?|m|seconds?|secs?|s)\b/ig)];
  if (!units.length) return null;
  let seconds = 0;
  for (const [, raw, unit] of units) {
    const u = unit.toLowerCase();
    const n = Number(raw);
    seconds += /^(?:h|hr|hrs|hour|hours)$/.test(u) ? n * 3600 : /^(?:m|min|mins|minute|minutes)$/.test(u) ? n * 60 : n;
  }
  return seconds || null;
}

export function parseAbsoluteDate(value) {
  if (value == null) return { value: null, precision: "unknown" };
  const text = String(value).trim();
  if (!text) return { value: null, precision: "unknown" };
  if (/\b(?:ago|yesterday|today|just now|streamed live)\b/i.test(text)) {
    return { value: null, precision: "relative" };
  }

  const match = text.match(/^(?:Published\s+)?([A-Za-z]{3,9}\s+\d{1,2},?\s+\d{4}|\d{4}-\d{2}-\d{2})$/i);
  if (!match) return { value: null, precision: "display_only" };

  // Parse calendar components explicitly. Date.parse("Sep 1, 2026") is
  // interpreted as local midnight by some runtimes/time zones; converting that
  // instant to UTC can shift the calendar date backwards in UTC+ time zones.
  // These source strings describe calendar dates, not instants in time.
  const source = match[1];
  let year, month, day;
  const iso = source.match(/^(\d{4})-(\d{2})-(\d{2})$/);
  if (iso) {
    [, year, month, day] = iso;
    year = Number(year); month = Number(month); day = Number(day);
  } else {
    const named = source.match(/^([A-Za-z]{3,9})\s+(\d{1,2}),?\s+(\d{4})$/);
    if (!named) return { value: null, precision: "display_only" };
    const monthIndex = new Map([
      ["jan", 1], ["feb", 2], ["mar", 3], ["apr", 4], ["may", 5], ["jun", 6],
      ["jul", 7], ["aug", 8], ["sep", 9], ["oct", 10], ["nov", 11], ["dec", 12]
    ]).get(named[1].slice(0, 3).toLowerCase());
    if (!monthIndex) return { value: null, precision: "display_only" };
    month = monthIndex; day = Number(named[2]); year = Number(named[3]);
  }

  const date = new Date(Date.UTC(year, month - 1, day));
  if (date.getUTCFullYear() !== year || date.getUTCMonth() !== month - 1 || date.getUTCDate() !== day) {
    return { value: null, precision: "display_only" };
  }
  return {
    value: `${String(year).padStart(4, "0")}-${String(month).padStart(2, "0")}-${String(day).padStart(2, "0")}`,
    precision: "date"
  };
}

export function bestThumbnailUrl(thumbnails) {
  if (!Array.isArray(thumbnails)) return null;
  const usable = thumbnails.filter(item => typeof item?.url === "string" && item.url.length > 0);
  usable.sort((a, b) => ((b.width ?? 0) * (b.height ?? 0)) - ((a.width ?? 0) * (a.height ?? 0)));
  return usable[0]?.url ?? null;
}

function metadataRows(node) {
  const candidates = [
    node?.metadata?.metadata?.metadata_rows,
    node?.metadata?.metadata_rows,
    node?.metadata?.metadata?.rows,
    node?.metadata_rows
  ];
  return candidates.find(Array.isArray) ?? [];
}

export function extractFeedMetadataTexts(node) {
  return metadataRows(node).flatMap(row => Array.isArray(row?.metadata_parts) ? row.metadata_parts : [])
    .map(part => textValue(part?.text ?? part?.primary_text ?? part))
    .filter(Boolean);
}

function durationFromNode(node) {
  const direct = textValue(node?.duration);
  if (direct) return direct;
  const label = node?.renderer_context?.accessibility_context?.label ?? node?.accessibility_label ?? "";
  const match = String(label).match(/(?:^|\s)(\d+\s*(?:hours?|hrs?|h)\s*)?(?:,?\s*\d+\s*(?:minutes?|mins?|m)\s*)?(?:,?\s*\d+\s*(?:seconds?|secs?|s))?\s*$/i);
  const candidate = match?.[0]?.trim();
  return candidate && /\d/.test(candidate) ? candidate : null;
}

function safeRawPreview(node, texts) {
  return JSON.stringify({
    constructor: node?.constructor?.name ?? node?.type ?? null,
    type: node?.type ?? null,
    keys: node ? Object.keys(node).slice(0, 80) : [],
    metadata_texts: texts,
    id: node?.content_id ?? node?.video_id ?? node?.id ?? null,
    content_type: node?.content_type ?? null,
    thumbnail_urls: (node?.content_image?.image ?? node?.content_image?.sources ?? node?.thumbnails ?? [])
      .filter(item => item?.url).slice(0, 5).map(item => item.url),
    accessibility_label: node?.renderer_context?.accessibility_context?.label ?? node?.accessibility_label ?? null
  });
}

export function normalizeChannelVideoNode(node, channelName = null) {
  if (!node) return null;
  const videoId = node.content_id ?? node.video_id ?? node.id ?? null;
  if (!videoId) return null;
  const contentType = String(node.content_type ?? "").toUpperCase();
  if (contentType && !["VIDEO", "SHORT", "MUSIC_VIDEO_TYPE_ATV"].includes(contentType)) return null;

  const title = textValue(node.metadata?.title) ?? textValue(node.metadata?.primary_text) ?? textValue(node.title) ?? "";
  const texts = extractFeedMetadataTexts(node);
  const viewText = texts.find(value => /\bviews?\b/i.test(value)) ?? texts.find(value => /^\s*\d[\d,.]*(?:\s*[KMB])?\s*$/i.test(value)) ?? null;
  const publishedText = texts.find(value => /\b(?:ago|yesterday|today|\d{4}|jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\b/i.test(value) && value !== viewText) ?? null;
  const rawDuration = durationFromNode(node);
  const thumbnailList = node.content_image?.image ?? node.content_image?.sources ?? node.content_image?.thumbnails ?? node.thumbnails ?? [];
  const date = parseAbsoluteDate(publishedText);
  const viewCount = parseCount(node.view_count ?? viewText);

  return {
    video_id: String(videoId),
    url: `https://www.youtube.com/watch?v=${videoId}`,
    title,
    channel_name: channelName ?? null,
    thumbnail_url: bestThumbnailUrl(thumbnailList),
    duration: rawDuration,
    duration_seconds: parseDurationSeconds(rawDuration),
    duration_precision: durationPrecision(rawDuration),
    view_count: viewCount,
    view_count_text: viewText,
    view_count_precision: countPrecision(node.view_count ?? viewText),
    published_at: date.value,
    published_at_text: publishedText,
    published_at_precision: date.precision,
    likes_count: null,
    category: null,
    description: null,
    video_channel_id: null,
    metadata_source: "youtubei_channel_feed",
    raw_metadata_json: safeRawPreview(node, texts)
  };
}

function safeStringify(value) {
  try { return JSON.stringify(value); } catch { return null; }
}

export function normalizeSearchVideo(video) {
  const viewText = video.viewCountText ?? video.viewCount ?? video.view_count_text ?? video.view_count ?? null;
  const dateText = video.publishedAtText ?? video.publishedAt ?? video.published_at_text ?? video.published_at ?? null;
  const date = parseAbsoluteDate(dateText);
  const duration = video.duration ?? null;
  const author = video.author ?? {};
  return {
    video_id: video.id ?? video.video_id ?? null,
    title: video.title ?? "",
    url: video.url ?? (video.id ? `https://www.youtube.com/watch?v=${video.id}` : null),
    channel_name: author.name ?? video.channel_name ?? null,
    channel_url: author.url ?? video.channel_url ?? null,
    thumbnail_url: video.thumbnail?.url ?? bestThumbnailUrl(video.thumbnail?.thumbnails) ?? video.image ?? null,
    duration: typeof duration === "string" ? duration : duration?.text ?? null,
    duration_seconds: parseDurationSeconds(duration?.seconds ?? duration),
    duration_precision: durationPrecision(duration?.seconds ?? duration),
    view_count: parseCount(viewText),
    view_count_text: typeof viewText === "string" ? viewText : null,
    view_count_precision: countPrecision(viewText),
    published_at: date.value,
    published_at_text: dateText == null ? null : String(dateText),
    published_at_precision: date.precision,
    likes_count: null,
    category: video.category ?? null,
    description: video.description ?? null,
    video_channel_id: video.channelId ?? video.channel_id ?? null,
    metadata_source: "ytsearch.js",
    raw_metadata_json: safeStringify(video)
  };
}

Shown as it is in the repository at build time (2026-11-03); the home directory appears as <home>. If this page and the file differ, the file is right. Back to the catalogue.