Docs/Skills and code/skill video-script-production

.claude/skills/video-script-production/scripts/fetch-articles.mjs

Code88 lines5.2 KBSkill: video-script-production

#!/usr/bin/env node
// S1b: save pre-existing articles and documents about the topic as plain text, so the research is in the repo.
//
//   node fetch-articles.js --dir research/<slug>/sources --urls urls.txt [--pause 1500]
//   node fetch-articles.js --dir research/<slug>/sources --url https://... --url https://...
//
// urls.txt: one URL per line; text after a space is a note ("why this source"); lines starting with # are skipped.
// Writes <dir>/NN-<host>.txt (header: URL, fetched date, HTTP status, length; then the text) and <dir>/INDEX.md.
// PDFs are saved as-is (<dir>/NN-<host>.pdf) with a stub .txt; read them with the Read tool (pages option).
// Pages that block scripted fetches (403, a short body, a consent wall) are listed in INDEX.md as "blocked": open them
// with the browser tools or WebFetch and mark the fact [V-search] until you have read the page.
// It saves what the page says. It does not judge it: a saved article is a lead until FACTS.md cites it with a status.

import fs from 'node:fs';
import path from 'node:path';

const args = process.argv.slice(2);
const opt = (k, d) => { const i = args.indexOf('--' + k); return i >= 0 ? args[i + 1] : d; };
const all = (k) => args.flatMap((a, i) => (a === '--' + k ? [args[i + 1]] : []));
const dir = opt('dir');
if (!dir) { console.error('usage: fetch-articles.js --dir <folder> (--urls file | --url U ...) [--pause ms]'); process.exit(2); }
const pause = Number(opt('pause', 1500));

let entries = all('url').map((u) => ({ url: u, note: '' }));
const f = opt('urls');
if (f) {
  for (const line of fs.readFileSync(f, 'utf8').split(/\r?\n/)) {
    const s = line.trim();
    if (!s || s.startsWith('#')) continue;
    const sp = s.indexOf(' ');
    entries.push(sp < 0 ? { url: s, note: '' } : { url: s.slice(0, sp), note: s.slice(sp + 1).trim() });
  }
}
fs.mkdirSync(dir, { recursive: true });

const UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36';
const decode = (s) => s
  .replace(/&nbsp;/g, ' ').replace(/&amp;/g, '&').replace(/&lt;/g, '<').replace(/&gt;/g, '>').replace(/&quot;/g, '"')
  .replace(/&#39;|&rsquo;|&lsquo;/g, "'").replace(/&rdquo;|&ldquo;/g, '"').replace(/&#8377;/g, 'Rs ')
  .replace(/&#(\d+);/g, (_, n) => String.fromCharCode(Number(n)));

function htmlToText(html) {
  let h = html.replace(/<(script|style|noscript|svg|nav|footer|header|form|iframe)[\s\S]*?<\/\1>/gi, ' ');
  const title = (h.match(/<title[^>]*>([\s\S]*?)<\/title>/i) || [])[1] || '';
  const art = h.match(/<article[\s\S]*?<\/article>/i);
  if (art && art[0].length > 1500) h = art[0];
  h = h.replace(/<\/(p|div|h[1-6]|li|tr|blockquote|section)>/gi, '\n').replace(/<br\s*\/?>/gi, '\n').replace(/<[^>]+>/g, ' ');
  const text = decode(h).split(/\n/).map((l) => l.replace(/\s+/g, ' ').trim()).filter((l) => l.length > 0).join('\n');
  return { title: decode(title).replace(/\s+/g, ' ').trim(), text };
}

const index = [];
let n = 0;
for (const e of entries) {
  n += 1;
  const host = (() => { try { return new URL(e.url).hostname.replace(/^www\./, ''); } catch { return 'bad-url'; } })();
  const base = `${String(n).padStart(2, '0')}-${host.replace(/[^a-z0-9.-]/gi, '_')}`;
  const row = { n, url: e.url, note: e.note, file: '', status: '', chars: 0, title: '' };
  try {
    const r = await fetch(e.url, { headers: { 'user-agent': UA, accept: 'text/html,application/pdf,*/*', 'accept-language': 'en-IN,en;q=0.9' }, redirect: 'follow' });
    row.status = String(r.status);
    const type = r.headers.get('content-type') || '';
    if (!r.ok) { row.file = 'blocked'; }
    else if (/pdf/i.test(type) || /\.pdf($|\?)/i.test(e.url)) {
      fs.writeFileSync(path.join(dir, base + '.pdf'), Buffer.from(await r.arrayBuffer()));
      row.file = base + '.pdf'; row.title = 'PDF (read it with the Read tool)';
    } else {
      const { title, text } = htmlToText(await r.text());
      row.title = title; row.chars = text.length;
      if (text.length < 1200) row.file = 'blocked'; // a consent wall, a paywall or a script-only page
      else {
        row.file = base + '.txt';
        fs.writeFileSync(path.join(dir, row.file), `URL: ${e.url}\nFetched: ${new Date().toISOString().slice(0, 10)}\nHTTP: ${r.status}\nTitle: ${title}\nNote: ${e.note}\nChars: ${text.length}\n---\n${text}\n`);
      }
    }
  } catch (err) { row.status = 'error'; row.file = 'blocked'; row.title = String(err.message || err).slice(0, 80); }
  console.log(`${String(n).padStart(2)} ${row.file === 'blocked' ? 'BLOCKED' : 'saved  '} ${row.status} ${row.chars} ${e.url.slice(0, 90)}`);
  index.push(row);
  await new Promise((res) => setTimeout(res, pause));
}

const md = ['# Saved sources', '', `Fetched ${new Date().toISOString().slice(0, 10)} by fetch-articles.js. "blocked" = not saved; open it another way and keep the fact at [V-search] until you have read it.`, '',
  '| # | file | HTTP | chars | title | note | URL |', '|---|---|---|---|---|---|---|',
  ...index.map((r) => `| ${r.n} | ${r.file} | ${r.status} | ${r.chars} | ${r.title.replace(/\|/g, '/').slice(0, 70)} | ${r.note.replace(/\|/g, '/')} | ${r.url} |`), ''];
const prev = path.join(dir, 'INDEX.md');
fs.writeFileSync(prev, md.join('\n'));
console.log(`saved ${index.filter((r) => r.file !== 'blocked').length} of ${index.length}; index: ${prev}`);

Shown as it is in the repository at build time (2026-11-03); the home directory appears as <home>. If this page and the file differ, the file is right. Back to the catalogue.