init: Vite+React+Tailwind v2 site with HTML content from WP, RSS feed, external feed aggregator, prerender

scripts/pull-external-rss.js

@@ -0,0 +1,125 @@
+#!/usr/bin/env node
+/**
+ * Тянет внешние RSS-фиды из src/content/feeds.json, дедуплицирует по guid/link,
+ * пишет агрегированный news.json в DATA_DIR (default: ./data).
+ * Запускается по cron на хосте.
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { XMLParser } from 'fast-xml-parser';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const ROOT = path.resolve(__dirname, '..');
+const FEEDS_FILE = path.join(ROOT, 'src/content/feeds.json');
+const DATA_DIR = process.env.DATA_DIR || path.join(ROOT, 'data');
+const OUT_FILE = path.join(DATA_DIR, 'news.json');
+const TIMEOUT_MS = 15000;
+const HARD_CAP = 200;
+
+const parser = new XMLParser({
+  ignoreAttributes: false,
+  attributeNamePrefix: '@_',
+  textNodeName: '#text',
+});
+
+async function fetchFeed(url, timeoutMs) {
+  const ctl = new AbortController();
+  const t = setTimeout(() => ctl.abort(), timeoutMs);
+  try {
+    const r = await fetch(url, {
+      signal: ctl.signal,
+      headers: { 'User-Agent': 'pushkinohistory-ru-v2 RSS aggregator' },
+    });
+    if (!r.ok) throw new Error(`HTTP ${r.status}`);
+    return await r.text();
+  } finally {
+    clearTimeout(t);
+  }
+}
+
+function extractItems(xml, feed) {
+  const parsed = parser.parse(xml);
+  // RSS 2.0
+  const rssItems = parsed?.rss?.channel?.item;
+  if (rssItems) {
+    const arr = Array.isArray(rssItems) ? rssItems : [rssItems];
+    return arr.map((it) => ({
+      title: typeof it.title === 'string' ? it.title : it.title?.['#text'] || '',
+      link: typeof it.link === 'string' ? it.link : it.link?.['#text'] || '',
+      guid: typeof it.guid === 'string' ? it.guid : it.guid?.['#text'] || it.link || '',
+      pubDate: it.pubDate ? new Date(it.pubDate).toISOString() : null,
+      description: stripHtml(it.description || it['content:encoded'] || ''),
+      source: feed.name,
+    }));
+  }
+  // Atom
+  const atomEntries = parsed?.feed?.entry;
+  if (atomEntries) {
+    const arr = Array.isArray(atomEntries) ? atomEntries : [atomEntries];
+    return arr.map((e) => {
+      const link = Array.isArray(e.link) ? e.link[0]?.['@_href'] : e.link?.['@_href'] || e.link;
+      return {
+        title: typeof e.title === 'string' ? e.title : e.title?.['#text'] || '',
+        link: link || '',
+        guid: e.id || link || '',
+        pubDate: e.updated || e.published ? new Date(e.updated || e.published).toISOString() : null,
+        description: stripHtml(e.summary?.['#text'] || e.summary || e.content?.['#text'] || ''),
+        source: feed.name,
+      };
+    });
+  }
+  return [];
+}
+
+function stripHtml(s) {
+  if (!s) return '';
+  return String(s).replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim().slice(0, 400);
+}
+
+async function main() {
+  const feeds = JSON.parse(fs.readFileSync(FEEDS_FILE, 'utf8')).filter((f) => f.enabled);
+  if (feeds.length === 0) {
+    console.log('no enabled feeds — writing empty news.json');
+    fs.mkdirSync(DATA_DIR, { recursive: true });
+    fs.writeFileSync(OUT_FILE, JSON.stringify({ updatedAt: new Date().toISOString(), items: [] }, null, 2));
+    return;
+  }
+
+  const all = [];
+  for (const feed of feeds) {
+    try {
+      const xml = await fetchFeed(feed.url, TIMEOUT_MS);
+      const items = extractItems(xml, feed);
+      const max = feed.max || 20;
+      all.push(...items.slice(0, max));
+      console.log(`✓ ${feed.name}: ${items.length} items (kept ${Math.min(items.length, max)})`);
+    } catch (e) {
+      console.warn(`✗ ${feed.name}: ${e.message}`);
+    }
+  }
+
+  const seen = new Set();
+  const deduped = [];
+  for (const it of all) {
+    const key = it.guid || it.link;
+    if (!key || seen.has(key)) continue;
+    seen.add(key);
+    deduped.push(it);
+  }
+  deduped.sort((a, b) => (b.pubDate || '').localeCompare(a.pubDate || ''));
+
+  const out = {
+    updatedAt: new Date().toISOString(),
+    items: deduped.slice(0, HARD_CAP),
+  };
+
+  fs.mkdirSync(DATA_DIR, { recursive: true });
+  fs.writeFileSync(OUT_FILE, JSON.stringify(out, null, 2));
+  console.log(`→ ${OUT_FILE}: ${out.items.length} items`);
+}
+
+main().catch((e) => {
+  console.error(e);
+  process.exit(1);
+});