pipeline/feed.js
// Feed intake — shared by local scripts and the Worker. Pure given a fetch impl.
import { hashId, utcDate } from "./util.js";
export const FEED_UA = "fact.ngo-emergence/0.1 (nonprofit live-events sensor; takes only what feeds offer)";
export function decodeEntities(s) {
return String(s)
.replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, "$1")
.replace(/<[^>]+>/g, " ")
.replace(/ /g, " ")
.replace(/&/g, "&")
.replace(/</g, "<")
.replace(/>/g, ">")
.replace(/"/g, '"')
.replace(/'/g, "'")
.replace(/'/g, "'")
.replace(/&#(\d+);/g, (_, n) => String.fromCodePoint(+n))
.replace(/\s+/g, " ")
.trim();
}
function tag(xml, name) {
const m = xml.match(new RegExp(`<${name}[^>]*>([\\s\\S]*?)</${name}>`));
if (m) return decodeEntities(m[1]);
const self = xml.match(new RegExp(`<${name}[^>]*href="([^"]+)"`));
if (self) return self[1];
return null;
}
export function parseFeed(xml) {
const items = [];
const blocks = [...xml.matchAll(/<item>([\s\S]*?)<\/item>/g)].map((m) => m[1]);
const entries = [...xml.matchAll(/<entry[\s>][\s\S]*?<\/entry>/g)].map((m) => m[0]);
for (const b of blocks.length ? blocks : entries) {
const link = tag(b, "link");
const title = tag(b, "title");
if (!link || !title) continue;
items.push({
title,
link,
description: tag(b, "description") ?? tag(b, "summary") ?? null,
pubDate: tag(b, "pubDate") ?? tag(b, "published") ?? tag(b, "updated") ?? null,
});
}
return items;
}
// Conditional GET: pass the source's stored validator so unchanged feeds answer
// 304 with no body — the polite, near-free way to poll at high frequency.
export async function fetchSource(src, fetchImpl, validator) {
const headers = { "User-Agent": FEED_UA, Accept: "application/rss+xml, application/atom+xml, application/xml, text/xml, */*" };
if (validator?.etag) headers["If-None-Match"] = validator.etag;
if (validator?.lastModified) headers["If-Modified-Since"] = validator.lastModified;
const res = await fetchImpl(src.url, { headers, redirect: "follow" });
if (res.status === 304) return { notModified: true, http: 304, xml: "" };
if (!res.ok) throw new Error(`HTTP ${res.status}`);
return {
xml: await res.text(),
http: res.status,
validator: { etag: res.headers.get("etag") || null, lastModified: res.headers.get("last-modified") || null },
};
}
// One feed item -> one article record (or null when already seen).
export function articleFromItem(src, item, seen, date) {
let title = item.title;
if (src.via === "google-news-rss") title = title.replace(/ - [^-]{2,40}$/, "").trim();
const norm = title.toLowerCase().replace(/[^a-z0-9]+/g, " ").trim();
const id = hashId("art", `${src.id}|${norm}`);
if (seen.has(id)) return null;
seen.add(id);
return {
id,
status: "unverified",
source_id: src.id,
source_name: src.name,
via: src.via,
url: item.link,
url_direct: src.via === "rss" ? item.link : null,
title,
summary: src.depth === "summary" && item.description ? item.description : null,
published_at: item.pubDate ? new Date(item.pubDate).toISOString() : null,
scraped_at: new Date().toISOString(),
date: date || utcDate(),
};
}