scripts/sync-from-r2.sh
#!/usr/bin/env bash
# sync-from-r2.sh — pulls the live day partitions from the emergence-net Worker's
# R2 store into the git archive (emergence-data) and commits them. Run on the self-hosted remote
# (cron or by hand) or from the Mac: R2 is the live primary, git is the durable
# archive. One commit per sync; ledger rides along.
#
# scripts/sync-from-r2.sh [date ...] # defaults: yesterday + today (UTC)
#
# Requires WORKER_BASE (default https://emergence-net.zauh.workers.dev) and, if
# set, no auth for partitions (read routes are public); the data repo resolves as
# EMERGENCE_DATA or ../emergence-data.
set -euo pipefail
BASE="${WORKER_BASE:-https://emergence-net.zauh.workers.dev}"
DATA="${EMERGENCE_DATA:-$(cd "$(dirname "$0")/../.." && pwd)/emergence-data}"
HERE="$(cd "$(dirname "$0")/.." && pwd)"
DATES=("$@")
if [ -z "${DATES:-}" ] || [ "${#DATES[@]}" -eq 0 ] || [ -z "${DATES[0]}" ]; then
DATES=("$(date -u -v-1d +%F 2>/dev/null || date -u -d yesterday +%F)" "$(date -u +%F)")
fi
mkdir -p "$DATA"
cd "$DATA"
pull() { # pull <r2-key> <local-path>
local code
code=$(curl -s --max-time 60 -o "$2.tmp" -w "%{http_code}" "$BASE/$1" || echo 000)
if [ "$code" = "200" ]; then mv "$2.tmp" "$2"; else rm -f "$2.tmp"; return 1; fi
}
changed=0
for d in "${DATES[@]}"; do
mkdir -p "raw/$d" "distilled/$d" "aggregates/$d"
for f in articles.jsonl sources.json; do
pull "partitions/$d/$f" "raw/$d/$f" && changed=1 || true
done
pull "partitions/$d/essence.jsonl" "distilled/$d/essence.jsonl" && changed=1 || true
for f in events.json topics.json day.json gis.json; do
pull "partitions/$d/$f" "aggregates/$d/$f" && changed=1 || true
done
done
pull "ledger.jsonl" "ledger.jsonl" && changed=1 || true
if [ "$changed" = 1 ]; then
git add -A
git commit -q -m "Sync from live store: ${DATES[*]}" || true
git push -q 2>/dev/null || echo "note: push deferred (remote unreachable)"
echo "synced ${DATES[*]} and committed"
else
echo "nothing new to sync"
fi