scripts/init-data.sh
#!/usr/bin/env bash
# init-data.sh — creates the emergence-data repo: bare canonical at
# ~/remotes/fact.ngo/emergence-data.git on the self-hosted remote, working clones at
# ~/coherence/fact.ngo/emergence-data (the self-hosted remote) and the local mono-folder.
# Unlike model datasets, this repo is DURABLE (append-only archive), not disposable.
set -euo pipefail
REMOTE_HOST="${REMOTE_HOST:-remote}"
LOCAL_ROOT="${EMERGENCE_PARENT:-$HOME/Documents/fact.ngo}"
DATA="$LOCAL_ROOT/emergence-data"
if ssh -o ConnectTimeout=8 "$REMOTE_HOST" true 2>/dev/null; then
ssh "$REMOTE_HOST" "set -e
mkdir -p ~/remotes/fact.ngo ~/coherence/fact.ngo
if [ ! -d ~/remotes/fact.ngo/emergence-data.git ]; then
git init --bare -b main ~/remotes/fact.ngo/emergence-data.git
fi
if [ ! -d ~/coherence/fact.ngo/emergence-data/.git ]; then
git clone -q ~/remotes/fact.ngo/emergence-data.git ~/coherence/fact.ngo/emergence-data
cd ~/coherence/fact.ngo/emergence-data
git config user.name zauh
git config user.email zauh@coherence.ngo
fi"
REMOTE_UP=1
else
echo "note: the self-hosted remote unreachable — remote will be set, push later with:"
echo " git -C $DATA push -u origin main"
REMOTE_UP=0
fi
if [ ! -d "$DATA" ]; then
mkdir -p "$DATA"
cd "$DATA"
git init -b main
git config user.name zauh
git config user.email zauh@coherence.ngo
else
cd "$DATA"
fi
if [ "$REMOTE_UP" = 1 ]; then
git remote remove origin 2>/dev/null || true
git remote add origin "$REMOTE_HOST:remotes/fact.ngo/emergence-data.git"
else
git remote get-url origin >/dev/null 2>&1 || \
git remote add origin "$REMOTE_HOST:remotes/fact.ngo/emergence-data.git"
fi
if ! git rev-parse --verify -q HEAD >/dev/null; then
cat > README.md <<'EOF'
# emergence-data
Append-only data repo of the emergence live-events sensor net (fact.ngo).
Written exclusively by emergence engine scripts; never edited by hand.
- raw/<date>/articles.jsonl — scraped reports (teaser depth)
- distilled/<date>/essence.jsonl — unverified distillation records
- aggregates/<date>/ — fractal topic rollups, GIS points, day stats
- verified/ — council verdicts (lane inactive; empty)
- ledger.jsonl — per-call cost accounting
Contract: ../emergence/AGENTS.md. System model: ../emergence/docs/design.md.
Code and curation AGPL-3.0; newsroom content remains its owners'.
EOF
mkdir -p raw distilled aggregates verified
cat > manifest.json <<'EOF'
{
"repo": "emergence-data",
"engine": "emergence",
"purpose": "unverified live-events second-order heap for fact.ngo/emergence",
"distill_model": "@cf/zai-org/glm-5.3-flash",
"council_model": "@cf/zai-org/glm-5.3",
"status_field": "every record is born unverified; only council.js writes other statuses",
"created": ""
}
EOF
# created timestamp set below (portable date)
perl -i -pe 's/"created": ""/"created": "'$(date -u +%Y-%m-%dT%H:%M:%SZ)'"/' manifest.json
cat > .gitignore <<'EOF'
.DS_Store
EOF
git add -A
git commit -m "Init emergence-data append-only repo"
fi
if [ "$REMOTE_UP" = 1 ]; then
git push -q -u origin main 2>/dev/null && echo "pushed to origin" || echo "push deferred (remote ahead or empty)"
fi
echo "data repo ready: $DATA (origin: remote:remotes/fact.ngo/emergence-data.git)"