#!/usr/bin/env bash
# Reproduces every published number, top to bottom, with one command:
#   ./run_all.sh
# Optional: ZV_DATA_DIR=/path/with/10GB ./run_all.sh   (default: ./data)
# Needs python3 (the reference run used 3.13.5, see .python-version) and pdftotext
# (poppler-utils). Everything else is installed into .venv from requirements.lock.
set -euo pipefail
cd "$(dirname "$0")"
export ZV_DATA_DIR="${ZV_DATA_DIR:-$PWD/data}"

for cmd in python3 pdftotext; do
  command -v "$cmd" >/dev/null || { echo "missing required command: $cmd" >&2; exit 1; }
done
want=$(cat .python-version)
have=$(python3 -c 'import platform; print(platform.python_version())')
[ "$have" = "$want" ] || echo "note: python3 is $have, the reference run used $want. Output hashes may differ." >&2
# (Re)build .venv whenever requirements.lock changes.
lock_sum=$(python3 -c 'import hashlib; print(hashlib.sha256(open("requirements.lock","rb").read()).hexdigest())')
if [ ! -x .venv/bin/python ] || [ "$(cat .venv/requirements.lock.sha256 2>/dev/null)" != "$lock_sum" ]; then
  rm -rf .venv
  python3 -m venv .venv
  .venv/bin/pip install --quiet --require-hashes -r requirements.lock
  echo "$lock_sum" > .venv/requirements.lock.sha256
fi
PY=".venv/bin/python -I -B"

start=$(date +%s)
# Clean start: rebuild every intermediate and output. Raw downloads and the
# private analyst labels are kept.
rm -rf "$ZV_DATA_DIR/derived" results interactive
rm -f "$ZV_DATA_DIR"/private/*.csv
mkdir -p results

$PY 01_download.py         # public sources, checked against sources.lock.tsv
$PY 02_load.py             # raw zips to Parquet
$PY 03_build.py            # one table per section, with place flags
$PY 04_models.py           # Questions 1 to 5: counts, models, U1 to U3, scale
$PY 05_consistency.py      # Question 6: consistency checks (U4)
$PY 11_external_check.py   # outside registers (IBGE, FUNAI, INCRA), added after results.
                           # Runs before 06 and 08, which read its register categories.
$PY 06_diagnostics.py      # EXPLORATORY, added after results
$PY 07_charts.py
$PY 08_interactive.py      # privacy-safe aggregates only
$PY 09_classifier_check.py # needs the private analyst labels for its full output
echo "pipeline finished in $(( $(date +%s) - start )) s"
$PY 10_verify.py           # compare with the reference run's checksums