"""Self-provisioning dataset refresh — the server never needs a second machine. Runs as the `refresh` compose sidecar. On boot it provisions immediately when anything is missing (no success stamp, empty `places`, no basemap); afterwards it re-provisions weekly. One provision, ordered so peak disk stays ~140 GB on a 150 GB budget and steady state ~40 GB: 1. download the raw OSM planet (+94 GB, resumable across restarts) 2. filter it to the food CSV (+0.7 GB) 3. load places_new -> atomic table swap (app keeps serving throughout) 4. DELETE the raw planet + intermediates (-95 GB — the whole point) 5. extract the new planet basemap to .new (+36 GB, old file still serving) 6. verified atomic rename over basemap.pmtiles (build_basemap does both) Every step is an existing pipeline CLI run as a subprocess. A failure stops the run and leaves the stamp unwritten, but must never stop the sidecar: exiting would hand `restart: unless-stopped` a hot loop, and since the work is still due the next process would re-pull 94 GB from planet.openstreetmap.org immediately — the way to get the server's IP banned. Failures are therefore caught and retried on an exponential backoff (1 h, doubling to 6 h), with the planet download resuming its `.part` file. A database that simply does not answer is not a failure at all: it means "look again later", never "the dataset is missing". python -m pipeline.refresh # one provision, then exit python -m pipeline.refresh --loop # sidecar mode: provision when due, sleep 1 h """ import os import subprocess import sys import time from datetime import UTC, datetime from pathlib import Path import psycopg from pipeline.build_basemap import require_free_space DATA_DIR = Path(__file__).resolve().parent.parent / "data" STAMP = DATA_DIR / "last-refresh" BASEMAP = DATA_DIR / "basemap.pmtiles" PLANET_URL = "https://planet.openstreetmap.org/pbf/planet-latest.osm.pbf" PLANET = DATA_DIR / "planet-latest.osm.pbf" PLANET_PART = PLANET.with_name(PLANET.name + ".part") FOOD_PBF = DATA_DIR / "planet-food.food.osm.pbf" FOOD_CSV = DATA_DIR / "planet-food.csv" WEEK_S = 7 * 24 * 3600 CHECK_INTERVAL_S = 3600 BACKOFF_START_S = 3600 BACKOFF_MAX_S = 6 * 3600 # Measured 2026-08-25: planet PBF 94.4 GB; the filter's food PBF + CSV 0.6 GB, and the load # writes ~0.8 GB of table and GiST index into pgdata on the same filesystem. PLANET_PBF_BYTES = 95 * 10**9 HEADROOM_BYTES = 5 * 10**9 FILTER_LOAD_BYTES = 5 * 10**9 class DatabaseUnreachable(RuntimeError): """The database did not answer at all — the db container restarting, say. Distinct from an empty or absent `places`, which is what actually calls for a provision.""" def _run(*argv: str) -> None: print(f"refresh: {' '.join(argv)}", flush=True) subprocess.run([sys.executable, "-m", *argv], check=True) def _places_loaded() -> bool: dsn = os.environ["DATABASE_URL"].replace("postgis://", "postgresql://", 1) try: with psycopg.connect(dsn, connect_timeout=10) as conn: return bool(conn.execute("SELECT 1 FROM places LIMIT 1").fetchone()) except psycopg.errors.UndefinedTable: return False # table not created yet: the app's migrate has not run — provision except psycopg.OperationalError as exc: # Failing open here once meant a db restart mid-tick re-downloaded the whole planet. raise DatabaseUnreachable(str(exc).strip()) from exc def due(now_s: float) -> bool: if not STAMP.exists() or not BASEMAP.exists() or not _places_loaded(): return True return now_s - STAMP.stat().st_mtime >= WEEK_S def _download_need() -> int: """Bytes the planet download still has to write — a resumed `.part` is already paid for, and demanding the full 94 GB again would refuse a download that can actually finish.""" if PLANET.exists(): return HEADROOM_BYTES have = PLANET_PART.stat().st_size if PLANET_PART.exists() else 0 return max(PLANET_PBF_BYTES - have, 0) + HEADROOM_BYTES def provision() -> None: t0 = time.monotonic() require_free_space(DATA_DIR, _download_need(), "download the OSM planet") _run("pipeline.download_extract", PLANET_URL) require_free_space(DATA_DIR, FILTER_LOAD_BYTES, "filter and load the planet") _run("pipeline.filter_food", str(PLANET), str(FOOD_CSV)) _run("pipeline.load_places", str(FOOD_CSV)) # The raw planet exists only to be filtered; keeping it would double the disk bill for # data the CSV already carries. Next week's download starts from zero — by design. PLANET.unlink(missing_ok=True) FOOD_PBF.unlink(missing_ok=True) _run("pipeline.build_basemap", "--planet") STAMP.write_text(datetime.now(UTC).isoformat()) print(f"refresh: complete in {(time.monotonic() - t0) / 3600:.1f} h", flush=True) def main(loop: bool) -> None: penalty_s = 0.0 while True: try: if due(time.time()): provision() penalty_s = 0.0 except DatabaseUnreachable as exc: if not loop: raise print(f"refresh: database unreachable ({exc}) — checking again later", flush=True) except Exception as exc: if not loop: raise penalty_s = min(penalty_s * 2, BACKOFF_MAX_S) if penalty_s else BACKOFF_START_S print( f"refresh: provision failed ({type(exc).__name__}: {exc}) — retrying in " f"{penalty_s / 3600:.1f} h", flush=True, ) if not loop: return time.sleep(penalty_s or CHECK_INTERVAL_S) if __name__ == "__main__": main(loop="--loop" in sys.argv[1:])