"""Build data/basemap.pmtiles — the self-hosted vector basemap for the creation map. `pmtiles extract` pulls tiles from Protomaps' daily planet basemap build (https://build.protomaps.com, ODbL, © OpenStreetMap contributors + Protomaps) into one static file the app serves itself with HTTP range requests (rooms/basemap.py) — no tile server process, and no third-party tile traffic from users' browsers. Two modes: python -m pipeline.build_basemap [YYYYMMDD] # REGIONS below (~1 GB) python -m pipeline.build_basemap --planet [YYYYMMDD] # whole world z13 (~36 GB) Regional extracts mirror a partially loaded `places` table; --planet matches the full planet import and is what the server's weekly refresh runs. The extract lands in basemap.pmtiles.new and only replaces the served file once three things hold: the disk had room before it started, the bytes really arrived (the killed-download failure mode is a sparse file whose directories parse while the tile data is holes — shipped once, 2026-08-25), and the tile schema still matches the vendored renderer. Needs the go-pmtiles binary (https://github.com/protomaps/go-pmtiles) on PATH or at data/pmtiles. """ import json import os import shutil import subprocess import sys from datetime import UTC, datetime, timedelta from pathlib import Path from urllib.request import Request, urlopen DATA_DIR = Path(__file__).resolve().parent.parent / "data" BINARY = shutil.which("pmtiles") or str(DATA_DIR / "pmtiles") OUT = DATA_DIR / "basemap.pmtiles" REGION = DATA_DIR / "basemap-region.geojson" # Measured 2026-08-25: the z13 planet extract is 35.5 GB, the two-region one 1.0 GB. PLANET_EXTRACT_BYTES = 40 * 10**9 REGION_EXTRACT_BYTES = 2 * 10**9 # The tileset schema and the renderer vendored in templates/rooms/create.html are a matched # pair: protomaps-leaflet 5.x paints v4 layer names, 4.x painted v3's. A mismatch draws # NOTHING and says nothing — tiles decode, zero features match the paint rules — which is # exactly how the app once shipped a blank map. Protomaps bumping the schema on any weekly # build would re-introduce it, so the automated path asserts the pairing. RENDERER = "protomaps-leaflet 5.1.0" SCHEMA_MAJOR = "4" REQUIRED_LAYERS = frozenset({"earth", "water", "roads", "places"}) # 16 x 64 KB spread through the archive: enough to catch holes, nothing on a 36 GB read. PROBE_WINDOWS = 16 PROBE_BYTES = 1 << 16 # Generous lon/lat boxes; the extract cost scales with area, not with box tightness. REGIONS = { "Finland": [[18.5, 59.2], [32.2, 59.2], [32.2, 70.4], [18.5, 70.4], [18.5, 59.2]], "Kenya": [[33.4, -5.2], [42.2, -5.2], [42.2, 5.6], [33.4, 5.6], [33.4, -5.2]], } def latest_build() -> str: """Yesterday's build: today's may still be uploading when this runs.""" for days_back in (1, 2, 3): stamp = (datetime.now(UTC) - timedelta(days=days_back)).strftime("%Y%m%d") # The bucket 403s HEAD requests (observed 2026-08-25); a one-byte GET probes # existence the same way the extract itself will read. req = Request( f"https://build.protomaps.com/{stamp}.pmtiles", headers={"Range": "bytes=0-0"} ) try: with urlopen(req): return stamp except OSError: continue raise SystemExit("no recent build found at build.protomaps.com") def require_free_space(path: Path, need: int, what: str) -> None: """Refuse before the write rather than 30 GB into it: pgdata shares this filesystem, and Postgres stops accepting writes when it fills.""" free = shutil.disk_usage(path).free if free < need: raise RuntimeError( f"refusing to {what}: {free / 1e9:.1f} GB free on {path}, {need / 1e9:.1f} GB needed" ) def _allocated_bytes(path: Path) -> int: return path.stat().st_blocks * 512 def _blank_fraction(path: Path) -> float: """Share of evenly spaced sample windows that are pure zeros. Tile data is gzipped, so a real archive has none; a killed extract's holes read back as zeros.""" span = max(path.stat().st_size - PROBE_BYTES, 0) blank = 0 with open(path, "rb") as fh: for i in range(1, PROBE_WINDOWS + 1): fh.seek(span * i // (PROBE_WINDOWS + 1)) blank += not fh.read(PROBE_BYTES).strip(b"\0") return blank / PROBE_WINDOWS def check_complete(path: Path) -> None: """Refuse an archive whose bytes never all arrived. A killed `pmtiles extract` leaves a plausible file: full apparent size, parseable header and directories, tile data all zeros — and, being sparse, few allocated blocks. Allocated blocks alone cannot decide it, because btrfs/ZFS transparent compression makes a perfectly good archive report the same shortfall, so a refusal needs the content probe to agree that what is down there really is holes. """ size = path.stat().st_size allocated = _allocated_bytes(path) blank = _blank_fraction(path) if blank >= 0.5 or (blank > 0 and allocated < size * 0.9): raise SystemExit( f"{path} reads as holes, not tiles: {blank:.0%} of sampled windows are all zeros " f"and {allocated / 1e9:.1f} of {size / 1e9:.1f} GB are allocated — the extract " "died mid-download; rerun it." ) def read_metadata(path: Path) -> dict: """`pmtiles show --metadata` parses the header and the directories on its way to the JSON, so this is the structural check and the schema check in one call.""" shown = subprocess.run( [BINARY, "show", "--metadata", str(path)], check=True, capture_output=True, text=True ) return json.loads(shown.stdout) def check_schema(meta: dict) -> None: version = str(meta.get("version", "?")) layers = {layer["id"] for layer in meta.get("vector_layers", [])} if version.split(".")[0] != SCHEMA_MAJOR: raise SystemExit( f"basemap schema v{version}, expected v{SCHEMA_MAJOR}.x: the vendored {RENDERER} " f"paints v{SCHEMA_MAJOR} layer names and would draw a blank map without an error. " "Upgrade the renderer in templates/rooms/create.html to the major that pairs with " f"v{version}, then rebuild." ) missing = REQUIRED_LAYERS - layers if missing: raise SystemExit( f"basemap v{version} lacks the layers {sorted(missing)} that {RENDERER} paints; it " f"carries {sorted(layers)}." ) print(f"schema v{version}, layers {sorted(layers)}") def main(stamp: str | None, planet: bool) -> None: stamp = stamp or latest_build() fresh = OUT.with_suffix(".pmtiles.new") cmd = [ BINARY, "extract", f"https://build.protomaps.com/{stamp}.pmtiles", str(fresh), # Must match maxDataZoom in templates/rooms/create.html: deeper zooms overzoom # this z13 vector data client-side. Each level deeper ~4x the tile count. "--maxzoom=13", ] if planet: print(f"extracting the whole planet (z13) from build {stamp}") else: REGION.write_text( json.dumps( {"type": "MultiPolygon", "coordinates": [[ring] for ring in REGIONS.values()]} ) ) cmd.append(f"--region={REGION}") print(f"extracting {', '.join(REGIONS)} from build {stamp}") need = PLANET_EXTRACT_BYTES if planet else REGION_EXTRACT_BYTES require_free_space(OUT.parent, need, f"extract build {stamp}") fresh.unlink(missing_ok=True) try: subprocess.run(cmd, check=True) check_complete(fresh) check_schema(read_metadata(fresh)) # The app opens the file per request, so the swap is invisible to in-flight readers. os.replace(fresh, OUT) finally: # Without this a crashed extract parks 36 GB beside the 36 GB live file until the # next weekly attempt, on the disk pgdata is writing to. fresh.unlink(missing_ok=True) print(f"{OUT} ({OUT.stat().st_size / 1e9:.2f} GB)") if __name__ == "__main__": args = [a for a in sys.argv[1:] if a != "--planet"] main(args[0] if args else None, planet="--planet" in sys.argv[1:])