"""Fetch one OSM PBF — a Geofabrik regional extract, or the whole planet — resumably. See stack doc §10, "Third-party data etiquette": one connection, no accelerators, a descriptive User-Agent, and never a second download of a file we already hold. Resuming is the delicate part. `planet-latest.osm.pbf` is regenerated weekly, so a `.part` left by an interrupted run may belong to a planet that no longer exists upstream; appending this week's tail to last week's head yields a plausible, corrupt PBF that would flow straight into the atomic table swap. The upstream validator (ETag, else Last-Modified) is therefore recorded beside the `.part` and replayed as `If-Range`: the server answers 206 only if the bytes we hold still belong to the file it is serving, and answers 200 — the whole body, from zero — when they do not. A partial with no recorded validator cannot be proven and is dropped. Finally, servers that publish a `.md5` sibling (planet.openstreetmap.org does, verified 2026-08-25; Geofabrik too) get the finished file checksummed before the rename. """ import hashlib import sys import time import urllib.error from pathlib import Path from urllib.request import Request, urlopen URL = "https://download.geofabrik.de/europe/finland-latest.osm.pbf" DATA_DIR = Path(__file__).resolve().parent.parent / "data" USER_AGENT = "lunchvote-import/0.1" CHUNK = 1 << 20 HASH_BLOCK = 1 << 22 # Nothing supervises this download: without a socket timeout a stalled connection hangs the # refresh sidecar forever, and the weekly refresh silently stops happening. TIMEOUT_S = 120 # ~94k chunks per planet; one line each would be 94k lines into an uncapped docker log. PROGRESS_EVERY_S = 30 def _resume_offset(part: Path, meta: Path) -> tuple[int, str]: """How much of `part` may be resumed, and the validator that vouches for it.""" if not part.exists(): return 0, "" validator = meta.read_text().strip() if meta.exists() else "" if not validator: # Unprovable provenance: re-downloading is cheaper than importing a spliced planet. print(f"discarding {part.name}: no recorded upstream validator") part.unlink() return 0, "" return part.stat().st_size, validator def _stream(resp, part: Path, have: int, total: int) -> int: got = have spoke_at = 0.0 with open(part, "ab" if have else "wb") as fh: while chunk := resp.read(CHUNK): fh.write(chunk) got += len(chunk) if time.monotonic() - spoke_at >= PROGRESS_EVERY_S: print(f"{got / 1e9:.2f} / {total / 1e9:.2f} GB", flush=True) spoke_at = time.monotonic() return got def _published_md5(url: str) -> str | None: """The digest the server publishes beside the file, or None when it publishes none.""" req = Request(f"{url}.md5", headers={"User-Agent": USER_AGENT}) try: with urlopen(req, timeout=TIMEOUT_S) as resp: return resp.read(1 << 12).split()[0].decode() except urllib.error.HTTPError: return None def _file_md5(path: Path) -> str: digest = hashlib.md5() with open(path, "rb") as fh: while block := fh.read(HASH_BLOCK): digest.update(block) return digest.hexdigest() def download(url: str = URL, data_dir: Path = DATA_DIR) -> Path: data_dir.mkdir(exist_ok=True) target = data_dir / url.rsplit("/", 1)[-1] if target.exists(): print(f"{target} already present ({target.stat().st_size} bytes) — not re-downloading") return target part = target.with_name(target.name + ".part") meta = part.with_name(part.name + ".meta") have, held = _resume_offset(part, meta) headers = {"User-Agent": USER_AGENT} if have: headers["Range"] = f"bytes={have}-" headers["If-Range"] = held print(f"resuming {part.name} at {have / 1e9:.2f} GB (If-Range: {held})", flush=True) try: with urlopen(Request(url, headers=headers), timeout=TIMEOUT_S) as resp: upstream = resp.headers.get("ETag") or resp.headers.get("Last-Modified") if have and resp.status == 200: print("upstream file changed — restarting the download from zero", flush=True) have = 0 elif have and upstream and upstream != held: # A 206 for bytes of a file the response itself says is a different one: # some intermediary dropped our If-Range. The head is unusable. part.unlink() meta.unlink(missing_ok=True) raise RuntimeError( f"{url}: resumed a partial recorded as {held} but the server answered for " f"{upstream}; dropped {part.name} — the next run downloads it cleanly" ) if upstream: meta.write_text(upstream) total = int(resp.headers.get("Content-Length", 0)) + have got = _stream(resp, part, have, total) except urllib.error.HTTPError as exc: if exc.code != 416: raise # Every byte is already here: an earlier run died between the last chunk and the # rename (a failed checksum fetch does this too). Nothing to fetch, so verify below. print(f"{part.name} already holds the whole file (416 for the resume request)") got = total = part.stat().st_size if total and got != total: raise RuntimeError( f"{url}: stream ended at {got} of {total} bytes — {part.name} kept, the next run " "resumes it" ) expected = _published_md5(url) if expected is None: print(f"{url}.md5 is not published — skipping checksum verification") else: actual = _file_md5(part) if actual != expected: part.unlink() meta.unlink(missing_ok=True) raise RuntimeError( f"{part.name} hashes to {actual}, {url}.md5 says {expected} — deleted. An " "upstream file that rolled over mid-download hashes like this too; rerun." ) print(f"md5 {actual} matches {url}.md5") part.rename(target) meta.unlink(missing_ok=True) print(f"{target} ({target.stat().st_size} bytes)") return target if __name__ == "__main__": download(sys.argv[1] if len(sys.argv) > 1 else URL)