"""PBF -> CSV of named food POIs, with centroids for ways and relations. Two passes, mirroring the osmium-tool recipe in stack doc §10. Pass 1 streams the source PBF and copies matching objects into a small temporary PBF *together with the objects they reference* (nodes of matched ways, ways and nodes of matched relations). That is the trick spec §8 calls critical: the geometry pass then runs on a self-contained file a few MB in size, so building-outline restaurants keep their coordinates without a planet-sized node cache. Neither pass holds the source in memory; both stream. """ import csv import sys import time from pathlib import Path import osmium # Whitelists tuned against the planet-wide survey (pipeline/diagnose_tags.py, 2026-08-25): # every addition carried a measured food signal — canteen 5.9% cuisine-tagged, pastry 34k # objects 93% named, coffee 9.9% cuisine, ice_cream 14.7% — and marketplace (98k, 77% named) # is the lunch venue across much of the global south. Measured and rejected: kiosk (0.6% # cuisine, 47% named), caterer (19% cuisine but not walk-in), convenience/supermarket # (0.3-0.4%), tea (3.2%), nightclub (0.3%). AMENITIES = frozenset( ( "restaurant", "cafe", "fast_food", "bar", "pub", "biergarten", "food_court", "ice_cream", "canteen", "marketplace", ) ) SHOPS = frozenset(("bakery", "deli", "pastry", "confectionery", "ice_cream", "coffee")) FOOD_KEYS = (("amenity", AMENITIES), ("shop", SHOPS)) COLUMNS = ( "osm_type", "osm_id", "name", "lon", "lat", "category", "cuisine", "opening_hours", "diet_vegan", "diet_vegetarian", "diet_gluten_free", ) DIET_KEYS = ("diet:vegan", "diet:vegetarian", "diet:gluten_free") def is_food(tags) -> bool: # Values are split on ';': the planet holds 1,246 real venues tagged restaurant;cafe, # pub;restaurant and the like, which an exact match silently drops. for key, values in FOOD_KEYS: raw = tags.get(key) if raw and (raw in values or (";" in raw and any(p.strip() in values for p in raw.split(";")))): return True return False def category(tags) -> str: """The whitelist value that matched, e.g. 'restaurant' or 'marketplace'. For a semicolon multi-value, the first whitelisted part wins — amenity before shop, matching is_food's precedence.""" for key, values in FOOD_KEYS: raw = tags.get(key) if not raw: continue if raw in values: return raw for part in raw.split(";"): if part.strip() in values: return part.strip() return "" def display_name(tags) -> str | None: """The votable name. Planet-wide, 44.7k venues carry only name:en and 4.1k only a brand — a real name either way; only the truly anonymous (~282k) stay unvotable.""" return tags.get("name") or tags.get("name:en") or tags.get("brand") class Counts: def __init__(self): self.node = self.way = self.relation = 0 self.unnamed = 0 self.relations_named = 0 self.relations_no_area = 0 self.ways_no_location = 0 def prefilter(src: Path, dest: Path) -> None: """Copy food-tagged objects plus the objects they reference into `dest`. KeyFilter + the Python is_food test, not the C++ TagFilter: exact k=v filtering is faster but cannot see the whitelisted part inside a semicolon multi-value.""" scanned = 0 with osmium.BackReferenceWriter( str(dest), ref_src=str(src), overwrite=True, relation_depth=1 ) as writer: processor = osmium.FileProcessor(str(src)).with_filter( osmium.filter.KeyFilter("amenity", "shop") ) for obj in processor: if not is_food(obj.tags): continue writer.add(obj) scanned += 1 if scanned % 20_000 == 0: print(f" matched {scanned}") print(f" matched {scanned} objects") def ring_centroid(coords: list[tuple[float, float]]) -> tuple[float, float]: """Area-weighted centroid of a ring, falling back to the vertex mean when the ring encloses no area (an open way, or one folded back on itself). The shoelace runs on raw degrees. Scaling longitude by cos(lat) would scale every sub-triangle area by the same factor, which cancels in the weighted mean, so the local equirectangular correction is a no-op here. """ n = len(coords) a2 = cx = cy = 0.0 for i in range(n): x0, y0 = coords[i] x1, y1 = coords[(i + 1) % n] cross = x0 * y1 - x1 * y0 a2 += cross cx += (x0 + x1) * cross cy += (y0 + y1) * cross if a2 == 0.0: return sum(x for x, _ in coords) / n, sum(y for _, y in coords) / n return cx / (3.0 * a2), cy / (3.0 * a2) def row(osm_type: str, osm_id: int, tags, lon: float, lat: float) -> list: return [ osm_type, osm_id, display_name(tags), f"{lon:.7f}", f"{lat:.7f}", category(tags), tags.get("cuisine", ""), tags.get("opening_hours", ""), *(tags.get(k, "") for k in DIET_KEYS), ] def emit(filtered: Path, out_csv: Path, counts: Counts) -> None: with open(out_csv, "w", newline="", encoding="utf-8") as fh: out = csv.writer(fh) out.writerow(COLUMNS) # with_areas() also enables the node-location index, which is what puts coordinates # on w.nodes below. Ways are read directly rather than through their assembled Area # so that non-closed ways — rare but real — still yield a centroid. processor = osmium.FileProcessor(str(filtered)).with_areas( osmium.filter.KeyFilter("amenity", "shop") ) for obj in processor: if isinstance(obj, osmium.osm.Area): if obj.from_way() or not is_food(obj.tags) or display_name(obj.tags) is None: continue # Multi-outer-ring relations get one concatenated shoelace, which adds small # cross-terms between rings. At 55 relations in Finland, nearly all single-ring, # the error is metres; per-ring area weighting is the fix if that ever changes. coords = [(n.lon, n.lat) for ring in obj.outer_rings() for n in ring] if not coords: # Planet-only: an assembled area whose rings carry no locations. Skip; # it lands in relations_no_area, like any relation without geometry. continue lon, lat = ring_centroid(coords) out.writerow(row("r", obj.orig_id(), obj.tags, lon, lat)) counts.relation += 1 continue if not is_food(obj.tags): continue named = display_name(obj.tags) is not None counts.unnamed += not named if isinstance(obj, osmium.osm.Relation): counts.relations_named += named elif not named: continue elif isinstance(obj, osmium.osm.Node): out.writerow(row("n", obj.id, obj.tags, obj.location.lon, obj.location.lat)) counts.node += 1 else: coords = [(n.lon, n.lat) for n in obj.nodes if n.location.valid()] if not coords: counts.ways_no_location += 1 continue if len(coords) > 1 and coords[0] == coords[-1]: coords.pop() lon, lat = ring_centroid(coords) out.writerow(row("w", obj.id, obj.tags, lon, lat)) counts.way += 1 counts.relations_no_area = counts.relations_named - counts.relation # Sidecar for analyse_coverage: what the CSV cannot show, because it is what was left out. out_csv.with_suffix(".dropped").write_text( f"unnamed={counts.unnamed}\n" f"relations_no_area={counts.relations_no_area}\n" f"ways_no_location={counts.ways_no_location}\n" ) def run(src: Path, out_csv: Path) -> Counts: filtered = out_csv.parent / (out_csv.stem + ".food.osm.pbf") counts = Counts() t0 = time.monotonic() print(f"pass 1: filtering {src}") prefilter(src, filtered) t1 = time.monotonic() print(f"pass 1 done in {t1 - t0:.1f}s -> {filtered} ({filtered.stat().st_size} bytes)") print("pass 2: geometry + csv") emit(filtered, out_csv, counts) t2 = time.monotonic() print(f"pass 2 done in {t2 - t1:.1f}s") total = counts.node + counts.way + counts.relation print( f"{out_csv}: {total} rows (n={counts.node} w={counts.way} r={counts.relation}); " f"dropped {counts.unnamed} unnamed; " f"{counts.relations_no_area} named matched relations produced no area; " f"{counts.ways_no_location} ways had no usable node locations" ) print(f"total {t2 - t0:.1f}s") return counts if __name__ == "__main__": run(Path(sys.argv[1]), Path(sys.argv[2]))