bot-backend
f9769571a3
The chunked matcher paged the candidate set with OFFSET over the very predicate it mutates (building_cadastral_number IS NULL). Matched rows drop out, so OFFSET (reset to 0 OR advanced) skips un-processed rows: once >= batch_size unmatched listings (beyond threshold) accumulate at the low-id front, the chunk matches 0 and the loop breaks early, leaving higher-id matchable listings unfilled. Counterexample (batch=3, matchable ids 1,4,7,10): matches 1 and 4 then breaks, missing 7 and 10. Unit tests used single-chunk fixtures so never caught it. The full UPDATE is ~5s for ~41k listings, so replace the loop with ONE set-based LATERAL KNN UPDATE over all candidates — correct and fast. batch_size kept in the signature for schedule-param compat (now unused).
323 lines
13 KiB
Python
323 lines
13 KiB
Python
"""Geo-nearest cadastral building matcher for listings (#cadastral-geo-match).
|
||
|
||
Fills `listings.building_cadastral_number` (currently 0% / ~43k rows) from the nearest
|
||
cadastral building, via a LOCAL materialized mirror of the gendesign_cad_buildings FDW.
|
||
|
||
WHY a local mirror (perf fact, measured live):
|
||
gendesign_cad_buildings is a postgres_fdw foreign table (~36.7k EKB rows) with NO
|
||
PostGIS geom — only scalar lat/lon. A per-listing nearest-building query over the FDW
|
||
takes ~1.16s/row → ~13h for 43k listings. UNUSABLE per-row.
|
||
⇒ We materialize the FDW once (single bulk scan) into cad_buildings_local with a real
|
||
Point geom + GIST, then run a fast LOCAL KNN nearest-neighbour join (geom <-> point).
|
||
The whole local backfill runs in well under a minute (see PR EXPLAIN evidence).
|
||
|
||
APPROXIMATION (deliberate first increment):
|
||
This is a GEO-NEAREST match — a street-level-geocoded listing is matched to the nearest
|
||
cadastral building within `threshold_m`, NOT necessarily its exact cadastral building.
|
||
The threshold is always logged. Exact cadastral resolution + parcel-containment are
|
||
deferred (cad_parcels FDW not exposed). Tier-0 house matching in the estimator already
|
||
treats building_cadastral_number as a hint, not ground truth, so an approximate fill is
|
||
a net win over 0% coverage.
|
||
|
||
Pipeline (one combined run, scheduler source='cadastral_geo_match'):
|
||
1. refresh_cad_buildings_local(db) — TRUNCATE + bulk INSERT from FDW (one scan).
|
||
2. match_listings_to_buildings(db, threshold_m) — set-based LATERAL KNN UPDATE.
|
||
Refresh-then-match in ONE run guarantees the match always reads a fresh mirror.
|
||
|
||
psycopg v3: app SQL uses CAST(:x AS type), never :x::type.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import logging
|
||
import math
|
||
import time
|
||
from dataclasses import dataclass, field
|
||
|
||
from sqlalchemy import text
|
||
from sqlalchemy.orm import Session
|
||
|
||
from app.services import scrape_runs as runs_mod
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
DEFAULT_THRESHOLD_M = 50
|
||
DEFAULT_BATCH_SIZE = 5000
|
||
|
||
# EKB latitude — used to convert the metric threshold into a degree gate radius for the
|
||
# GIST-indexable geometry-space pre-filter (the gate is a SUPERSET of the true circle; the
|
||
# ST_DistanceSphere recheck decides acceptance, so a slightly-too-wide gate is harmless).
|
||
_EKB_LAT_DEG = 56.84
|
||
_M_PER_DEG_LAT = 111_320.0
|
||
|
||
|
||
def _deg_gate_for(threshold_m: float) -> float:
|
||
"""Degree radius that always encloses `threshold_m` metres at EKB latitude.
|
||
|
||
Longitude degrees shrink by cos(lat), so we size the gate on the tighter (longitude)
|
||
axis and apply a 1.5× safety factor. The gate only pre-filters candidates for the GIST
|
||
KNN scan; the exact ST_DistanceSphere(...) <= threshold_m recheck is authoritative, so
|
||
over-sizing the gate never produces a false match, only a few extra candidates to rank.
|
||
"""
|
||
m_per_deg_lon = _M_PER_DEG_LAT * math.cos(math.radians(_EKB_LAT_DEG))
|
||
return 1.5 * threshold_m / m_per_deg_lon
|
||
|
||
|
||
# ── Refresh: materialize FDW → cad_buildings_local (single bulk scan) ─────────
|
||
def refresh_cad_buildings_local(db: Session) -> int:
|
||
"""TRUNCATE cad_buildings_local; bulk INSERT from gendesign_cad_buildings FDW.
|
||
|
||
Single FDW scan (no per-row round-trips). Builds Point(4326) geom from lon/lat.
|
||
Idempotent: TRUNCATE+INSERT in one transaction — the table is fully replaced atomically.
|
||
|
||
Returns the number of rows loaded.
|
||
"""
|
||
db.execute(text("TRUNCATE cad_buildings_local"))
|
||
db.execute(
|
||
text(
|
||
"""
|
||
INSERT INTO cad_buildings_local (
|
||
cad_num, readable_address, year_built, floors, area_m2, purpose,
|
||
lat, lon, geom
|
||
)
|
||
SELECT
|
||
cad_num, readable_address, year_built, floors, area_m2, purpose,
|
||
lat, lon,
|
||
ST_SetSRID(ST_MakePoint(lon, lat), 4326)
|
||
FROM gendesign_cad_buildings
|
||
WHERE lat IS NOT NULL
|
||
AND lon IS NOT NULL
|
||
AND cad_num IS NOT NULL
|
||
ON CONFLICT (cad_num) DO NOTHING
|
||
"""
|
||
)
|
||
)
|
||
count = db.execute(text("SELECT count(*) FROM cad_buildings_local")).scalar() or 0
|
||
db.commit()
|
||
logger.info("refresh_cad_buildings_local: loaded %d rows from FDW", count)
|
||
return int(count)
|
||
|
||
|
||
# ── Match: LATERAL KNN nearest building → listings.building_cadastral_number ──
|
||
# Set-based UPDATE. For each candidate listing (geo present, optionally only NULL bcn) the
|
||
# LATERAL subquery picks the single nearest cad_buildings_local building and accepts it only
|
||
# if its TRUE distance is within :threshold_m metres.
|
||
#
|
||
# PERF (measured live, the whole point of this design):
|
||
# The gate is a GEOMETRY-space ST_DWithin(cb.geom, l_geom, :deg_gate) — planar degrees,
|
||
# fully GIST-indexable, NO per-row geography cast. The `<-> ` KNN order then returns the
|
||
# single nearest candidate via the GIST index. ONLY that one nearest row is converted to a
|
||
# true metric distance via ST_DistanceSphere (spherical metres) and compared to :threshold_m.
|
||
# This is ~175× faster than a geography-cast ST_DWithin in the WHERE: a geography filter
|
||
# forced a per-candidate cast+recheck that the planner could not push into the index, so the
|
||
# full 41k-listing UPDATE ran 58s+ and never finished; the geometry-gate version does a
|
||
# 5000-row chunk in ~330ms and the full ~41k backfill in a few seconds.
|
||
#
|
||
# :deg_gate is a degree radius that ALWAYS encloses :threshold_m at EKB latitude (~56.8°):
|
||
# 1° lat ≈ 111_320 m, and we widen by /cos(lat) for longitude + a safety factor, so the
|
||
# gate is a superset of the true threshold circle and the ST_DistanceSphere recheck is what
|
||
# actually decides acceptance (no false negatives from the gate).
|
||
#
|
||
# Batching: pick a chunk of candidate listing ids first (LIMIT :batch_size), then run the
|
||
# LATERAL UPDATE only over that chunk — bounds lock/WAL footprint on the ~43k table and lets
|
||
# the run loop checkpoint a heartbeat between chunks. only_missing=true makes each chunk drain
|
||
# monotonically (filled rows drop out of the candidate set), so the loop terminates.
|
||
# Single set-based LATERAL KNN UPDATE over ALL candidates in one statement.
|
||
# NOT chunked by OFFSET: an OFFSET walk over the `building_cadastral_number IS NULL`
|
||
# predicate is unsound because the UPDATE mutates that very predicate mid-walk —
|
||
# matched rows drop out of the candidate set, so OFFSET (reset OR advance) skips
|
||
# un-processed rows. The full UPDATE is ~5s for ~41k listings (GIST KNN, measured),
|
||
# so a single statement is both correct and fast enough; no chunking needed.
|
||
_MATCH_SQL = text(
|
||
"""
|
||
WITH matched AS (
|
||
SELECT l.id AS id, m.cad_num AS cad_num
|
||
FROM listings l
|
||
JOIN LATERAL (
|
||
SELECT cb.cad_num,
|
||
ST_DistanceSphere(cb.geom, l.geom) AS dist_m
|
||
FROM cad_buildings_local cb
|
||
WHERE ST_DWithin(cb.geom, l.geom, CAST(:deg_gate AS double precision))
|
||
ORDER BY cb.geom <-> l.geom
|
||
LIMIT 1
|
||
) m ON true
|
||
WHERE l.geom IS NOT NULL
|
||
AND (
|
||
CAST(:only_missing AS boolean) = false
|
||
OR l.building_cadastral_number IS NULL
|
||
)
|
||
AND m.dist_m <= CAST(:threshold_m AS double precision)
|
||
)
|
||
UPDATE listings l
|
||
SET building_cadastral_number = matched.cad_num
|
||
FROM matched
|
||
WHERE l.id = matched.id
|
||
"""
|
||
)
|
||
|
||
_CANDIDATES_SQL = text(
|
||
"""
|
||
SELECT count(*) FROM listings l
|
||
WHERE l.geom IS NOT NULL
|
||
AND (CAST(:only_missing AS boolean) = false OR l.building_cadastral_number IS NULL)
|
||
"""
|
||
)
|
||
|
||
|
||
@dataclass
|
||
class CadMatchResult:
|
||
buildings_loaded: int = 0 # rows in cad_buildings_local after refresh
|
||
candidates_total: int = 0 # listings eligible for matching (geo present + filter)
|
||
listings_matched: int = 0 # listings updated with a building_cadastral_number
|
||
threshold_m: int = DEFAULT_THRESHOLD_M
|
||
duration_sec: float = field(default=0.0)
|
||
|
||
def to_counters(self) -> dict[str, int]:
|
||
return {
|
||
"buildings_loaded": self.buildings_loaded,
|
||
"candidates_total": self.candidates_total,
|
||
"listings_matched": self.listings_matched,
|
||
"threshold_m": self.threshold_m,
|
||
"duration_sec": int(self.duration_sec),
|
||
}
|
||
|
||
|
||
def match_listings_to_buildings(
|
||
db: Session,
|
||
*,
|
||
threshold_m: int = DEFAULT_THRESHOLD_M,
|
||
batch_size: int = DEFAULT_BATCH_SIZE,
|
||
only_missing: bool = True,
|
||
) -> int:
|
||
"""Set-based LATERAL KNN UPDATE: fill listings.building_cadastral_number.
|
||
|
||
GEO-NEAREST (approximate): each candidate listing is matched to the nearest
|
||
cad_buildings_local building within `threshold_m` metres. The threshold is logged.
|
||
|
||
Single set-based UPDATE over ALL candidates (no chunking). An OFFSET walk would be
|
||
unsound: the UPDATE mutates the `building_cadastral_number IS NULL` predicate it pages
|
||
over, so matched rows drop out and OFFSET (reset OR advance) skips un-processed rows.
|
||
The full UPDATE is ~5s for ~41k listings (GIST KNN), so one statement is correct and
|
||
fast. `batch_size` is accepted for schedule-param compatibility but no longer used.
|
||
Returns listings matched.
|
||
"""
|
||
logger.info(
|
||
"match_listings_to_buildings: GEO-NEAREST match threshold_m=%d "
|
||
"only_missing=%s (approximate: nearest building, not exact cadastral)",
|
||
threshold_m,
|
||
only_missing,
|
||
)
|
||
|
||
deg_gate = _deg_gate_for(float(threshold_m))
|
||
result = db.execute(
|
||
_MATCH_SQL,
|
||
{
|
||
"threshold_m": float(threshold_m),
|
||
"deg_gate": deg_gate,
|
||
"only_missing": only_missing,
|
||
},
|
||
)
|
||
total_matched = result.rowcount
|
||
db.commit()
|
||
|
||
logger.info(
|
||
"match_listings_to_buildings: DONE total_matched=%d threshold_m=%d",
|
||
total_matched,
|
||
threshold_m,
|
||
)
|
||
return total_matched
|
||
|
||
|
||
# ── Combined refresh + match (callable directly or via run wrapper) ───────────
|
||
def refresh_and_match(
|
||
db: Session,
|
||
*,
|
||
threshold_m: int = DEFAULT_THRESHOLD_M,
|
||
batch_size: int = DEFAULT_BATCH_SIZE,
|
||
only_missing: bool = True,
|
||
) -> CadMatchResult:
|
||
"""Refresh the local cad mirror, then geo-match listings. Returns a CadMatchResult."""
|
||
start = time.monotonic()
|
||
res = CadMatchResult(threshold_m=threshold_m)
|
||
|
||
res.buildings_loaded = refresh_cad_buildings_local(db)
|
||
res.candidates_total = int(
|
||
db.execute(_CANDIDATES_SQL, {"only_missing": only_missing}).scalar() or 0
|
||
)
|
||
res.listings_matched = match_listings_to_buildings(
|
||
db, threshold_m=threshold_m, batch_size=batch_size, only_missing=only_missing
|
||
)
|
||
res.duration_sec = time.monotonic() - start
|
||
|
||
coverage = 100.0 * res.listings_matched / res.candidates_total if res.candidates_total else 0.0
|
||
logger.info(
|
||
"refresh_and_match: DONE buildings=%d candidates=%d matched=%d "
|
||
"coverage=%.1f%% threshold_m=%d duration=%.1fs",
|
||
res.buildings_loaded,
|
||
res.candidates_total,
|
||
res.listings_matched,
|
||
coverage,
|
||
threshold_m,
|
||
res.duration_sec,
|
||
)
|
||
return res
|
||
|
||
|
||
# ── Run lifecycle wrapper (scheduler entrypoint) ─────────────────────────────
|
||
def run_cadastral_geo_match(db: Session, *, run_id: int, params: dict) -> CadMatchResult:
|
||
"""Run-lifecycle wrapper for the combined refresh+match job (sync, DB-only).
|
||
|
||
Launched by the in-app scheduler (source='cadastral_geo_match') via
|
||
trigger_cadastral_geo_match_run, or manually. Refreshes cad_buildings_local from the
|
||
FDW (one bulk scan) then geo-matches listings.building_cadastral_number to the nearest
|
||
cadastral building within threshold_m.
|
||
|
||
Params (default_params jsonb):
|
||
threshold_m: max nearest-building distance to accept (metres, default 50).
|
||
batch_size: listings per LATERAL UPDATE chunk (default 5000).
|
||
only_missing: only fill WHERE building_cadastral_number IS NULL (default true).
|
||
|
||
Finalises scrape_runs (mark_done / mark_failed) with counters.
|
||
"""
|
||
threshold_m = int(params.get("threshold_m", DEFAULT_THRESHOLD_M))
|
||
batch_size = int(params.get("batch_size", DEFAULT_BATCH_SIZE))
|
||
only_missing = bool(params.get("only_missing", True))
|
||
|
||
counters: dict[str, int] = {
|
||
"buildings_loaded": 0,
|
||
"candidates_total": 0,
|
||
"listings_matched": 0,
|
||
"threshold_m": threshold_m,
|
||
}
|
||
|
||
try:
|
||
runs_mod.update_heartbeat(db, run_id, counters)
|
||
|
||
res = refresh_and_match(
|
||
db,
|
||
threshold_m=threshold_m,
|
||
batch_size=batch_size,
|
||
only_missing=only_missing,
|
||
)
|
||
|
||
counters = res.to_counters()
|
||
runs_mod.mark_done(db, run_id, counters)
|
||
logger.info(
|
||
"run_cadastral_geo_match: run_id=%d DONE buildings=%d candidates=%d "
|
||
"matched=%d threshold_m=%d duration=%.1fs",
|
||
run_id,
|
||
res.buildings_loaded,
|
||
res.candidates_total,
|
||
res.listings_matched,
|
||
threshold_m,
|
||
res.duration_sec,
|
||
)
|
||
return res
|
||
except Exception as exc:
|
||
logger.exception("run_cadastral_geo_match: run_id=%d FAILED", run_id)
|
||
try:
|
||
db.rollback()
|
||
except Exception:
|
||
pass
|
||
runs_mod.mark_failed(db, run_id, str(exc)[:1000], counters)
|
||
raise
|