From fa724f60ac80e2d477b60d8d92c989aeef49b561 Mon Sep 17 00:00:00 2001 From: lekss361 Date: Fri, 15 May 2026 17:10:03 +0300 Subject: [PATCH] fix(cadastre): count snapshot+per-cat requests in cadastre_jobs.requests_count MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit scrape_cadastre.py previously incremented requests_count only by grid_walk_requests, so snapshot-only quarters (and the entire ekb_full job after PR #179 skips grid-walk on broken geom) reported 0 NSPD requests — misleading for observability. Add HarvestResult.snapshot_requests counter (Phase 1 search + Phase 1.5 per-cat probes) and a total_requests property. Update Celery task to write total_requests into cadastre_jobs.requests_count. Tests added for the new property + Phase 1 snapshot_requests=1 assertion. --- backend/app/services/cadastre/bulk_harvest.py | 10 +++++++++- backend/app/workers/tasks/scrape_cadastre.py | 8 ++++++-- backend/tests/services/test_cadastre_bulk.py | 15 +++++++++++++++ 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/backend/app/services/cadastre/bulk_harvest.py b/backend/app/services/cadastre/bulk_harvest.py index bcc262a3..78f19fc2 100644 --- a/backend/app/services/cadastre/bulk_harvest.py +++ b/backend/app/services/cadastre/bulk_harvest.py @@ -69,9 +69,15 @@ class HarvestResult: oncs_upserted: int = 0 enks_upserted: int = 0 zouit_upserted: int = 0 - grid_walk_requests: int = 0 + snapshot_requests: int = 0 # search_by_quarter (Phase 1) + per-cat probes (Phase 1.5) + grid_walk_requests: int = 0 # wms_feature_info (Phase 2-3) phase_state: dict[str, Any] | None = None + @property + def total_requests(self) -> int: + """Все NSPD HTTP requests для этого квартала (snapshot + grid-walk).""" + return self.snapshot_requests + self.grid_walk_requests + # ── Основной оркестратор ───────────────────────────────────────────────────── @@ -127,6 +133,7 @@ async def harvest_quarter( update_progress({"phase": "snapshot_started", "quarter": quarter}) snapshot: QuarterSnapshot = await client.search_by_quarter(quarter) + result.snapshot_requests += 1 snapshot_stats = upsert_features(db, snapshot.features, source="search") result.parcels_upserted += snapshot_stats["parcels"] @@ -179,6 +186,7 @@ async def harvest_quarter( try: cat_snapshot = await client.search_by_quarter(quarter, category_id=cat_id) + result.snapshot_requests += 1 except Exception as e: logger.warning( "harvest_quarter: per-cat probe failed cat=%d quarter=%s: %s", diff --git a/backend/app/workers/tasks/scrape_cadastre.py b/backend/app/workers/tasks/scrape_cadastre.py index 19cb7699..27dd9ec9 100644 --- a/backend/app/workers/tasks/scrape_cadastre.py +++ b/backend/app/workers/tasks/scrape_cadastre.py @@ -208,17 +208,21 @@ def bulk_harvest_quarter_task(self: Any, quarter: str, job_id: int) -> dict[str, "oncs": result.oncs_upserted, "enks": result.enks_upserted, "zouit": result.zouit_upserted, + "snapshot_requests": result.snapshot_requests, "grid_walk_requests": result.grid_walk_requests, + "total_requests": result.total_requests, } result_dict = asyncio.run(_run()) - # Успех: increment targets_done + # Успех: increment targets_done. requests_count учитывает ВСЕ NSPD HTTP calls + # (Phase 1 search + Phase 1.5 per-cat probes + Phase 2-3 grid-walk), не только + # grid_walk_requests — иначе snapshot-only quarters давали бы 0 запросов в metrics. _update_job( db, job_id, targets_done_inc=1, - requests_count_inc=result_dict.get("grid_walk_requests", 0), + requests_count_inc=result_dict.get("total_requests", 0), heartbeat=True, ) # Проверить не завершён ли job (все targets done) diff --git a/backend/tests/services/test_cadastre_bulk.py b/backend/tests/services/test_cadastre_bulk.py index c5e9b722..58e61a2c 100644 --- a/backend/tests/services/test_cadastre_bulk.py +++ b/backend/tests/services/test_cadastre_bulk.py @@ -429,11 +429,26 @@ async def test_harvest_quarter_calls_upsert_features() -> None: client.search_by_quarter.assert_awaited_once_with("66:41:0303161") mock_upsert.assert_called() assert result.parcels_upserted == 1 + # Phase 1 = 1 NSPD search call → snapshot_requests=1 + assert result.snapshot_requests == 1 + assert result.grid_walk_requests == 0 + assert result.total_requests == 1 # Проверить что последний progress = done assert progress_states[-1]["phase"] == "done" +def test_harvest_result_total_requests_property() -> None: + """HarvestResult.total_requests = snapshot_requests + grid_walk_requests.""" + from app.services.cadastre.bulk_harvest import HarvestResult + + r = HarvestResult(quarter="66:41:0303161", snapshot_requests=3, grid_walk_requests=225) + assert r.total_requests == 228 + + empty = HarvestResult(quarter="66:41:0303161") + assert empty.total_requests == 0 + + # ── Dedup тест для grid-walk ─────────────────────────────────────────────────