fix(cadastre): count snapshot+per-cat requests in cadastre_jobs.requests_count

scrape_cadastre.py previously incremented requests_count only by
grid_walk_requests, so snapshot-only quarters (and the entire ekb_full
job after PR #179 skips grid-walk on broken geom) reported 0 NSPD
requests — misleading for observability.

Add HarvestResult.snapshot_requests counter (Phase 1 search + Phase 1.5
per-cat probes) and a total_requests property. Update Celery task to
write total_requests into cadastre_jobs.requests_count.

Tests added for the new property + Phase 1 snapshot_requests=1 assertion.
This commit is contained in:
lekss361 2026-05-15 17:10:03 +03:00
parent d5b57ccbf1
commit fa724f60ac
3 changed files with 30 additions and 3 deletions

View file

@ -69,9 +69,15 @@ class HarvestResult:
oncs_upserted: int = 0
enks_upserted: int = 0
zouit_upserted: int = 0
grid_walk_requests: int = 0
snapshot_requests: int = 0 # search_by_quarter (Phase 1) + per-cat probes (Phase 1.5)
grid_walk_requests: int = 0 # wms_feature_info (Phase 2-3)
phase_state: dict[str, Any] | None = None
@property
def total_requests(self) -> int:
"""Все NSPD HTTP requests для этого квартала (snapshot + grid-walk)."""
return self.snapshot_requests + self.grid_walk_requests
# ── Основной оркестратор ─────────────────────────────────────────────────────
@ -127,6 +133,7 @@ async def harvest_quarter(
update_progress({"phase": "snapshot_started", "quarter": quarter})
snapshot: QuarterSnapshot = await client.search_by_quarter(quarter)
result.snapshot_requests += 1
snapshot_stats = upsert_features(db, snapshot.features, source="search")
result.parcels_upserted += snapshot_stats["parcels"]
@ -179,6 +186,7 @@ async def harvest_quarter(
try:
cat_snapshot = await client.search_by_quarter(quarter, category_id=cat_id)
result.snapshot_requests += 1
except Exception as e:
logger.warning(
"harvest_quarter: per-cat probe failed cat=%d quarter=%s: %s",

View file

@ -208,17 +208,21 @@ def bulk_harvest_quarter_task(self: Any, quarter: str, job_id: int) -> dict[str,
"oncs": result.oncs_upserted,
"enks": result.enks_upserted,
"zouit": result.zouit_upserted,
"snapshot_requests": result.snapshot_requests,
"grid_walk_requests": result.grid_walk_requests,
"total_requests": result.total_requests,
}
result_dict = asyncio.run(_run())
# Успех: increment targets_done
# Успех: increment targets_done. requests_count учитывает ВСЕ NSPD HTTP calls
# (Phase 1 search + Phase 1.5 per-cat probes + Phase 2-3 grid-walk), не только
# grid_walk_requests — иначе snapshot-only quarters давали бы 0 запросов в metrics.
_update_job(
db,
job_id,
targets_done_inc=1,
requests_count_inc=result_dict.get("grid_walk_requests", 0),
requests_count_inc=result_dict.get("total_requests", 0),
heartbeat=True,
)
# Проверить не завершён ли job (все targets done)

View file

@ -429,11 +429,26 @@ async def test_harvest_quarter_calls_upsert_features() -> None:
client.search_by_quarter.assert_awaited_once_with("66:41:0303161")
mock_upsert.assert_called()
assert result.parcels_upserted == 1
# Phase 1 = 1 NSPD search call → snapshot_requests=1
assert result.snapshot_requests == 1
assert result.grid_walk_requests == 0
assert result.total_requests == 1
# Проверить что последний progress = done
assert progress_states[-1]["phase"] == "done"
def test_harvest_result_total_requests_property() -> None:
"""HarvestResult.total_requests = snapshot_requests + grid_walk_requests."""
from app.services.cadastre.bulk_harvest import HarvestResult
r = HarvestResult(quarter="66:41:0303161", snapshot_requests=3, grid_walk_requests=225)
assert r.total_requests == 228
empty = HarvestResult(quarter="66:41:0303161")
assert empty.total_requests == 0
# ── Dedup тест для grid-walk ─────────────────────────────────────────────────