fix(cadastre): count snapshot+per-cat requests in cadastre_jobs.requests_count
scrape_cadastre.py previously incremented requests_count only by grid_walk_requests, so snapshot-only quarters (and the entire ekb_full job after PR #179 skips grid-walk on broken geom) reported 0 NSPD requests — misleading for observability. Add HarvestResult.snapshot_requests counter (Phase 1 search + Phase 1.5 per-cat probes) and a total_requests property. Update Celery task to write total_requests into cadastre_jobs.requests_count. Tests added for the new property + Phase 1 snapshot_requests=1 assertion.
This commit is contained in:
parent
d5b57ccbf1
commit
fa724f60ac
3 changed files with 30 additions and 3 deletions
|
|
@ -69,9 +69,15 @@ class HarvestResult:
|
|||
oncs_upserted: int = 0
|
||||
enks_upserted: int = 0
|
||||
zouit_upserted: int = 0
|
||||
grid_walk_requests: int = 0
|
||||
snapshot_requests: int = 0 # search_by_quarter (Phase 1) + per-cat probes (Phase 1.5)
|
||||
grid_walk_requests: int = 0 # wms_feature_info (Phase 2-3)
|
||||
phase_state: dict[str, Any] | None = None
|
||||
|
||||
@property
|
||||
def total_requests(self) -> int:
|
||||
"""Все NSPD HTTP requests для этого квартала (snapshot + grid-walk)."""
|
||||
return self.snapshot_requests + self.grid_walk_requests
|
||||
|
||||
|
||||
# ── Основной оркестратор ─────────────────────────────────────────────────────
|
||||
|
||||
|
|
@ -127,6 +133,7 @@ async def harvest_quarter(
|
|||
update_progress({"phase": "snapshot_started", "quarter": quarter})
|
||||
|
||||
snapshot: QuarterSnapshot = await client.search_by_quarter(quarter)
|
||||
result.snapshot_requests += 1
|
||||
|
||||
snapshot_stats = upsert_features(db, snapshot.features, source="search")
|
||||
result.parcels_upserted += snapshot_stats["parcels"]
|
||||
|
|
@ -179,6 +186,7 @@ async def harvest_quarter(
|
|||
|
||||
try:
|
||||
cat_snapshot = await client.search_by_quarter(quarter, category_id=cat_id)
|
||||
result.snapshot_requests += 1
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"harvest_quarter: per-cat probe failed cat=%d quarter=%s: %s",
|
||||
|
|
|
|||
|
|
@ -208,17 +208,21 @@ def bulk_harvest_quarter_task(self: Any, quarter: str, job_id: int) -> dict[str,
|
|||
"oncs": result.oncs_upserted,
|
||||
"enks": result.enks_upserted,
|
||||
"zouit": result.zouit_upserted,
|
||||
"snapshot_requests": result.snapshot_requests,
|
||||
"grid_walk_requests": result.grid_walk_requests,
|
||||
"total_requests": result.total_requests,
|
||||
}
|
||||
|
||||
result_dict = asyncio.run(_run())
|
||||
|
||||
# Успех: increment targets_done
|
||||
# Успех: increment targets_done. requests_count учитывает ВСЕ NSPD HTTP calls
|
||||
# (Phase 1 search + Phase 1.5 per-cat probes + Phase 2-3 grid-walk), не только
|
||||
# grid_walk_requests — иначе snapshot-only quarters давали бы 0 запросов в metrics.
|
||||
_update_job(
|
||||
db,
|
||||
job_id,
|
||||
targets_done_inc=1,
|
||||
requests_count_inc=result_dict.get("grid_walk_requests", 0),
|
||||
requests_count_inc=result_dict.get("total_requests", 0),
|
||||
heartbeat=True,
|
||||
)
|
||||
# Проверить не завершён ли job (все targets done)
|
||||
|
|
|
|||
|
|
@ -429,11 +429,26 @@ async def test_harvest_quarter_calls_upsert_features() -> None:
|
|||
client.search_by_quarter.assert_awaited_once_with("66:41:0303161")
|
||||
mock_upsert.assert_called()
|
||||
assert result.parcels_upserted == 1
|
||||
# Phase 1 = 1 NSPD search call → snapshot_requests=1
|
||||
assert result.snapshot_requests == 1
|
||||
assert result.grid_walk_requests == 0
|
||||
assert result.total_requests == 1
|
||||
|
||||
# Проверить что последний progress = done
|
||||
assert progress_states[-1]["phase"] == "done"
|
||||
|
||||
|
||||
def test_harvest_result_total_requests_property() -> None:
|
||||
"""HarvestResult.total_requests = snapshot_requests + grid_walk_requests."""
|
||||
from app.services.cadastre.bulk_harvest import HarvestResult
|
||||
|
||||
r = HarvestResult(quarter="66:41:0303161", snapshot_requests=3, grid_walk_requests=225)
|
||||
assert r.total_requests == 228
|
||||
|
||||
empty = HarvestResult(quarter="66:41:0303161")
|
||||
assert empty.total_requests == 0
|
||||
|
||||
|
||||
# ── Dedup тест для grid-walk ─────────────────────────────────────────────────
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue