fix(cadastre): count snapshot+per-cat requests in cadastre_jobs.requests_count
scrape_cadastre.py previously incremented requests_count only by grid_walk_requests, so snapshot-only quarters (and the entire ekb_full job after PR #179 skips grid-walk on broken geom) reported 0 NSPD requests — misleading for observability. Add HarvestResult.snapshot_requests counter (Phase 1 search + Phase 1.5 per-cat probes) and a total_requests property. Update Celery task to write total_requests into cadastre_jobs.requests_count. Tests added for the new property + Phase 1 snapshot_requests=1 assertion.
This commit is contained in:
parent
d5b57ccbf1
commit
fa724f60ac
3 changed files with 30 additions and 3 deletions
|
|
@ -69,9 +69,15 @@ class HarvestResult:
|
||||||
oncs_upserted: int = 0
|
oncs_upserted: int = 0
|
||||||
enks_upserted: int = 0
|
enks_upserted: int = 0
|
||||||
zouit_upserted: int = 0
|
zouit_upserted: int = 0
|
||||||
grid_walk_requests: int = 0
|
snapshot_requests: int = 0 # search_by_quarter (Phase 1) + per-cat probes (Phase 1.5)
|
||||||
|
grid_walk_requests: int = 0 # wms_feature_info (Phase 2-3)
|
||||||
phase_state: dict[str, Any] | None = None
|
phase_state: dict[str, Any] | None = None
|
||||||
|
|
||||||
|
@property
|
||||||
|
def total_requests(self) -> int:
|
||||||
|
"""Все NSPD HTTP requests для этого квартала (snapshot + grid-walk)."""
|
||||||
|
return self.snapshot_requests + self.grid_walk_requests
|
||||||
|
|
||||||
|
|
||||||
# ── Основной оркестратор ─────────────────────────────────────────────────────
|
# ── Основной оркестратор ─────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
@ -127,6 +133,7 @@ async def harvest_quarter(
|
||||||
update_progress({"phase": "snapshot_started", "quarter": quarter})
|
update_progress({"phase": "snapshot_started", "quarter": quarter})
|
||||||
|
|
||||||
snapshot: QuarterSnapshot = await client.search_by_quarter(quarter)
|
snapshot: QuarterSnapshot = await client.search_by_quarter(quarter)
|
||||||
|
result.snapshot_requests += 1
|
||||||
|
|
||||||
snapshot_stats = upsert_features(db, snapshot.features, source="search")
|
snapshot_stats = upsert_features(db, snapshot.features, source="search")
|
||||||
result.parcels_upserted += snapshot_stats["parcels"]
|
result.parcels_upserted += snapshot_stats["parcels"]
|
||||||
|
|
@ -179,6 +186,7 @@ async def harvest_quarter(
|
||||||
|
|
||||||
try:
|
try:
|
||||||
cat_snapshot = await client.search_by_quarter(quarter, category_id=cat_id)
|
cat_snapshot = await client.search_by_quarter(quarter, category_id=cat_id)
|
||||||
|
result.snapshot_requests += 1
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"harvest_quarter: per-cat probe failed cat=%d quarter=%s: %s",
|
"harvest_quarter: per-cat probe failed cat=%d quarter=%s: %s",
|
||||||
|
|
|
||||||
|
|
@ -208,17 +208,21 @@ def bulk_harvest_quarter_task(self: Any, quarter: str, job_id: int) -> dict[str,
|
||||||
"oncs": result.oncs_upserted,
|
"oncs": result.oncs_upserted,
|
||||||
"enks": result.enks_upserted,
|
"enks": result.enks_upserted,
|
||||||
"zouit": result.zouit_upserted,
|
"zouit": result.zouit_upserted,
|
||||||
|
"snapshot_requests": result.snapshot_requests,
|
||||||
"grid_walk_requests": result.grid_walk_requests,
|
"grid_walk_requests": result.grid_walk_requests,
|
||||||
|
"total_requests": result.total_requests,
|
||||||
}
|
}
|
||||||
|
|
||||||
result_dict = asyncio.run(_run())
|
result_dict = asyncio.run(_run())
|
||||||
|
|
||||||
# Успех: increment targets_done
|
# Успех: increment targets_done. requests_count учитывает ВСЕ NSPD HTTP calls
|
||||||
|
# (Phase 1 search + Phase 1.5 per-cat probes + Phase 2-3 grid-walk), не только
|
||||||
|
# grid_walk_requests — иначе snapshot-only quarters давали бы 0 запросов в metrics.
|
||||||
_update_job(
|
_update_job(
|
||||||
db,
|
db,
|
||||||
job_id,
|
job_id,
|
||||||
targets_done_inc=1,
|
targets_done_inc=1,
|
||||||
requests_count_inc=result_dict.get("grid_walk_requests", 0),
|
requests_count_inc=result_dict.get("total_requests", 0),
|
||||||
heartbeat=True,
|
heartbeat=True,
|
||||||
)
|
)
|
||||||
# Проверить не завершён ли job (все targets done)
|
# Проверить не завершён ли job (все targets done)
|
||||||
|
|
|
||||||
|
|
@ -429,11 +429,26 @@ async def test_harvest_quarter_calls_upsert_features() -> None:
|
||||||
client.search_by_quarter.assert_awaited_once_with("66:41:0303161")
|
client.search_by_quarter.assert_awaited_once_with("66:41:0303161")
|
||||||
mock_upsert.assert_called()
|
mock_upsert.assert_called()
|
||||||
assert result.parcels_upserted == 1
|
assert result.parcels_upserted == 1
|
||||||
|
# Phase 1 = 1 NSPD search call → snapshot_requests=1
|
||||||
|
assert result.snapshot_requests == 1
|
||||||
|
assert result.grid_walk_requests == 0
|
||||||
|
assert result.total_requests == 1
|
||||||
|
|
||||||
# Проверить что последний progress = done
|
# Проверить что последний progress = done
|
||||||
assert progress_states[-1]["phase"] == "done"
|
assert progress_states[-1]["phase"] == "done"
|
||||||
|
|
||||||
|
|
||||||
|
def test_harvest_result_total_requests_property() -> None:
|
||||||
|
"""HarvestResult.total_requests = snapshot_requests + grid_walk_requests."""
|
||||||
|
from app.services.cadastre.bulk_harvest import HarvestResult
|
||||||
|
|
||||||
|
r = HarvestResult(quarter="66:41:0303161", snapshot_requests=3, grid_walk_requests=225)
|
||||||
|
assert r.total_requests == 228
|
||||||
|
|
||||||
|
empty = HarvestResult(quarter="66:41:0303161")
|
||||||
|
assert empty.total_requests == 0
|
||||||
|
|
||||||
|
|
||||||
# ── Dedup тест для grid-walk ─────────────────────────────────────────────────
|
# ── Dedup тест для grid-walk ─────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue