Light1YT
f87b6ada4c
Root: city-sweep counters (yandex/avito/cian/domclick/newbuilding) wrote lots_fetched/lots_inserted only into scrape_runs.counters jsonb; the dedicated columns total_seen/new_count (schema 015) were never SET, so admin/observability reported total_seen=0 despite real rows saved (audit #1871/#1926). Fix (Option A): single-place extraction in scrape_runs.py via _column_counts() maps counters lots_fetched->total_seen and lots_inserted->new_count (with explicit total_seen/new_count keys taking priority). update_heartbeat, mark_done, mark_failed and mark_banned now SET both columns with COALESCE(CAST(:x AS int), col) so a missing key preserves the existing value. Covers all sweep sources at once since every pipeline routes counters through these finalizers.
327 lines
11 KiB
Python
327 lines
11 KiB
Python
"""Offline smoke tests for city sweep utilities + endpoints.
|
||
|
||
Тесты НЕ требуют реального DB — мокируем SessionLocal и scrape_runs модуль.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||
|
||
from datetime import UTC
|
||
from unittest.mock import AsyncMock, MagicMock, patch
|
||
|
||
import pytest
|
||
|
||
# ── EKB_ANCHORS ────────────────────────────────────────────────────────────
|
||
|
||
|
||
def test_ekb_anchors_count() -> None:
|
||
from app.services.scrape_pipeline import EKB_ANCHORS
|
||
|
||
assert len(EKB_ANCHORS) == 5
|
||
for lat, lon, name in EKB_ANCHORS:
|
||
assert 56.5 < lat < 57.0, f"lat={lat} out of EKB range for anchor {name}"
|
||
assert 60.3 < lon < 60.8, f"lon={lon} out of EKB range for anchor {name}"
|
||
assert isinstance(name, str) and name
|
||
|
||
|
||
# ── CitySweepCounters ───────────────────────────────────────────────────────
|
||
|
||
|
||
def test_city_sweep_counters_defaults() -> None:
|
||
from app.services.scrape_pipeline import CitySweepCounters
|
||
|
||
c = CitySweepCounters()
|
||
assert c.anchors_total == 0
|
||
assert c.lots_fetched == 0
|
||
assert c.errors_count == 0
|
||
|
||
|
||
def test_city_sweep_counters_to_dict() -> None:
|
||
from app.services.scrape_pipeline import CitySweepCounters
|
||
|
||
c = CitySweepCounters(anchors_total=5, anchors_done=2, lots_fetched=150)
|
||
d = c.to_dict()
|
||
assert d["anchors_total"] == 5
|
||
assert d["anchors_done"] == 2
|
||
assert d["lots_fetched"] == 150
|
||
assert "errors_count" in d
|
||
assert "houses_enriched" in d
|
||
assert "detail_enriched" in d
|
||
|
||
|
||
def test_city_sweep_counters_to_dict_all_keys() -> None:
|
||
from dataclasses import fields
|
||
|
||
from app.services.scrape_pipeline import CitySweepCounters
|
||
|
||
c = CitySweepCounters()
|
||
d = c.to_dict()
|
||
expected_keys = {f.name for f in fields(c)}
|
||
assert set(d.keys()) == expected_keys
|
||
|
||
|
||
# ── scrape_runs utilities (unit, no DB) ────────────────────────────────────
|
||
|
||
|
||
def test_scrape_runs_create_run_returns_id() -> None:
|
||
from app.services.scrape_runs import create_run
|
||
|
||
mock_db = MagicMock()
|
||
mock_row = MagicMock()
|
||
mock_row.id = 42
|
||
mock_db.execute.return_value.fetchone.return_value = mock_row
|
||
|
||
run_id = create_run(mock_db, source="avito_city_sweep", params={"pages_per_anchor": 3})
|
||
|
||
assert run_id == 42
|
||
mock_db.commit.assert_called()
|
||
|
||
|
||
def test_scrape_runs_is_cancelled_true() -> None:
|
||
from app.services.scrape_runs import is_cancelled
|
||
|
||
mock_db = MagicMock()
|
||
mock_row = MagicMock()
|
||
mock_row.status = "cancelled"
|
||
mock_db.execute.return_value.fetchone.return_value = mock_row
|
||
|
||
assert is_cancelled(mock_db, 99) is True
|
||
|
||
|
||
def test_scrape_runs_is_cancelled_false_when_running() -> None:
|
||
from app.services.scrape_runs import is_cancelled
|
||
|
||
mock_db = MagicMock()
|
||
mock_row = MagicMock()
|
||
mock_row.status = "running"
|
||
mock_db.execute.return_value.fetchone.return_value = mock_row
|
||
|
||
assert is_cancelled(mock_db, 99) is False
|
||
|
||
|
||
def test_scrape_runs_is_cancelled_false_when_not_found() -> None:
|
||
from app.services.scrape_runs import is_cancelled
|
||
|
||
mock_db = MagicMock()
|
||
mock_db.execute.return_value.fetchone.return_value = None
|
||
|
||
assert is_cancelled(mock_db, 999) is False
|
||
|
||
|
||
def test_scrape_runs_mark_cancelled_returns_bool() -> None:
|
||
from app.services.scrape_runs import mark_cancelled
|
||
|
||
mock_db = MagicMock()
|
||
mock_db.execute.return_value.fetchone.return_value = MagicMock() # row found
|
||
|
||
result = mark_cancelled(mock_db, 10)
|
||
assert result is True
|
||
mock_db.commit.assert_called()
|
||
|
||
|
||
def test_scrape_runs_mark_cancelled_returns_false_when_not_running() -> None:
|
||
from app.services.scrape_runs import mark_cancelled
|
||
|
||
mock_db = MagicMock()
|
||
mock_db.execute.return_value.fetchone.return_value = None # no row (not running)
|
||
|
||
result = mark_cancelled(mock_db, 10)
|
||
assert result is False
|
||
|
||
|
||
# ── total_seen / new_count column population (audit #1926) ──────────────────
|
||
|
||
|
||
def _captured_params(mock_db: MagicMock) -> dict:
|
||
"""Извлечь dict bind-параметров из последнего db.execute(text(...), params)."""
|
||
assert mock_db.execute.call_args is not None, "db.execute was not called"
|
||
return mock_db.execute.call_args.args[1]
|
||
|
||
|
||
def test_column_counts_maps_lots_fetched_inserted() -> None:
|
||
"""_column_counts: lots_fetched→total_seen, lots_inserted→new_count."""
|
||
from app.services.scrape_runs import _column_counts
|
||
|
||
total_seen, new_count = _column_counts(
|
||
{"lots_fetched": 150, "lots_inserted": 12, "anchors_done": 5}
|
||
)
|
||
assert total_seen == 150
|
||
assert new_count == 12
|
||
|
||
|
||
def test_column_counts_prefers_explicit_total_seen_new_count() -> None:
|
||
"""Если в counters уже есть total_seen/new_count — они приоритетнее lots_*."""
|
||
from app.services.scrape_runs import _column_counts
|
||
|
||
total_seen, new_count = _column_counts(
|
||
{"total_seen": 99, "new_count": 7, "lots_fetched": 1, "lots_inserted": 1}
|
||
)
|
||
assert total_seen == 99
|
||
assert new_count == 7
|
||
|
||
|
||
def test_column_counts_returns_none_when_absent() -> None:
|
||
"""Нет ни одного из ключей → None (COALESCE сохранит старое значение колонки)."""
|
||
from app.services.scrape_runs import _column_counts
|
||
|
||
total_seen, new_count = _column_counts({"anchors_done": 3})
|
||
assert total_seen is None
|
||
assert new_count is None
|
||
|
||
|
||
def test_mark_done_persists_total_seen_new_count_columns() -> None:
|
||
"""mark_done пишет total_seen/new_count из lots_fetched/lots_inserted (audit #1926)."""
|
||
from app.services.scrape_runs import mark_done
|
||
|
||
mock_db = MagicMock()
|
||
mock_db.execute.return_value.first.return_value = MagicMock(id=1) # row updated
|
||
|
||
mark_done(mock_db, 1, {"lots_fetched": 200, "lots_inserted": 18, "anchors_done": 5})
|
||
|
||
params = _captured_params(mock_db)
|
||
assert params["total_seen"] == 200
|
||
assert params["new_count"] == 18
|
||
# SQL must SET the dedicated columns, not only the jsonb blob
|
||
sql = str(mock_db.execute.call_args.args[0])
|
||
assert "total_seen" in sql
|
||
assert "new_count" in sql
|
||
mock_db.commit.assert_called()
|
||
|
||
|
||
def test_update_heartbeat_persists_total_seen_new_count_columns() -> None:
|
||
"""update_heartbeat также обновляет колонки (live progress, не только финал)."""
|
||
from app.services.scrape_runs import update_heartbeat
|
||
|
||
mock_db = MagicMock()
|
||
|
||
update_heartbeat(mock_db, 1, {"lots_fetched": 75, "lots_inserted": 4})
|
||
|
||
params = _captured_params(mock_db)
|
||
assert params["total_seen"] == 75
|
||
assert params["new_count"] == 4
|
||
mock_db.commit.assert_called()
|
||
|
||
|
||
def test_mark_done_yandex_counters_populate_columns() -> None:
|
||
"""End-to-end shape: YandexCitySweepCounters.to_dict() → mark_done → columns set."""
|
||
from app.services.scrape_pipeline import YandexCitySweepCounters
|
||
from app.services.scrape_runs import mark_done
|
||
|
||
counters = YandexCitySweepCounters(
|
||
anchors_total=1, anchors_done=1, lots_fetched=540, lots_inserted=42
|
||
)
|
||
mock_db = MagicMock()
|
||
mock_db.execute.return_value.first.return_value = MagicMock(id=1)
|
||
|
||
mark_done(mock_db, 1, counters.to_dict())
|
||
|
||
params = _captured_params(mock_db)
|
||
assert params["total_seen"] == 540
|
||
assert params["new_count"] == 42
|
||
|
||
|
||
# ── FastAPI endpoints (offline) ─────────────────────────────────────────────
|
||
|
||
|
||
@pytest.fixture
|
||
def app_with_admin():
|
||
from fastapi import FastAPI
|
||
from fastapi.testclient import TestClient
|
||
|
||
# Lazy import to avoid side effects before env is set
|
||
from app.api.v1 import admin as admin_module
|
||
from app.core.db import get_db
|
||
|
||
app = FastAPI()
|
||
app.include_router(admin_module.router, prefix="/api/v1/admin")
|
||
|
||
def fake_db():
|
||
yield MagicMock()
|
||
|
||
app.dependency_overrides[get_db] = fake_db
|
||
return TestClient(app)
|
||
|
||
|
||
def test_start_endpoint_validates_pages_per_anchor(app_with_admin) -> None:
|
||
"""pages_per_anchor=99 превышает max=10 → 422."""
|
||
r = app_with_admin.post(
|
||
"/api/v1/admin/scrape/avito-city-sweep",
|
||
json={"pages_per_anchor": 99},
|
||
)
|
||
assert r.status_code == 422
|
||
|
||
|
||
def test_start_endpoint_validates_request_delay_too_low(app_with_admin) -> None:
|
||
"""request_delay_sec=1.0 меньше min=3.0 → 422."""
|
||
r = app_with_admin.post(
|
||
"/api/v1/admin/scrape/avito-city-sweep",
|
||
json={"request_delay_sec": 1.0},
|
||
)
|
||
assert r.status_code == 422
|
||
|
||
|
||
def test_start_endpoint_ok(app_with_admin) -> None:
|
||
"""Valid request → 200, run_id в ответе."""
|
||
with (
|
||
patch("app.api.v1.admin.has_running_run", return_value=False),
|
||
patch("app.services.scrape_runs.create_run", return_value=7),
|
||
patch("app.services.scrape_pipeline.run_avito_city_sweep", new_callable=AsyncMock),
|
||
):
|
||
r = app_with_admin.post(
|
||
"/api/v1/admin/scrape/avito-city-sweep",
|
||
json={"pages_per_anchor": 2, "detail_top_n": 10},
|
||
)
|
||
assert r.status_code == 200
|
||
body = r.json()
|
||
assert body["run_id"] == 7
|
||
assert body["status"] == "running"
|
||
assert body["pages_per_anchor"] == 2
|
||
|
||
|
||
def test_cancel_endpoint_exists(app_with_admin) -> None:
|
||
"""Cancel endpoint returns run_id + cancelled flag."""
|
||
with patch("app.services.scrape_runs.mark_cancelled", return_value=True):
|
||
r = app_with_admin.post("/api/v1/admin/scrape/avito-city-sweep/123/cancel")
|
||
assert r.status_code == 200
|
||
body = r.json()
|
||
assert body["run_id"] == 123
|
||
assert body["cancelled"] is True
|
||
assert body["ok"] is True
|
||
|
||
|
||
def test_cancel_endpoint_not_running(app_with_admin) -> None:
|
||
"""Cancel на уже завершённый run → cancelled=False."""
|
||
with patch("app.services.scrape_runs.mark_cancelled", return_value=False):
|
||
r = app_with_admin.post("/api/v1/admin/scrape/avito-city-sweep/456/cancel")
|
||
assert r.status_code == 200
|
||
assert r.json()["cancelled"] is False
|
||
|
||
|
||
def test_list_runs_endpoint(app_with_admin) -> None:
|
||
"""List runs endpoint возвращает список."""
|
||
from datetime import datetime
|
||
|
||
fake_rows = [
|
||
{
|
||
"run_id": 1,
|
||
"source": "avito_city_sweep",
|
||
"status": "done",
|
||
"params": {"pages_per_anchor": 3},
|
||
"counters": {"lots_fetched": 300},
|
||
"error": None,
|
||
"started_at": datetime(2025, 1, 1, tzinfo=UTC),
|
||
"finished_at": datetime(2025, 1, 1, 1, tzinfo=UTC),
|
||
"heartbeat_at": datetime(2025, 1, 1, 1, tzinfo=UTC),
|
||
}
|
||
]
|
||
with patch("app.services.scrape_runs.list_recent", return_value=fake_rows):
|
||
r = app_with_admin.get("/api/v1/admin/scrape/avito-city-sweep/runs")
|
||
assert r.status_code == 200
|
||
body = r.json()
|
||
assert len(body) == 1
|
||
assert body[0]["run_id"] == 1
|
||
assert body[0]["status"] == "done"
|
||
assert "2025-01-01" in body[0]["started_at"]
|