lekss361
2914ff2ef2
Stage 4d: full city sweep ЕКБ (5 anchors × N pages + enrichment + cooperative cancel). - avito.py fetch_around(pages=N) — pagination backward compat default=1 - scrape_pipeline.run_avito_city_sweep — anchors loop + heartbeat + cancel check - scrape_runs.py utility (create/heartbeat/done/failed/cancelled/list_recent) - admin.py 3 endpoints: start (BackgroundTasks) / runs / cancel - migration 051 ADD COLUMN IF NOT EXISTS + DROP/ADD CHECK constraint (text+CHECK, не enum — safe в transaction) - deploy/avito-city-sweep.sh с random delay 0-3h (anti-pattern) - 16 offline tests Deep review approved: migration idempotent, cancel priority correct (mark_done guarded by status='running'), no SQL injection, psycopg v3 compliant.
234 lines
7.7 KiB
Python
234 lines
7.7 KiB
Python
"""Offline smoke tests for city sweep utilities + endpoints.
|
||
|
||
Тесты НЕ требуют реального DB — мокируем SessionLocal и scrape_runs модуль.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||
|
||
from unittest.mock import AsyncMock, MagicMock, patch
|
||
|
||
import pytest
|
||
|
||
|
||
# ── EKB_ANCHORS ────────────────────────────────────────────────────────────
|
||
|
||
|
||
def test_ekb_anchors_count() -> None:
|
||
from app.services.scrape_pipeline import EKB_ANCHORS
|
||
|
||
assert len(EKB_ANCHORS) == 5
|
||
for lat, lon, name in EKB_ANCHORS:
|
||
assert 56.5 < lat < 57.0, f"lat={lat} out of EKB range for anchor {name}"
|
||
assert 60.3 < lon < 60.8, f"lon={lon} out of EKB range for anchor {name}"
|
||
assert isinstance(name, str) and name
|
||
|
||
|
||
# ── CitySweepCounters ───────────────────────────────────────────────────────
|
||
|
||
|
||
def test_city_sweep_counters_defaults() -> None:
|
||
from app.services.scrape_pipeline import CitySweepCounters
|
||
|
||
c = CitySweepCounters()
|
||
assert c.anchors_total == 0
|
||
assert c.lots_fetched == 0
|
||
assert c.errors_count == 0
|
||
|
||
|
||
def test_city_sweep_counters_to_dict() -> None:
|
||
from app.services.scrape_pipeline import CitySweepCounters
|
||
|
||
c = CitySweepCounters(anchors_total=5, anchors_done=2, lots_fetched=150)
|
||
d = c.to_dict()
|
||
assert d["anchors_total"] == 5
|
||
assert d["anchors_done"] == 2
|
||
assert d["lots_fetched"] == 150
|
||
assert "errors_count" in d
|
||
assert "houses_enriched" in d
|
||
assert "detail_enriched" in d
|
||
|
||
|
||
def test_city_sweep_counters_to_dict_all_keys() -> None:
|
||
from app.services.scrape_pipeline import CitySweepCounters
|
||
from dataclasses import fields
|
||
|
||
c = CitySweepCounters()
|
||
d = c.to_dict()
|
||
expected_keys = {f.name for f in fields(c)}
|
||
assert set(d.keys()) == expected_keys
|
||
|
||
|
||
# ── scrape_runs utilities (unit, no DB) ────────────────────────────────────
|
||
|
||
|
||
def test_scrape_runs_create_run_returns_id() -> None:
|
||
from app.services.scrape_runs import create_run
|
||
|
||
mock_db = MagicMock()
|
||
mock_row = MagicMock()
|
||
mock_row.id = 42
|
||
mock_db.execute.return_value.fetchone.return_value = mock_row
|
||
|
||
run_id = create_run(mock_db, source="avito_city_sweep", params={"pages_per_anchor": 3})
|
||
|
||
assert run_id == 42
|
||
mock_db.commit.assert_called()
|
||
|
||
|
||
def test_scrape_runs_is_cancelled_true() -> None:
|
||
from app.services.scrape_runs import is_cancelled
|
||
|
||
mock_db = MagicMock()
|
||
mock_row = MagicMock()
|
||
mock_row.status = "cancelled"
|
||
mock_db.execute.return_value.fetchone.return_value = mock_row
|
||
|
||
assert is_cancelled(mock_db, 99) is True
|
||
|
||
|
||
def test_scrape_runs_is_cancelled_false_when_running() -> None:
|
||
from app.services.scrape_runs import is_cancelled
|
||
|
||
mock_db = MagicMock()
|
||
mock_row = MagicMock()
|
||
mock_row.status = "running"
|
||
mock_db.execute.return_value.fetchone.return_value = mock_row
|
||
|
||
assert is_cancelled(mock_db, 99) is False
|
||
|
||
|
||
def test_scrape_runs_is_cancelled_false_when_not_found() -> None:
|
||
from app.services.scrape_runs import is_cancelled
|
||
|
||
mock_db = MagicMock()
|
||
mock_db.execute.return_value.fetchone.return_value = None
|
||
|
||
assert is_cancelled(mock_db, 999) is False
|
||
|
||
|
||
def test_scrape_runs_mark_cancelled_returns_bool() -> None:
|
||
from app.services.scrape_runs import mark_cancelled
|
||
|
||
mock_db = MagicMock()
|
||
mock_db.execute.return_value.fetchone.return_value = MagicMock() # row found
|
||
|
||
result = mark_cancelled(mock_db, 10)
|
||
assert result is True
|
||
mock_db.commit.assert_called()
|
||
|
||
|
||
def test_scrape_runs_mark_cancelled_returns_false_when_not_running() -> None:
|
||
from app.services.scrape_runs import mark_cancelled
|
||
|
||
mock_db = MagicMock()
|
||
mock_db.execute.return_value.fetchone.return_value = None # no row (not running)
|
||
|
||
result = mark_cancelled(mock_db, 10)
|
||
assert result is False
|
||
|
||
|
||
# ── FastAPI endpoints (offline) ─────────────────────────────────────────────
|
||
|
||
|
||
@pytest.fixture
|
||
def app_with_admin():
|
||
from fastapi import FastAPI
|
||
from fastapi.testclient import TestClient
|
||
|
||
# Lazy import to avoid side effects before env is set
|
||
from app.api.v1 import admin as admin_module
|
||
from app.core.db import get_db
|
||
|
||
app = FastAPI()
|
||
app.include_router(admin_module.router, prefix="/api/v1/admin")
|
||
|
||
def fake_db():
|
||
yield MagicMock()
|
||
|
||
app.dependency_overrides[get_db] = fake_db
|
||
return TestClient(app)
|
||
|
||
|
||
def test_start_endpoint_validates_pages_per_anchor(app_with_admin) -> None:
|
||
"""pages_per_anchor=99 превышает max=10 → 422."""
|
||
r = app_with_admin.post(
|
||
"/api/v1/admin/scrape/avito-city-sweep",
|
||
json={"pages_per_anchor": 99},
|
||
)
|
||
assert r.status_code == 422
|
||
|
||
|
||
def test_start_endpoint_validates_request_delay_too_low(app_with_admin) -> None:
|
||
"""request_delay_sec=1.0 меньше min=3.0 → 422."""
|
||
r = app_with_admin.post(
|
||
"/api/v1/admin/scrape/avito-city-sweep",
|
||
json={"request_delay_sec": 1.0},
|
||
)
|
||
assert r.status_code == 422
|
||
|
||
|
||
def test_start_endpoint_ok(app_with_admin) -> None:
|
||
"""Valid request → 200, run_id в ответе."""
|
||
with (
|
||
patch("app.services.scrape_runs.create_run", return_value=7) as mock_create,
|
||
patch("app.services.scrape_pipeline.run_avito_city_sweep", new_callable=AsyncMock),
|
||
):
|
||
r = app_with_admin.post(
|
||
"/api/v1/admin/scrape/avito-city-sweep",
|
||
json={"pages_per_anchor": 2, "detail_top_n": 10},
|
||
)
|
||
assert r.status_code == 200
|
||
body = r.json()
|
||
assert body["run_id"] == 7
|
||
assert body["status"] == "running"
|
||
assert body["pages_per_anchor"] == 2
|
||
|
||
|
||
def test_cancel_endpoint_exists(app_with_admin) -> None:
|
||
"""Cancel endpoint returns run_id + cancelled flag."""
|
||
with patch("app.services.scrape_runs.mark_cancelled", return_value=True):
|
||
r = app_with_admin.post("/api/v1/admin/scrape/avito-city-sweep/123/cancel")
|
||
assert r.status_code == 200
|
||
body = r.json()
|
||
assert body["run_id"] == 123
|
||
assert body["cancelled"] is True
|
||
assert body["ok"] is True
|
||
|
||
|
||
def test_cancel_endpoint_not_running(app_with_admin) -> None:
|
||
"""Cancel на уже завершённый run → cancelled=False."""
|
||
with patch("app.services.scrape_runs.mark_cancelled", return_value=False):
|
||
r = app_with_admin.post("/api/v1/admin/scrape/avito-city-sweep/456/cancel")
|
||
assert r.status_code == 200
|
||
assert r.json()["cancelled"] is False
|
||
|
||
|
||
def test_list_runs_endpoint(app_with_admin) -> None:
|
||
"""List runs endpoint возвращает список."""
|
||
from datetime import datetime, timezone
|
||
|
||
fake_rows = [
|
||
{
|
||
"run_id": 1,
|
||
"source": "avito_city_sweep",
|
||
"status": "done",
|
||
"params": {"pages_per_anchor": 3},
|
||
"counters": {"lots_fetched": 300},
|
||
"error": None,
|
||
"started_at": datetime(2025, 1, 1, tzinfo=timezone.utc),
|
||
"finished_at": datetime(2025, 1, 1, 1, tzinfo=timezone.utc),
|
||
"heartbeat_at": datetime(2025, 1, 1, 1, tzinfo=timezone.utc),
|
||
}
|
||
]
|
||
with patch("app.services.scrape_runs.list_recent", return_value=fake_rows):
|
||
r = app_with_admin.get("/api/v1/admin/scrape/avito-city-sweep/runs")
|
||
assert r.status_code == 200
|
||
body = r.json()
|
||
assert len(body) == 1
|
||
assert body[0]["run_id"] == 1
|
||
assert body[0]["status"] == "done"
|
||
assert "2025-01-01" in body[0]["started_at"]
|