Compare commits
No commits in common. "29ea46193ebfb323c764136024ef71699e82ee86" and "70838c7dfed74f8d0df00bd7c7abb2a80e5a3bf9" have entirely different histories.
29ea46193e
...
70838c7dfe
4 changed files with 7 additions and 44 deletions
|
|
@ -22,7 +22,6 @@ from app.schemas.trade_in import ScheduleConfig, ScheduleConfigUpdate
|
||||||
from app.services import cian_session as cian_session_svc
|
from app.services import cian_session as cian_session_svc
|
||||||
from app.services import scrape_runs as runs_mod
|
from app.services import scrape_runs as runs_mod
|
||||||
from app.services.geocoder import geocode
|
from app.services.geocoder import geocode
|
||||||
from app.services.scheduler import has_running_run
|
|
||||||
from app.services.scrape_pipeline import (
|
from app.services.scrape_pipeline import (
|
||||||
run_avito_city_sweep,
|
run_avito_city_sweep,
|
||||||
run_cian_city_sweep,
|
run_cian_city_sweep,
|
||||||
|
|
@ -635,22 +634,7 @@ async def start_avito_city_sweep(
|
||||||
|
|
||||||
Coop cancel: POST /scrape/avito-city-sweep/{run_id}/cancel помечает run,
|
Coop cancel: POST /scrape/avito-city-sweep/{run_id}/cancel помечает run,
|
||||||
pipeline проверяет статус каждый anchor.
|
pipeline проверяет статус каждый anchor.
|
||||||
|
|
||||||
Single-run guard: второй sweep запустить нельзя, пока первый running —
|
|
||||||
параллельные раны делят один прокси-IP → удвоенный rate → Avito бан
|
|
||||||
(incident 2026-05-31: runs #26+#27 шли одновременно → оба banned).
|
|
||||||
Scheduler уже защищён (has_running_run + advisory lock); этот guard
|
|
||||||
закрывает ручной admin-триггер.
|
|
||||||
"""
|
"""
|
||||||
if has_running_run(db, "avito_city_sweep"):
|
|
||||||
raise HTTPException(
|
|
||||||
status_code=409,
|
|
||||||
detail=(
|
|
||||||
"avito city-sweep уже выполняется — одновременно допустим только один "
|
|
||||||
"(параллельные раны делят прокси-IP → Avito бан). Дождитесь завершения "
|
|
||||||
"или отмените текущий через POST /scrape/avito-city-sweep/{run_id}/cancel."
|
|
||||||
),
|
|
||||||
)
|
|
||||||
run_id = runs_mod.create_run(db, source="avito_city_sweep", params=payload.model_dump())
|
run_id = runs_mod.create_run(db, source="avito_city_sweep", params=payload.model_dump())
|
||||||
|
|
||||||
async def _sweep_task() -> None:
|
async def _sweep_task() -> None:
|
||||||
|
|
|
||||||
|
|
@ -104,21 +104,3 @@ def test_get_imv_benchmark_unavailable(app):
|
||||||
# endpoint вернёт 404 потому что fallback тоже не находит address → нужен estimate row
|
# endpoint вернёт 404 потому что fallback тоже не находит address → нужен estimate row
|
||||||
# Этот тест проверяет что эндпоинт mounted; 404 ОК
|
# Этот тест проверяет что эндпоинт mounted; 404 ОК
|
||||||
assert r.status_code in (200, 404)
|
assert r.status_code in (200, 404)
|
||||||
|
|
||||||
|
|
||||||
def test_avito_city_sweep_single_run_guard(app, monkeypatch):
|
|
||||||
"""#single-run: второй avito sweep при уже running → 409.
|
|
||||||
|
|
||||||
Параллельные sweep'ы делят один прокси-IP → удвоенный rate → Avito бан
|
|
||||||
(incident 2026-05-31: runs #26+#27 одновременно → оба banned).
|
|
||||||
"""
|
|
||||||
from app.api.v1 import admin as admin_module
|
|
||||||
|
|
||||||
monkeypatch.setattr(admin_module, "has_running_run", lambda db, source: True)
|
|
||||||
client = TestClient(app)
|
|
||||||
r = client.post(
|
|
||||||
"/api/v1/admin/scrape/avito-city-sweep",
|
|
||||||
json={"pages_per_anchor": 1, "request_delay_sec": 5.0},
|
|
||||||
)
|
|
||||||
assert r.status_code == 409
|
|
||||||
assert "уже выполняется" in r.json()["detail"]
|
|
||||||
|
|
|
||||||
|
|
@ -9,11 +9,11 @@ import os
|
||||||
|
|
||||||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||||||
|
|
||||||
from datetime import UTC
|
|
||||||
from unittest.mock import AsyncMock, MagicMock, patch
|
from unittest.mock import AsyncMock, MagicMock, patch
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
# ── EKB_ANCHORS ────────────────────────────────────────────────────────────
|
# ── EKB_ANCHORS ────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -53,9 +53,8 @@ def test_city_sweep_counters_to_dict() -> None:
|
||||||
|
|
||||||
|
|
||||||
def test_city_sweep_counters_to_dict_all_keys() -> None:
|
def test_city_sweep_counters_to_dict_all_keys() -> None:
|
||||||
from dataclasses import fields
|
|
||||||
|
|
||||||
from app.services.scrape_pipeline import CitySweepCounters
|
from app.services.scrape_pipeline import CitySweepCounters
|
||||||
|
from dataclasses import fields
|
||||||
|
|
||||||
c = CitySweepCounters()
|
c = CitySweepCounters()
|
||||||
d = c.to_dict()
|
d = c.to_dict()
|
||||||
|
|
@ -175,8 +174,7 @@ def test_start_endpoint_validates_request_delay_too_low(app_with_admin) -> None:
|
||||||
def test_start_endpoint_ok(app_with_admin) -> None:
|
def test_start_endpoint_ok(app_with_admin) -> None:
|
||||||
"""Valid request → 200, run_id в ответе."""
|
"""Valid request → 200, run_id в ответе."""
|
||||||
with (
|
with (
|
||||||
patch("app.api.v1.admin.has_running_run", return_value=False),
|
patch("app.services.scrape_runs.create_run", return_value=7) as mock_create,
|
||||||
patch("app.services.scrape_runs.create_run", return_value=7),
|
|
||||||
patch("app.services.scrape_pipeline.run_avito_city_sweep", new_callable=AsyncMock),
|
patch("app.services.scrape_pipeline.run_avito_city_sweep", new_callable=AsyncMock),
|
||||||
):
|
):
|
||||||
r = app_with_admin.post(
|
r = app_with_admin.post(
|
||||||
|
|
@ -211,7 +209,7 @@ def test_cancel_endpoint_not_running(app_with_admin) -> None:
|
||||||
|
|
||||||
def test_list_runs_endpoint(app_with_admin) -> None:
|
def test_list_runs_endpoint(app_with_admin) -> None:
|
||||||
"""List runs endpoint возвращает список."""
|
"""List runs endpoint возвращает список."""
|
||||||
from datetime import datetime
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
fake_rows = [
|
fake_rows = [
|
||||||
{
|
{
|
||||||
|
|
@ -221,9 +219,9 @@ def test_list_runs_endpoint(app_with_admin) -> None:
|
||||||
"params": {"pages_per_anchor": 3},
|
"params": {"pages_per_anchor": 3},
|
||||||
"counters": {"lots_fetched": 300},
|
"counters": {"lots_fetched": 300},
|
||||||
"error": None,
|
"error": None,
|
||||||
"started_at": datetime(2025, 1, 1, tzinfo=UTC),
|
"started_at": datetime(2025, 1, 1, tzinfo=timezone.utc),
|
||||||
"finished_at": datetime(2025, 1, 1, 1, tzinfo=UTC),
|
"finished_at": datetime(2025, 1, 1, 1, tzinfo=timezone.utc),
|
||||||
"heartbeat_at": datetime(2025, 1, 1, 1, tzinfo=UTC),
|
"heartbeat_at": datetime(2025, 1, 1, 1, tzinfo=timezone.utc),
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
with patch("app.services.scrape_runs.list_recent", return_value=fake_rows):
|
with patch("app.services.scrape_runs.list_recent", return_value=fake_rows):
|
||||||
|
|
|
||||||
|
|
@ -443,7 +443,6 @@ def test_city_sweep_start_request_enrich_imv_false() -> None:
|
||||||
def test_sweep_endpoint_passes_enrich_imv(app_with_admin) -> None:
|
def test_sweep_endpoint_passes_enrich_imv(app_with_admin) -> None:
|
||||||
"""POST /scrape/avito-city-sweep с enrich_imv=False → sweep вызывается без IMV."""
|
"""POST /scrape/avito-city-sweep с enrich_imv=False → sweep вызывается без IMV."""
|
||||||
with (
|
with (
|
||||||
patch("app.api.v1.admin.has_running_run", return_value=False),
|
|
||||||
patch("app.services.scrape_runs.create_run", return_value=42),
|
patch("app.services.scrape_runs.create_run", return_value=42),
|
||||||
patch(
|
patch(
|
||||||
"app.services.scrape_pipeline.run_avito_city_sweep",
|
"app.services.scrape_pipeline.run_avito_city_sweep",
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue