diff --git a/tradein-mvp/backend/app/api/v1/admin.py b/tradein-mvp/backend/app/api/v1/admin.py index 0570cbe7..882c2d0d 100644 --- a/tradein-mvp/backend/app/api/v1/admin.py +++ b/tradein-mvp/backend/app/api/v1/admin.py @@ -1962,7 +1962,9 @@ def _parse_proxy_host_port(proxy_url: str | None) -> tuple[str | None, int | Non def list_scrape_runs_unified( db: Annotated[Session, Depends(get_db)], source: Annotated[str | None, Query()] = None, - status: Annotated[str | None, Query()] = None, + status: Annotated[ + Literal["done", "running", "banned", "zombie", "failed", "cancelled"] | None, Query() + ] = None, limit: Annotated[int, Query(ge=1, le=200)] = 50, offset: Annotated[int, Query(ge=0)] = 0, ) -> UnifiedScrapeRunsResponse: diff --git a/tradein-mvp/backend/app/services/scrapers/avito_houses.py b/tradein-mvp/backend/app/services/scrapers/avito_houses.py index 96d5cc22..24d764c9 100644 --- a/tradein-mvp/backend/app/services/scrapers/avito_houses.py +++ b/tradein-mvp/backend/app/services/scrapers/avito_houses.py @@ -1252,7 +1252,24 @@ def save_house_catalog_enrichment( Возвращает: {'house_id': N, 'reviews': N, 'sellers': N, 'listings_linked': N, 'placement_history': N} """ - # 1. Upsert дом с правильным URL из enrichment + # 1. Upsert дом с правильным URL из enrichment. + # Гард #1820: id-less graceful-skip новостройки (ext_id=0, нет адреса/координат) — + # НЕ персистим canonical house, иначе все безымянные URL склеиваются в один bogus + # avito:0 через source_exact-tier и плодят мусорную house_sources("avito","0"). + if not e.house.ext_id and not (e.house.short_address or e.house.full_address): + logger.warning( + "save_house_catalog_enrichment: пропуск id-less новостройки без адреса" + " (ext_id=%s url=%s) — не создаём bogus avito:0 в house_sources", + e.house.ext_id, + e.house_url, + ) + return { + "house_id": 0, + "reviews": 0, + "sellers": 0, + "listings_linked": 0, + "placement_history": 0, + } house_id = _upsert_house_with_url(db, e.house, e.house_url) # 2. Сохранить отзывы diff --git a/tradein-mvp/backend/pyproject.toml b/tradein-mvp/backend/pyproject.toml index a4ca8dbe..68a08254 100644 --- a/tradein-mvp/backend/pyproject.toml +++ b/tradein-mvp/backend/pyproject.toml @@ -14,7 +14,7 @@ dependencies = [ "Pillow>=10.3.0", # photo sanitization re-encode (#6 audit fix) "weasyprint>=62.0", "jinja2>=3.1.0", - "httpx>=0.27.0", # для geocoder + scrapers + "httpx[socks]>=0.27.0", # для geocoder + scrapers; [socks] тянет socksio для socks5:// "tenacity>=9.0.0", # retry с exp backoff "selectolax>=0.3.0", # быстрый HTML парсинг для scrapers "segno>=1.6.0", # QR-код для PDF shareable URL diff --git a/tradein-mvp/backend/tests/test_avito_houses_dedup.py b/tradein-mvp/backend/tests/test_avito_houses_dedup.py index 3fbaa75e..fc91d89d 100644 --- a/tradein-mvp/backend/tests/test_avito_houses_dedup.py +++ b/tradein-mvp/backend/tests/test_avito_houses_dedup.py @@ -6,6 +6,7 @@ 2. Enrichment-поля (rating_score, developer_name) реально записываются в UPDATE. 3. _persist_house не вызывает db.commit(). 4. upsert_house и _upsert_house_with_url делегируют в _persist_house. + 5. #1820: два id-less URL (ext_id=0, без адреса) НЕ создают house_sources(avito,"0"). Использует тот же паттерн mock-DB что в test_matching.py и test_same_building_match.py. """ @@ -13,14 +14,16 @@ import json import os -os.environ.setdefault("DATABASE_URL", "postgresql://test:test@localhost/test_db") +os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db") from unittest.mock import MagicMock, patch from app.services.scrapers.avito_houses import ( + HouseCatalogEnrichment, HouseInfo, _persist_house, _upsert_house_with_url, + save_house_catalog_enrichment, upsert_house, ) @@ -269,3 +272,62 @@ def test_persist_house_coalesce_fields_in_bind(): assert bind["lat"] == h.lat assert bind["lon"] == h.lon assert bind["year_built"] == h.year_built + + +# --------------------------------------------------------------------------- +# Test 8: #1820 — ext_id=0 graceful-skip гард +# Два id-less URL (ext_id=0, без адреса) → match_or_create_house НЕ вызывается, +# house_sources("avito","0") НЕ создаётся, возвращается sentinel dict с house_id=0. +# --------------------------------------------------------------------------- + + +def test_save_house_catalog_enrichment_skips_zero_ext_id_no_address(): + """#1820: ext_id=0 + нет адреса → гард пропускает персистирование canonical house.""" + db = MagicMock() + + # Создаём два id-less graceful-skip enrichment (как _parse_development_or_skip возвращает) + enrichment_a = HouseCatalogEnrichment( + house_url="https://www.avito.ru/catalog/novostroyki/dom-A/", + house=HouseInfo(ext_id=0), + ) + enrichment_b = HouseCatalogEnrichment( + house_url="https://www.avito.ru/catalog/novostroyki/dom-B/", + house=HouseInfo(ext_id=0), + ) + + with patch( + "app.services.scrapers.avito_houses.match_or_create_house", + ) as mock_match: + result_a = save_house_catalog_enrichment(db, enrichment_a) + result_b = save_house_catalog_enrichment(db, enrichment_b) + + # match_or_create_house НЕ должен вызываться — гард сработал до него + mock_match.assert_not_called() + + # Возвращается sentinel dict с house_id=0, не плодятся дубли + assert result_a["house_id"] == 0, f"ожидали house_id=0, получили {result_a['house_id']}" + assert result_b["house_id"] == 0, f"ожидали house_id=0, получили {result_b['house_id']}" + + # db.commit() НЕ вызывается (нет реальной транзакции) + db.commit.assert_not_called() + + +def test_save_house_catalog_enrichment_with_address_persists_despite_zero_ext_id(): + """#1820: ext_id=0 НО есть адрес → НЕ пропускаем (developmentPage с адресом, без avitoId).""" + db = MagicMock() + db.execute.return_value = MagicMock() + + enrichment = HouseCatalogEnrichment( + house_url="https://www.avito.ru/catalog/novostroyki/dom-C/", + house=HouseInfo(ext_id=0, short_address="ул. Ленина, 1"), + ) + + with patch( + "app.services.scrapers.avito_houses.match_or_create_house", + return_value=(77, 0.9, "fingerprint"), + ) as mock_match: + result = save_house_catalog_enrichment(db, enrichment) + + # Адрес есть → персистим через fingerprint tier + mock_match.assert_called_once() + assert result["house_id"] == 77 diff --git a/tradein-mvp/backend/tests/test_scraper_admin_apis.py b/tradein-mvp/backend/tests/test_scraper_admin_apis.py index c4f3b7a9..6e776d8d 100644 --- a/tradein-mvp/backend/tests/test_scraper_admin_apis.py +++ b/tradein-mvp/backend/tests/test_scraper_admin_apis.py @@ -111,6 +111,21 @@ def test_unified_runs_negative_offset_422(client: TestClient) -> None: assert r.status_code == 422 +def test_unified_runs_invalid_status_422(client: TestClient) -> None: + """status с непредусмотренным значением → 422 (Literal-валидация).""" + r = client.get("/api/v1/admin/scrape/runs", params={"status": "unknown_status"}) + assert r.status_code == 422 + + +def test_unified_runs_valid_statuses_200(client: TestClient) -> None: + """Все допустимые значения status проходят валидацию (ожидаем 200, не 422).""" + valid_statuses = ["done", "running", "banned", "zombie", "failed", "cancelled"] + for s in valid_statuses: + with patch("app.services.scrape_runs.list_all", return_value=(0, [])): + r = client.get("/api/v1/admin/scrape/runs", params={"status": s}) + assert r.status_code == 200, f"status={s!r} должен возвращать 200, получили {r.status_code}" + + # ── API 2: GET /scraper/health ───────────────────────────────────────────────