b6b7af6b42
All checks were successful
CI Trade-In / changes (pull_request) Successful in 9s
CI / changes (pull_request) Successful in 9s
CI Trade-In / frontend-checks (pull_request) Has been skipped
CI / backend-tests (pull_request) Has been skipped
CI / frontend-tests (pull_request) Has been skipped
CI / openapi-codegen-check (pull_request) Has been skipped
CI Trade-In / backend-tests (pull_request) Successful in 1m1s
463 lines
16 KiB
Python
463 lines
16 KiB
Python
"""Tests for yandex_newbuilding_sweep task (#974).
|
||
|
||
No real network, no real Postgres. BrowserFetcher, resolve_yandex_jk_slug and
|
||
fetch_jk are mocked. The DB session is an in-memory fake that models
|
||
market.yandex_jk_enrichment + houses.yandex_jk_slug persistence assertions.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
import sys
|
||
from unittest.mock import AsyncMock, MagicMock, patch
|
||
|
||
# DATABASE_URL required by config before any app import.
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||
|
||
# WeasyPrint stub — not installed in CI without GTK.
|
||
_wp_mock = MagicMock()
|
||
sys.modules.setdefault("weasyprint", _wp_mock)
|
||
|
||
import pytest # noqa: E402
|
||
|
||
from app.tasks.yandex_newbuilding_sweep import ( # noqa: E402
|
||
YandexNewbuildingSweepResult,
|
||
count_yandex_newbuilding_houses,
|
||
enrich_yandex_newbuilding_sweep,
|
||
)
|
||
|
||
# ── In-memory fake DB ─────────────────────────────────────────────────────────
|
||
|
||
|
||
class _FakeResult:
|
||
def __init__(self, *, scalar=None, rows=None, fetchone=None):
|
||
self._scalar = scalar
|
||
self._rows = rows or []
|
||
self._fetchone = fetchone
|
||
|
||
def scalar_one(self):
|
||
return self._scalar
|
||
|
||
def mappings(self):
|
||
return self
|
||
|
||
def all(self):
|
||
return self._rows
|
||
|
||
def fetchone(self):
|
||
return self._fetchone
|
||
|
||
|
||
class FakeDB:
|
||
"""Minimal SQLAlchemy-Session fake for yandex_newbuilding_sweep.
|
||
|
||
Tracks:
|
||
- enrichment: set of ext_id strings (inserted via UPSERT)
|
||
- slug_updates: dict house_id → slug (persisted via UPDATE houses)
|
||
- savepoints: int (# of begin_nested calls)
|
||
"""
|
||
|
||
def __init__(self, pending_rows: list[dict]):
|
||
self._pending_rows = pending_rows
|
||
# market.yandex_jk_enrichment keyed by ext_id
|
||
self.enrichment: dict[str, dict] = {}
|
||
# houses.yandex_jk_slug written during resolve
|
||
self.slug_updates: dict[int, str] = {}
|
||
# houses.house_class written after successful enrichment (#2003)
|
||
self.house_class_updates: dict[int, str] = {}
|
||
self.commits = 0
|
||
self.rollbacks = 0
|
||
self._sp_stack: list[_FakeSavepoint] = []
|
||
|
||
def execute(self, statement, params=None):
|
||
sql = str(statement)
|
||
params = params or {}
|
||
|
||
# COUNT queries
|
||
if "COUNT(DISTINCT h.id)" in sql and "yandex_realty_nb" in sql:
|
||
if "NOT EXISTS" in sql:
|
||
# _COUNT_PENDING
|
||
pending = sum(
|
||
1 for r in self._pending_rows if r.get("ext_id") not in self.enrichment
|
||
)
|
||
return _FakeResult(scalar=pending)
|
||
if "yandex_jk_slug IS NOT NULL" in sql:
|
||
# _COUNT_FETCHABLE
|
||
return _FakeResult(scalar=len(self._pending_rows))
|
||
# _COUNT_TOTAL
|
||
return _FakeResult(scalar=len(self._pending_rows))
|
||
|
||
# SELECT pending houses
|
||
if "SELECT DISTINCT ON (h.id)" in sql and "yandex_realty_nb" in sql:
|
||
lim = params.get("lim", len(self._pending_rows))
|
||
force = params.get("force", False)
|
||
out = []
|
||
for r in self._pending_rows[:lim]:
|
||
ext_id = r.get("ext_id")
|
||
if not force and ext_id in self.enrichment:
|
||
continue
|
||
# Reflect persisted slug
|
||
slug = self.slug_updates.get(r["house_id"]) or r.get("yandex_jk_slug")
|
||
out.append({**r, "yandex_jk_slug": slug})
|
||
return _FakeResult(rows=out)
|
||
|
||
# UPDATE houses SET yandex_jk_slug
|
||
if "UPDATE houses" in sql and "yandex_jk_slug" in sql:
|
||
self.slug_updates[params["hid"]] = params["slug"]
|
||
return _FakeResult()
|
||
|
||
# UPDATE houses SET house_class (#2003)
|
||
if "UPDATE houses" in sql and "house_class" in sql:
|
||
self.house_class_updates[params["hid"]] = params["house_class"]
|
||
return _FakeResult()
|
||
|
||
# Idempotency check: SELECT 1 FROM market.yandex_jk_enrichment
|
||
if "SELECT 1 FROM market.yandex_jk_enrichment" in sql:
|
||
ext_id = params.get("ext_id")
|
||
if ext_id in self.enrichment:
|
||
return _FakeResult(fetchone=object())
|
||
return _FakeResult(fetchone=None)
|
||
|
||
# UPSERT market.yandex_jk_enrichment
|
||
if "INSERT INTO market.yandex_jk_enrichment" in sql:
|
||
self.enrichment[params["ext_id"]] = dict(params)
|
||
return _FakeResult()
|
||
|
||
return _FakeResult()
|
||
|
||
def begin_nested(self):
|
||
sp = _FakeSavepoint(self)
|
||
self._sp_stack.append(sp)
|
||
return sp
|
||
|
||
def commit(self):
|
||
self.commits += 1
|
||
|
||
def rollback(self):
|
||
self.rollbacks += 1
|
||
|
||
|
||
class _FakeSavepoint:
|
||
def __init__(self, db: FakeDB):
|
||
self._db = db
|
||
|
||
def commit(self):
|
||
pass
|
||
|
||
def rollback(self):
|
||
pass
|
||
|
||
|
||
# ── Tests: count_yandex_newbuilding_houses ───────────────────────────────────
|
||
|
||
|
||
def test_count_returns_sizing_dict():
|
||
db = FakeDB([{"house_id": 1, "ext_id": "100", "yandex_jk_slug": None}])
|
||
sizing = count_yandex_newbuilding_houses(db)
|
||
assert "total" in sizing
|
||
assert "fetchable" in sizing
|
||
assert "pending" in sizing
|
||
assert sizing["total"] >= 0
|
||
|
||
|
||
# ── Tests: enrich_yandex_newbuilding_sweep ───────────────────────────────────
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_dry_run_returns_counts_without_fetch():
|
||
"""dry_run: популяция заполнена, fetch не вызван."""
|
||
db = FakeDB(
|
||
[
|
||
{"house_id": 1, "ext_id": "111", "yandex_jk_slug": None, "yandex_jk_id": None},
|
||
]
|
||
)
|
||
result = await enrich_yandex_newbuilding_sweep(db, limit=5, dry_run=True)
|
||
assert isinstance(result, YandexNewbuildingSweepResult)
|
||
assert result.processed == 0 # dry_run = no processing
|
||
assert result.rows_inserted == 0
|
||
assert result.total >= 0
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_sweep_with_existing_slug_upserts_enrichment():
|
||
"""House с уже известным slug: fetch_jk вызван, UPSERT market + счётчик."""
|
||
db = FakeDB(
|
||
[
|
||
{
|
||
"house_id": 10,
|
||
"ext_id": "999",
|
||
"yandex_jk_slug": "tatlin",
|
||
"yandex_jk_id": "999",
|
||
}
|
||
]
|
||
)
|
||
|
||
from scraper_kit.providers.yandex.newbuilding import YandexNewbuildingInfo
|
||
|
||
fake_info = YandexNewbuildingInfo(
|
||
ext_id="999",
|
||
ext_slug="tatlin",
|
||
source_url="https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-999/",
|
||
name="ЖК Татлин",
|
||
developer_name="ПРИНЦИПЗ",
|
||
lat=56.855,
|
||
lon=60.576,
|
||
rating=4.3,
|
||
ratings_count=100,
|
||
text_reviews_count=25,
|
||
)
|
||
|
||
from app.services.scraper_adapters import RealScraperConfig
|
||
|
||
with patch("scraper_kit.providers.yandex.newbuilding.YandexNewbuildingScraper") as mock_scraper:
|
||
scraper_instance = MagicMock()
|
||
scraper_instance.fetch_jk = AsyncMock(return_value=fake_info)
|
||
mock_scraper.return_value = scraper_instance
|
||
|
||
result = await enrich_yandex_newbuilding_sweep(db, limit=5, request_delay_sec=0)
|
||
|
||
assert result.processed == 1
|
||
assert result.succeeded == 1
|
||
assert result.rows_inserted == 1
|
||
assert result.failed_fetch == 0
|
||
assert result.failed_resolve == 0
|
||
assert "999" in db.enrichment
|
||
assert db.enrichment["999"]["name"] == "ЖК Татлин"
|
||
# house_class не распарсен в этой фикстуре (None) — UPDATE houses.house_class не должен
|
||
# выполниться вовсе (Bug 1 guard: лишний UPDATE с NULL пропускается).
|
||
assert 10 not in db.house_class_updates
|
||
# #2397 Part D5 regression guard: kit YandexNewbuildingScraper.fetch_jk builds
|
||
# BrowserFetcher(endpoint=self._config.browser_http_endpoint) — without
|
||
# config=RealScraperConfig() threaded at the call site, BrowserFetcher.__aenter__
|
||
# raises AssertionError (missing endpoint). call_count alone wouldn't catch a
|
||
# future drop of that kwarg (mirrors #2306/#2310's isinstance-call_kwargs pattern).
|
||
assert isinstance(mock_scraper.call_args.kwargs.get("config"), RealScraperConfig)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_sweep_persists_house_class_when_parsed():
|
||
"""#2003: info.house_class распарсен → UPDATE houses.house_class выполняется.
|
||
|
||
Ранее (Bug 1) house_class парсился в YandexNewbuildingInfo, но нигде не персистился —
|
||
только market.yandex_jk_enrichment (без колонки house_class). Регрессия на фикс.
|
||
"""
|
||
db = FakeDB(
|
||
[
|
||
{
|
||
"house_id": 11,
|
||
"ext_id": "1000",
|
||
"yandex_jk_slug": "tatlin",
|
||
"yandex_jk_id": "1000",
|
||
}
|
||
]
|
||
)
|
||
|
||
from scraper_kit.providers.yandex.newbuilding import YandexNewbuildingInfo
|
||
|
||
fake_info = YandexNewbuildingInfo(
|
||
ext_id="1000",
|
||
ext_slug="tatlin",
|
||
source_url="https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-1000/",
|
||
name="ЖК Татлин",
|
||
house_class="comfort_plus",
|
||
)
|
||
|
||
with patch("scraper_kit.providers.yandex.newbuilding.YandexNewbuildingScraper") as mock_scraper:
|
||
scraper_instance = MagicMock()
|
||
scraper_instance.fetch_jk = AsyncMock(return_value=fake_info)
|
||
mock_scraper.return_value = scraper_instance
|
||
|
||
result = await enrich_yandex_newbuilding_sweep(db, limit=5, request_delay_sec=0)
|
||
|
||
assert result.succeeded == 1
|
||
assert db.house_class_updates.get(11) == "comfort_plus"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_sweep_resolves_slug_when_missing():
|
||
"""House без slug: resolve_yandex_jk_slug вызван, slug сохранён, fetch_jk вызван."""
|
||
db = FakeDB(
|
||
[
|
||
{
|
||
"house_id": 20,
|
||
"ext_id": "888",
|
||
"yandex_jk_slug": None,
|
||
"yandex_jk_id": None,
|
||
}
|
||
]
|
||
)
|
||
|
||
from scraper_kit.providers.yandex.newbuilding import YandexNewbuildingInfo
|
||
|
||
from app.services.scraper_adapters import RealScraperConfig
|
||
|
||
fake_info = YandexNewbuildingInfo(
|
||
ext_id="888",
|
||
ext_slug="testjk",
|
||
source_url="https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/testjk-888/",
|
||
name="ЖК Тест",
|
||
lat=56.8,
|
||
lon=60.5,
|
||
)
|
||
|
||
with (
|
||
patch(
|
||
"scraper_kit.providers.yandex.newbuilding.resolve_yandex_jk_slug",
|
||
new_callable=AsyncMock,
|
||
return_value="testjk",
|
||
) as mock_resolve,
|
||
patch("scraper_kit.providers.yandex.newbuilding.YandexNewbuildingScraper") as mock_scraper,
|
||
):
|
||
scraper_instance = MagicMock()
|
||
scraper_instance.fetch_jk = AsyncMock(return_value=fake_info)
|
||
mock_scraper.return_value = scraper_instance
|
||
|
||
result = await enrich_yandex_newbuilding_sweep(db, limit=5, request_delay_sec=0)
|
||
|
||
# resolve_yandex_jk_slug теперь принимает config= (#2322 fix) — сравниваем
|
||
# позиционный arg + city= отдельно от config, т.к. RealScraperConfig
|
||
# instance-equality не поддерживает (нет __eq__); проверяем isinstance,
|
||
# тот же regression-guard паттерн что и для YandexNewbuildingScraper выше.
|
||
resolve_args, resolve_kwargs = mock_resolve.call_args
|
||
assert resolve_args == ("888",)
|
||
assert resolve_kwargs.get("city") == "ekaterinburg"
|
||
assert isinstance(resolve_kwargs.get("config"), RealScraperConfig)
|
||
assert result.resolved_slug == 1
|
||
assert result.succeeded == 1
|
||
assert result.rows_inserted == 1
|
||
# Slug persisted to houses
|
||
assert db.slug_updates.get(20) == "testjk"
|
||
assert "888" in db.enrichment
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_sweep_failed_resolve_counts():
|
||
"""resolve_yandex_jk_slug returns None → failed_resolve += 1, no upsert."""
|
||
db = FakeDB(
|
||
[
|
||
{
|
||
"house_id": 30,
|
||
"ext_id": "777",
|
||
"yandex_jk_slug": None,
|
||
"yandex_jk_id": None,
|
||
}
|
||
]
|
||
)
|
||
|
||
with patch(
|
||
"scraper_kit.providers.yandex.newbuilding.resolve_yandex_jk_slug",
|
||
new_callable=AsyncMock,
|
||
return_value=None,
|
||
):
|
||
result = await enrich_yandex_newbuilding_sweep(db, limit=5, request_delay_sec=0)
|
||
|
||
assert result.failed_resolve == 1
|
||
assert result.succeeded == 0
|
||
assert result.rows_inserted == 0
|
||
assert "777" not in db.enrichment
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_sweep_failed_fetch_counts():
|
||
"""fetch_jk returns None → failed_fetch += 1, no upsert."""
|
||
db = FakeDB(
|
||
[
|
||
{
|
||
"house_id": 40,
|
||
"ext_id": "666",
|
||
"yandex_jk_slug": "existingslug",
|
||
"yandex_jk_id": "666",
|
||
}
|
||
]
|
||
)
|
||
|
||
with patch("scraper_kit.providers.yandex.newbuilding.YandexNewbuildingScraper") as mock_scraper:
|
||
scraper_instance = MagicMock()
|
||
scraper_instance.fetch_jk = AsyncMock(return_value=None)
|
||
mock_scraper.return_value = scraper_instance
|
||
|
||
result = await enrich_yandex_newbuilding_sweep(db, limit=5, request_delay_sec=0)
|
||
|
||
assert result.failed_fetch == 1
|
||
assert result.succeeded == 0
|
||
assert "666" not in db.enrichment
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_sweep_idempotent_skip_already_enriched():
|
||
"""force=False: дом уже в enrichment → пропускается без fetch."""
|
||
db = FakeDB(
|
||
[
|
||
{
|
||
"house_id": 50,
|
||
"ext_id": "555",
|
||
"yandex_jk_slug": "existingslug",
|
||
"yandex_jk_id": "555",
|
||
}
|
||
]
|
||
)
|
||
# Pre-populate enrichment
|
||
db.enrichment["555"] = {"ext_id": "555", "name": "Already enriched"}
|
||
|
||
with patch("scraper_kit.providers.yandex.newbuilding.YandexNewbuildingScraper") as mock_scraper:
|
||
scraper_instance = MagicMock()
|
||
scraper_instance.fetch_jk = AsyncMock(return_value=None)
|
||
mock_scraper.return_value = scraper_instance
|
||
|
||
result = await enrich_yandex_newbuilding_sweep(db, limit=5, request_delay_sec=0)
|
||
|
||
# The SELECT filters out already-enriched rows (force=False in SQL),
|
||
# so processed=0 (SELECT returns empty)
|
||
assert result.succeeded == 0
|
||
scraper_instance.fetch_jk.assert_not_called()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_sweep_no_ext_id_fails_resolve():
|
||
"""House без ext_id и без slug → failed_resolve (нечего резолвить)."""
|
||
db = FakeDB(
|
||
[
|
||
{
|
||
"house_id": 60,
|
||
"ext_id": None,
|
||
"yandex_jk_slug": None,
|
||
"yandex_jk_id": None,
|
||
}
|
||
]
|
||
)
|
||
|
||
result = await enrich_yandex_newbuilding_sweep(db, limit=5, request_delay_sec=0)
|
||
|
||
assert result.failed_resolve == 1
|
||
assert result.succeeded == 0
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_sweep_slug_present_but_ext_id_null_skips_fetch():
|
||
"""House с jk_slug но ext_id=NULL → guard срабатывает, fetch_jk НЕ вызван.
|
||
|
||
M1 review: пустой jk_id даёт URL /{slug}-/, который является битым.
|
||
Ожидаем failed_fetch += 1 и scraper.fetch_jk не вызван.
|
||
"""
|
||
db = FakeDB(
|
||
[
|
||
{
|
||
"house_id": 70,
|
||
"ext_id": None,
|
||
"yandex_jk_slug": "tatlin",
|
||
"yandex_jk_id": None,
|
||
}
|
||
]
|
||
)
|
||
|
||
with patch("scraper_kit.providers.yandex.newbuilding.YandexNewbuildingScraper") as mock_scraper:
|
||
scraper_instance = MagicMock()
|
||
scraper_instance.fetch_jk = AsyncMock()
|
||
mock_scraper.return_value = scraper_instance
|
||
|
||
result = await enrich_yandex_newbuilding_sweep(db, limit=5, request_delay_sec=0)
|
||
|
||
scraper_instance.fetch_jk.assert_not_called()
|
||
assert result.failed_fetch == 1
|
||
assert result.succeeded == 0
|
||
assert result.rows_inserted == 0
|