feat(tradein-scripts): scrapers + probes + Phase C improvements (#551)

tradein-mvp/scripts/avito-playwright-sweep.py

@@ -0,0 +1,371 @@
+"""Playwright-based Avito sweep — обходит Qrator JS challenge через real headless Chromium.
+
+Use case: curl_cffi заблочен Qrator anti-DDoS (429 + captcha challenge). Real
+browser proходит JS challenge автоматически.
+
+Setup:
+    cd tradein-mvp/backend
+    uv pip install playwright
+    uv run playwright install chromium
+
+Usage:
+    export DATABASE_URL='postgresql+psycopg://tradein:<PASS>@localhost:15433/tradein'
+    uv run --project tradein-mvp/backend python tradein-mvp/scripts/avito-playwright-sweep.py
+    # options: --pages N --delay S --mode {citywide|byrooms} --headless
+"""
+from __future__ import annotations
+
+# Banner FIRST — даже до imports чтобы видно что скрипт стартанул:
+import sys as _sys
+
+print("=" * 60, flush=True)
+print("AVITO PLAYWRIGHT SWEEP — starting...", flush=True)
+print("=" * 60, flush=True)
+_sys.stdout.flush()
+
+import argparse
+import asyncio
+import json
+import os
+import random
+import sys
+import time
+import types
+from pathlib import Path
+from urllib.parse import urlencode
+
+# Force unbuffered output
+os.environ["PYTHONUNBUFFERED"] = "1"
+
+_BACKEND = Path(__file__).resolve().parent.parent / "backend"
+if str(_BACKEND) not in sys.path:
+    sys.path.insert(0, str(_BACKEND))
+
+# WeasyPrint stub (Windows без GTK)
+for _m in ("weasyprint", "weasyprint.text", "weasyprint.text.ffi", "weasyprint.css", "weasyprint.css.targets"):
+    sys.modules[_m] = types.ModuleType(_m)
+sys.modules["weasyprint"].HTML = type("HTML", (), {"__init__": lambda self, *a, **kw: None})
+sys.modules["weasyprint"].CSS = type("CSS", (), {"__init__": lambda self, *a, **kw: None})
+
+from playwright.async_api import Browser, BrowserContext, async_playwright  # noqa: E402
+
+try:
+    from playwright_stealth import Stealth  # noqa: E402
+    _STEALTH_AVAILABLE = True
+except ImportError:
+    _STEALTH_AVAILABLE = False
+    print("[warn] playwright-stealth not installed — без stealth, может triggered больше CAPTCHA", flush=True)
+
+from app.core.db import SessionLocal  # noqa: E402
+from app.services.scrapers.avito import AvitoScraper  # noqa: E402
+from app.services.scrapers.base import save_listings  # noqa: E402
+
+COOKIES_FILE = Path(__file__).resolve().parent / ".avito-cookies.json"
+AVITO_BASE = "https://www.avito.ru"
+UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36"
+
+# Reuse AvitoScraper's parser (DOM cards extraction)
+_parser_instance = AvitoScraper()
+_parser_instance._anchor_lat = 0.0  # noqa: SLF001
+_parser_instance._anchor_lon = 0.0  # noqa: SLF001
+_parser_instance._anchor_radius_m = 1000  # noqa: SLF001
+
+
+def load_cookies() -> list[dict]:
+    """Load DevTools-exported cookies → Playwright format."""
+    if not COOKIES_FILE.exists():
+        print(f"WARN: cookies file not found at {COOKIES_FILE} — будем без cookies")
+        return []
+    raw = json.loads(COOKIES_FILE.read_text(encoding="utf-8"))
+    out = []
+    for c in raw:
+        ck = {
+            "name": c["name"],
+            "value": c["value"],
+            "domain": c["domain"],
+            "path": c.get("path", "/"),
+            "secure": c.get("secure", True),
+            "httpOnly": c.get("httpOnly", False),
+        }
+        if c.get("expirationDate"):
+            ck["expires"] = int(c["expirationDate"])
+        ss = c.get("sameSite")
+        if ss == "no_restriction":
+            ck["sameSite"] = "None"
+        elif ss == "lax":
+            ck["sameSite"] = "Lax"
+        elif ss == "strict":
+            ck["sameSite"] = "Strict"
+        out.append(ck)
+    return out
+
+
+def build_citywide_url(page: int) -> str:
+    return f"{AVITO_BASE}/ekaterinburg/kvartiry/prodam-ASgBAgICAUSSA8YQ?{urlencode({'s':104,'p':page})}"
+
+
+ROOM_SLUGS: list[tuple[str, str]] = [
+    ("студии",       "studii-ASgBAgICAkSSA8YQygj~WA"),
+    ("1-комн.",      "1-komnatnye-ASgBAgICAkSSA8YQygiAWQ"),
+    ("2-комн.",      "2-komnatnye-ASgBAgICAkSSA8YQygiCWQ"),
+    ("3-комн.",      "3-komnatnye-ASgBAgICAkSSA8YQygiEWQ"),
+    ("4-комн.",      "4-komnatnye-ASgBAgICAkSSA8YQygiGWQ"),
+    ("5+комн.",      "5-komnatnye-ASgBAgICAkSSA8YQygiIWQ"),
+    ("своб.планир.", "svobodnaya_planirovka-ASgBAgICAkSSA8YQygj8zzI"),
+]
+
+
+def build_rooms_url(slug: str, page: int) -> str:
+    return f"{AVITO_BASE}/ekaterinburg/kvartiry/prodam/{slug}?{urlencode({'s':104,'p':page})}"
+
+
+async def fetch_page(context: BrowserContext, url: str) -> tuple[int, str | None]:
+    """Open page in Playwright, wait for content, return (status, html or None).
+
+    Strategy:
+    1. goto с wait_until='domcontentloaded' (быстро, не ждём всех scripts)
+    2. Если status 200 → wait_for_selector data-marker=item (Qrator challenge должен resolve пока)
+    3. Иначе если 429/403 — challenge не прошёл, дать ещё 5s + retry content (maybe challenge resolves async)
+    """
+    page = await context.new_page()
+    try:
+        try:
+            response = await page.goto(url, wait_until="domcontentloaded", timeout=45000)
+        except Exception as e:  # noqa: BLE001
+            print(f"  goto FAIL: {type(e).__name__}: {str(e)[:80]}")
+            return 0, None
+        status = response.status if response else 0
+        if status in (429, 403):
+            # Qrator challenge — wait для JS auto-resolve, потом reload
+            await asyncio.sleep(5)
+            try:
+                response = await page.reload(wait_until="domcontentloaded", timeout=30000)
+                status = response.status if response else status
+            except Exception:  # noqa: BLE001
+                pass
+        if status != 200:
+            return status, None
+        # Wait for item cards
+        try:
+            await page.wait_for_selector('[data-marker="item"]', timeout=15000)
+        except Exception:  # noqa: BLE001
+            pass
+        html = await page.content()
+        return status, html
+    finally:
+        await page.close()
+
+
+async def sweep_urls(
+    urls_with_label: list[tuple[str, str]],
+    delay: float,
+    headless: bool,
+) -> int:
+    """Generic sweep: list of (label, url) → fetch + parse + save per URL."""
+    cookies = load_cookies()
+    print(f"loaded {len(cookies)} cookies")
+    total_fetched = 0
+    total_ins = 0
+    total_upd = 0
+    challenge_hits = 0
+    print("=== Launching Playwright browser ===", flush=True)
+    async with async_playwright() as p:
+        print(f"  headless={headless}", flush=True)
+        browser: Browser = await p.chromium.launch(
+            headless=headless,
+            args=[
+                "--disable-blink-features=AutomationControlled",
+                "--no-sandbox",
+            ],
+        )
+        print("  browser launched, creating context...", flush=True)
+        context: BrowserContext = await browser.new_context(
+            user_agent=UA,
+            locale="ru-RU",
+            timezone_id="Asia/Yekaterinburg",
+            viewport={"width": 1366, "height": 900},
+        )
+        # Stealth — optional, skip on error
+        if _STEALTH_AVAILABLE:
+            try:
+                stealth = Stealth()
+                await stealth.apply_stealth_async(context)
+                print("  stealth applied", flush=True)
+            except Exception as e:  # noqa: BLE001
+                print(f"  stealth skipped (err): {type(e).__name__}: {e}", flush=True)
+        if cookies:
+            await context.add_cookies(cookies)
+            print(f"  cookies added ({len(cookies)})", flush=True)
+
+        # Initial warmup: open main Avito + wait для user pass CAPTCHA если нужно
+        print("\n=== WARMUP: opening Avito initial page ===", flush=True)
+        print("  creating new page tab...", flush=True)
+        warmup_page = await context.new_page()
+        print("  navigating to Avito (timeout 30s)...", flush=True)
+        try:
+            await warmup_page.goto(
+                "https://www.avito.ru/ekaterinburg/kvartiry/prodam-ASgBAgICAUSSA8YQ?s=104&p=1",
+                wait_until="domcontentloaded",
+                timeout=30000,
+            )
+            print("  goto done, reading content...", flush=True)
+        except Exception as e:  # noqa: BLE001
+            print(f"  goto FAIL: {type(e).__name__}: {e}", flush=True)
+        # Check if challenge page — reliable detector: NO listing cards в HTML.
+        # Слово "captcha" может быть в JS embed страницы normally (false positive).
+        warmup_html = await warmup_page.content()
+        print(f"  content read: {len(warmup_html)} bytes", flush=True)
+        try:
+            await warmup_page.wait_for_selector('[data-marker="item"]', timeout=5000)
+            has_items = True
+        except Exception:  # noqa: BLE001
+            has_items = False
+        is_qrator = "qrator" in warmup_html.lower() and len(warmup_html) < 100000
+        challenge_present = (not has_items) and (is_qrator or len(warmup_html) < 50000)
+        print(f"  has_items={has_items} qrator={is_qrator} size={len(warmup_html)} challenge={challenge_present}", flush=True)
+        if challenge_present:
+            print("\n🚨 CAPTCHA detected на initial page!")
+            print("👉 Окно браузера открыто — пройди капчу руками,")
+            print("👉 затем нажми Enter здесь чтобы продолжить sweep...")
+            try:
+                input(">>> Press ENTER when ready: ")
+            except (EOFError, KeyboardInterrupt):
+                print("aborted")
+                await browser.close()
+                return 0
+            # Verify challenge passed
+            await warmup_page.reload(wait_until="domcontentloaded", timeout=30000)
+            new_html = await warmup_page.content()
+            if any(m in new_html.lower() for m in ("captcha", "qrator")):
+                print("⚠️ всё ещё CAPTCHA — повторно пройди или Ctrl+C")
+                input(">>> Retry ENTER: ")
+            else:
+                print("✅ CAPTCHA пройдена, продолжаю sweep")
+        else:
+            print("✅ Initial page OK (no challenge)")
+        await warmup_page.close()
+
+        try:
+            # Per-category skip flag: 0 lots на странице категории = end of pagination
+            # ДЛЯ ТЕКУЩЕЙ КАТЕГОРИИ, а не для всего sweep'а (был bug: break ломал всё).
+            current_cat: str | None = None
+            skip_until_next_cat = False
+            for i, (label, url) in enumerate(urls_with_label, 1):
+                # Категория = всё до " page=" в label ("1-комн. page=5/60" → "1-комн.")
+                cat = label.split(" page=")[0] if " page=" in label else label
+                if cat != current_cat:
+                    current_cat = cat
+                    skip_until_next_cat = False
+                    challenge_hits = 0  # reset 429-counter на границе категорий
+                if skip_until_next_cat:
+                    continue
+
+                print(f"[{i}/{len(urls_with_label)}] {label}", flush=True)
+                status, html = await fetch_page(context, url)
+                if status != 200 or not html:
+                    print(f"  HTTP {status} → skip", flush=True)
+                    if status in (429, 403):
+                        challenge_hits += 1
+                        if challenge_hits >= 3:
+                            print(f"  ⚠️ {challenge_hits}x challenge hits — sleep 60s extra", flush=True)
+                            print("  если CAPTCHA в окне — пройди и нажми ENTER:", flush=True)
+                            try:
+                                input("  >>> ")
+                            except (EOFError, KeyboardInterrupt):
+                                break
+                            challenge_hits = 0
+                    await asyncio.sleep(delay)
+                    continue
+                # Parse + save
+                lots = _parser_instance._parse_html(html, source_url_base=url)  # noqa: SLF001
+                if not lots:
+                    print(f"  0 lots — end of pagination for {cat!r}", flush=True)
+                    if " page=" in label:
+                        # 0 lots на пагинированной странице = end of category, не sweep
+                        skip_until_next_cat = True
+                    continue
+                db = SessionLocal()
+                try:
+                    ins, upd = save_listings(db, lots)
+                    total_fetched += len(lots)
+                    total_ins += ins
+                    total_upd += upd
+                    print(
+                        f"  {len(lots)} lots → ins={ins} upd={upd} "
+                        f"[cum: tot={total_fetched} new={total_ins}]",
+                        flush=True,
+                    )
+                except Exception as e:  # noqa: BLE001
+                    print(f"  save FAIL: {type(e).__name__}: {e}", flush=True)
+                finally:
+                    db.close()
+                # Jittered delay
+                if i < len(urls_with_label):
+                    j = delay * random.uniform(0.7, 1.3)
+                    await asyncio.sleep(j)
+        finally:
+            await context.close()
+            await browser.close()
+    print(
+        f"\n=== TOTAL: fetched={total_fetched} ins={total_ins} upd={total_upd} ===",
+        flush=True,
+    )
+    return total_fetched
+
+
+async def main() -> None:
+    # Diagnostic — show env vars used:
+    db_url = os.environ.get("DATABASE_URL", "<NOT SET>")
+    # Mask password for logging
+    if "://" in db_url and "@" in db_url:
+        before_at, after_at = db_url.rsplit("@", 1)
+        scheme_user = before_at.split("://", 1)
+        if len(scheme_user) == 2 and ":" in scheme_user[1]:
+            user = scheme_user[1].split(":", 1)[0]
+            masked = f"{scheme_user[0]}://{user}:***@{after_at}"
+        else:
+            masked = db_url[:50] + "..."
+    else:
+        masked = db_url
+    print(f"DATABASE_URL: {masked}", flush=True)
+    print(f"ENVIRONMENT: {os.environ.get('ENVIRONMENT', '<default>')}", flush=True)
+
+    p = argparse.ArgumentParser()
+    p.add_argument("--mode", choices=["citywide", "byrooms"], default="citywide")
+    p.add_argument("--pages", type=int, default=100, help="pages per query")
+    p.add_argument("--delay", type=float, default=7, help="sec между requests")
+    p.add_argument("--headless", action="store_true", default=True)
+    p.add_argument("--no-headless", dest="headless", action="store_false")
+    p.add_argument(
+        "--start-cat",
+        type=int,
+        default=1,
+        help="byrooms: с какой категории начать (1-based). 1=студии, 2=1-комн, "
+             "3=2-комн, 4=3-комн, 5=4-комн, 6=5+комн, 7=своб.планир.",
+    )
+    args = p.parse_args()
+
+    urls: list[tuple[str, str]] = []
+    if args.mode == "citywide":
+        for page in range(1, args.pages + 1):
+            urls.append((f"citywide page={page}/{args.pages}", build_citywide_url(page)))
+    else:  # byrooms
+        cats = ROOM_SLUGS[args.start_cat - 1:]
+        if not cats:
+            print(f"ERROR: --start-cat={args.start_cat} > {len(ROOM_SLUGS)} cats", flush=True)
+            return
+        print(f"byrooms: starting from cat #{args.start_cat} ({cats[0][0]!r}), "
+              f"{len(cats)} cats total", flush=True)
+        for cat_name, slug in cats:
+            for page in range(1, args.pages + 1):
+                urls.append((f"{cat_name} page={page}/{args.pages}", build_rooms_url(slug, page)))
+
+    print(f"Mode: {args.mode}, pages: {args.pages}, delay: {args.delay}s, total URLs: {len(urls)}")
+    print(f"Estimated time: {len(urls) * args.delay / 60:.0f} min (без challenge)")
+    t0 = time.perf_counter()
+    await sweep_urls(urls, args.delay, args.headless)
+    print(f"\nDONE in {(time.perf_counter() - t0) / 60:.1f} min")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

tradein-mvp/scripts/backfill-house-imv.py

@@ -51,6 +51,19 @@ logging.basicConfig(
 )
 logger = logging.getLogger("phase-c")
 
+# Avito raw log — pipe everything from app.services.scrapers.avito_imv to a file
+# so we keep full request/response history for diagnosing contract changes.
+_AVITO_LOG_PATH = os.environ.get("AVITO_LOG_PATH", "avito-raw.log")
+_avito_fh = logging.FileHandler(_AVITO_LOG_PATH, encoding="utf-8")
+_avito_fh.setFormatter(logging.Formatter("%(asctime)s %(levelname)s [%(name)s] %(message)s"))
+_avito_fh.setLevel(logging.DEBUG)
+_avito_logger = logging.getLogger("app.services.scrapers.avito_imv")
+_avito_logger.addHandler(_avito_fh)
+_avito_logger.setLevel(logging.DEBUG)
+# Also pipe phase-c progress lines to the file
+logger.addHandler(_avito_fh)
+logger.info("avito raw log writing to %s", os.path.abspath(_AVITO_LOG_PATH))
+
 # Avito IMV API accepts only these house_type values (verified 2026-05-24).
 # Our `listings.house_type` may contain pre-2026 values like 'monolith' / 'monolith_brick'.
 _HOUSE_TYPE_TO_IMV = {
@@ -147,8 +160,10 @@ def pick_lot_params(db: Session, house_id: int) -> dict:
         {"hid": house_id},
     ).mappings().first()
 
+    # Avito IMV: rooms=0 (студия) rejected with HTTP 400. Clamp to 1.
+    rooms_raw = int(row["rooms"])
     return {
-        "rooms": int(row["rooms"]),
+        "rooms": max(rooms_raw, 1),
         "area_m2": float(row["area_m2"]),
         "floor": int(row["floor"] or 1),
         "floor_at_home": int(row["total_floors"] or (house and house["total_floors"]) or 9),

tradein-mvp/scripts/backfill-yandex-addresses-ekb.py

@@ -0,0 +1,173 @@
+"""Backfill listings.address with full address (street + house number) for
+active EKB yandex listings that currently have street-only addresses.
+
+Why: Yandex SERP returns only street (e.g. 'улица Горького') in card snippet.
+Detail page <title> has the full address ('Екатеринбург, улица Горького, 36').
+With house number we can call /otsenka-kvartiry-po-adresu-onlayn/ and get the
+building's history.
+
+Usage:
+  cd tradein-mvp/backend
+  uv run python ../scripts/backfill-yandex-addresses-ekb.py --limit 5         # smoke
+  uv run python ../scripts/backfill-yandex-addresses-ekb.py                   # full
+  uv run python ../scripts/backfill-yandex-addresses-ekb.py --dry-run         # no DB write
+
+Requires SSH tunnel + DATABASE_URL env (see sweep-yandex-valuation-ekb.py docstring).
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import random
+import re
+import sys
+import time
+import types
+from pathlib import Path
+
+# sys.path injection
+_BACKEND_DIR = Path(__file__).resolve().parent.parent / "backend"
+if str(_BACKEND_DIR) not in sys.path:
+    sys.path.insert(0, str(_BACKEND_DIR))
+
+# Stub weasyprint (Windows GTK-less)
+for _m in (
+    "weasyprint",
+    "weasyprint.text",
+    "weasyprint.text.ffi",
+    "weasyprint.css",
+    "weasyprint.css.targets",
+):
+    sys.modules[_m] = types.ModuleType(_m)
+sys.modules["weasyprint"].HTML = type("HTML", (), {"__init__": lambda self, *a, **kw: None})
+sys.modules["weasyprint"].CSS = type("CSS", (), {"__init__": lambda self, *a, **kw: None})
+
+from curl_cffi.requests import AsyncSession  # noqa: E402
+from sqlalchemy import text  # noqa: E402
+
+from app.core.db import SessionLocal  # noqa: E402
+
+# Extract '<City>, <addr> — id <number>' from <title>
+RE_TITLE_ADDRESS = re.compile(
+    r"<title>[^<]*?—\s*"
+    r"(?:[^—,]*?,\s*)?"  # optional ЖК/жилой комплекс prefix
+    r"(Екатеринбург,\s*[^<]+?)"
+    r"\s*—\s*id\s*\d+\s*</title>",
+    re.IGNORECASE,
+)
+
+
+def _eligible_listings(limit: int | None) -> list[dict]:
+    """Active EKB yandex listings without house number in address."""
+    db = SessionLocal()
+    try:
+        sql = text(
+            """
+            SELECT id, source_id, address, source_url
+            FROM listings
+            WHERE source = 'yandex'
+              AND region_code = 66
+              AND is_active = TRUE
+              AND address IS NOT NULL
+              AND NOT (address ~ ',\\s*\\d+')
+              AND source_url IS NOT NULL
+            ORDER BY scraped_at DESC
+            """
+        )
+        rows = db.execute(sql).mappings().all()
+        out = [dict(r) for r in rows]
+    finally:
+        db.close()
+    if limit is not None and limit > 0:
+        out = out[:limit]
+    return out
+
+
+def _update_listing_address(listing_id: int, new_address: str) -> None:
+    db = SessionLocal()
+    try:
+        db.execute(
+            text("UPDATE listings SET address = :addr WHERE id = :id"),
+            {"addr": new_address, "id": listing_id},
+        )
+        db.commit()
+    finally:
+        db.close()
+
+
+async def _fetch_address(session: AsyncSession, url: str) -> str | None:
+    """Fetch detail page, extract full address from <title>."""
+    try:
+        r = await session.get(url, timeout=30)
+    except Exception as e:
+        print(f"    fetch FAIL: {type(e).__name__}: {e}", flush=True)
+        return None
+    if r.status_code != 200:
+        print(f"    HTTP {r.status_code}", flush=True)
+        return None
+    html = r.text.replace("\xa0", " ")
+    m = RE_TITLE_ADDRESS.search(html)
+    if not m:
+        return None
+    addr = m.group(1).strip().rstrip(",").strip()
+    return addr if addr else None
+
+
+async def main() -> None:
+    p = argparse.ArgumentParser(
+        description="Backfill yandex listings.address with house number from detail page"
+    )
+    p.add_argument("--limit", type=int, default=None, help="Cap items (smoke testing)")
+    p.add_argument("--delay", type=float, default=3.0, help="Sec between requests")
+    p.add_argument("--dry-run", action="store_true", help="Don't UPDATE DB")
+    args = p.parse_args()
+
+    items = _eligible_listings(args.limit)
+    if not items:
+        print("No eligible listings.", flush=True)
+        return
+
+    print(
+        f"=== Backfill {len(items)} listings (delay={args.delay}s dry_run={args.dry_run}) ===",
+        flush=True,
+    )
+
+    t0 = time.perf_counter()
+    enriched = 0
+    skipped = 0
+    failed = 0
+
+    async with AsyncSession(impersonate="chrome120") as session:
+        for i, item in enumerate(items, 1):
+            url = item["source_url"]
+            old_addr = item["address"]
+            print(
+                f"[{i}/{len(items)}] id={item['id']} old={old_addr!r}",
+                flush=True,
+            )
+            new_addr = await _fetch_address(session, url)
+            if new_addr is None:
+                print("    -> no address extracted", flush=True)
+                failed += 1
+            elif new_addr == old_addr:
+                print("    -> unchanged", flush=True)
+                skipped += 1
+            else:
+                print(f"    -> {new_addr!r}", flush=True)
+                if not args.dry_run:
+                    _update_listing_address(item["id"], new_addr)
+                enriched += 1
+            if i < len(items):
+                await asyncio.sleep(args.delay * random.uniform(0.85, 1.3))
+
+    elapsed = time.perf_counter() - t0
+    print(
+        f"\n=== DONE in {elapsed / 60:.1f} min. "
+        f"enriched={enriched} skipped={skipped} failed={failed} ===",
+        flush=True,
+    )
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

tradein-mvp/scripts/inspect-houses-html.py

@@ -0,0 +1,90 @@
+"""One-off: fetch Avito houses HTML и анализ markers для понимания новой структуры.
+
+После 2026-05-23 Avito поменяли rendering — старый `window.__preloadedState__`
+больше не работает (Stage 2c parser fails). Этот скрипт fetch'ит один houses
+catalog page локально через residential IP и сохраняет HTML + проверяет markers
+чтобы понять new format (__NEXT_DATA__? React SSR? иной).
+"""
+import sys
+import types
+from pathlib import Path
+
+_BACKEND = Path(__file__).resolve().parent.parent / "backend"
+if str(_BACKEND) not in sys.path:
+    sys.path.insert(0, str(_BACKEND))
+
+for _m in ("weasyprint", "weasyprint.text", "weasyprint.text.ffi", "weasyprint.css", "weasyprint.css.targets"):
+    sys.modules[_m] = types.ModuleType(_m)
+sys.modules["weasyprint"].HTML = type("HTML", (), {"__init__": lambda self, *a, **kw: None})
+sys.modules["weasyprint"].CSS = type("CSS", (), {"__init__": lambda self, *a, **kw: None})
+
+import asyncio  # noqa: E402
+
+from curl_cffi.requests import AsyncSession  # noqa: E402
+
+URLS = [
+    "https://www.avito.ru/catalog/houses/ekaterinburg/ul_yumasheva_6/115188",
+    "https://www.avito.ru/catalog/houses/ekaterinburg/ul_akademika_postovskogo_17a/3171365",
+]
+
+
+async def main() -> None:
+    async with AsyncSession(
+        impersonate="chrome120",
+        timeout=25,
+        headers={
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+            "Accept-Language": "ru-RU,ru;q=0.9",
+            "Sec-Fetch-Dest": "document",
+            "Sec-Fetch-Mode": "navigate",
+            "Sec-Fetch-Site": "none",
+        },
+    ) as s:
+        for url in URLS:
+            print(f"\n=== {url} ===")
+            try:
+                r = await s.get(url)
+            except Exception as e:  # noqa: BLE001
+                print(f"  FAIL fetch: {type(e).__name__}: {e}")
+                continue
+            print(f"  status={r.status_code} size={len(r.content)}")
+            if r.status_code != 200:
+                continue
+            text = r.text
+            # Save first one для inspection
+            if "yumasheva" in url:
+                with Path("/tmp/avito-house-sample.html").open("wb") as f:
+                    f.write(r.content)
+                print("  saved → /tmp/avito-house-sample.html")
+            markers = {
+                "__preloadedState__": "__preloadedState__" in text,
+                "__NEXT_DATA__": "__NEXT_DATA__" in text,
+                "__INITIAL_STATE__": "__INITIAL_STATE__" in text,
+                "datadome": "datadome" in text.lower(),
+                "captcha": "captcha" in text.lower(),
+                "data-marker": "data-marker" in text,
+                "data-app-state": "data-app-state" in text,
+                "developmentData": "developmentData" in text,
+                "avitoId": "avitoId" in text,
+                "window.__": "window.__" in text,
+                "<script id=": "<script id=" in text,
+                "ssr-state": "ssr-state" in text,
+                "__NUXT__": "__NUXT__" in text,
+                "Не найдено": "Не найдено" in text,
+                "недоступна": "недоступна" in text,
+            }
+            for k, v in markers.items():
+                print(f"  {'✓' if v else '✗'} {k}")
+            # Look for script id="..." patterns (Next.js typical)
+            import re
+            script_ids = re.findall(r'<script[^>]*id="([^"]+)"', text)
+            if script_ids:
+                print(f"  script IDs found: {script_ids[:10]}")
+            data_attrs = re.findall(r'<\w+\s+(data-[\w-]+)=', text)
+            if data_attrs:
+                unique = sorted(set(data_attrs))[:15]
+                print(f"  data attrs sample: {unique}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

tradein-mvp/scripts/inspect-search-html.py

@@ -0,0 +1,55 @@
+"""Fetch Avito search page и анализ URL slugs для room-filter."""
+import re
+import sys
+import types
+from pathlib import Path
+
+_BACKEND = Path(__file__).resolve().parent.parent / "backend"
+if str(_BACKEND) not in sys.path:
+    sys.path.insert(0, str(_BACKEND))
+
+for _m in ("weasyprint", "weasyprint.text", "weasyprint.text.ffi", "weasyprint.css", "weasyprint.css.targets"):
+    sys.modules[_m] = types.ModuleType(_m)
+sys.modules["weasyprint"].HTML = type("HTML", (), {"__init__": lambda self, *a, **kw: None})
+sys.modules["weasyprint"].CSS = type("CSS", (), {"__init__": lambda self, *a, **kw: None})
+
+import asyncio  # noqa: E402
+
+from curl_cffi.requests import AsyncSession  # noqa: E402
+
+
+async def main() -> None:
+    url = "https://www.avito.ru/ekaterinburg/kvartiry/prodam-ASgBAgICAUSSA8YQ?s=104&p=1"
+    async with AsyncSession(
+        impersonate="chrome120",
+        headers={
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+            "Accept-Language": "ru-RU,ru;q=0.9",
+            "Sec-Fetch-Dest": "document",
+            "Sec-Fetch-Mode": "navigate",
+            "Sec-Fetch-Site": "none",
+        },
+    ) as s:
+        r = await s.get(url)
+        print(f"status={r.status_code} size={len(r.content)}")
+        text = r.text
+        with Path("/tmp/avito-search-sample.html").open("wb") as f:
+            f.write(r.content)
+        print("saved → /tmp/avito-search-sample.html")
+        # Find filter links — pattern /ekaterinburg/kvartiry/prodam/<slug>-ASg...
+        links = re.findall(r'href="(/ekaterinburg/kvartiry/prodam/[^"]+)"', text)
+        unique = sorted(set(links))
+        print(f"\nUnique filter links ({len(unique)} found):")
+        for link in unique[:30]:
+            # Decode shortened
+            if "komnatn" in link.lower() or "studii" in link.lower() or "svobodnaya" in link.lower():
+                print(f"  ROOMS: {link}")
+            else:
+                print(f"  other: {link[:120]}")
+        # Also check direct headings
+        room_headings = re.findall(r'>(Студии?|[0-9]-[Кк]омнатн\w+|Свободная планировка)<', text)
+        print(f"\nRoom headings in HTML: {sorted(set(room_headings))}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

tradein-mvp/scripts/local-sweep-ekb-cian.py

@@ -0,0 +1,359 @@
+"""Cian ЕКБ vtorichka sweep — local residential IP → prod tradein DB через SSH tunnel.
+
+Why: ekb.cian.ru SERP отдаёт ВЕСЬ город (без geo-фильтра по lat/lon), поэтому
+"10 anchors" из Avito-эталона тут бесполезны — одинаковый ответ × 10.
+Вместо anchors — proper room-segmented citywide pagination
+(rooms 1/2/3/4 × N pages × ~28 listings), per-page save (visible progress +
+не теряем данные при крэше).
+
+Cian реально менее агрессивен чем Avito, но **residential IP всё равно может попасть
+в blacklist при слишком частых запросах** — default delay 15s.
+
+Setup (один раз):
+  ssh -f -N -L 15433:172.20.0.2:5432 gendesign
+  cd tradein-mvp/backend
+  $env:DATABASE_URL='postgresql+psycopg://tradein:<PASS>@localhost:15433/tradein'
+  $env:ENVIRONMENT='local'; $env:GLITCHTIP_DSN=''
+
+Usage:
+  uv run python ../scripts/local-sweep-ekb-cian.py                 # default: 4 rooms × 8 pages
+  uv run python ../scripts/local-sweep-ekb-cian.py --pages 20      # больше глубины
+  uv run python ../scripts/local-sweep-ekb-cian.py --delay 10      # быстрее (risk)
+  uv run python ../scripts/local-sweep-ekb-cian.py --rooms 0       # всё (включая студии — no room param)
+"""
+from __future__ import annotations
+
+# stdlib
+import argparse
+import asyncio
+import json
+import sys
+import time
+import types
+from pathlib import Path
+
+# Auto-inject backend/ в sys.path чтобы найти app.* package
+_BACKEND_DIR = Path(__file__).resolve().parent.parent / "backend"
+if str(_BACKEND_DIR) not in sys.path:
+    sys.path.insert(0, str(_BACKEND_DIR))
+
+# Stub weasyprint (Windows без GTK ломает import chain через trade_in.py)
+for _m in (
+    "weasyprint",
+    "weasyprint.text",
+    "weasyprint.text.ffi",
+    "weasyprint.css",
+    "weasyprint.css.targets",
+):
+    sys.modules[_m] = types.ModuleType(_m)
+sys.modules["weasyprint"].HTML = type("HTML", (), {"__init__": lambda self, *a, **kw: None})
+sys.modules["weasyprint"].CSS = type("CSS", (), {"__init__": lambda self, *a, **kw: None})
+
+# app imports после stubs
+import logging  # noqa: E402
+import re  # noqa: E402
+
+from app.core.db import SessionLocal  # noqa: E402
+from app.services.scrapers.base import ScrapedLot, save_listings  # noqa: E402
+from app.services.scrapers.cian import CianScraper  # noqa: E402
+
+# Шумный WARNING из upstream cian_state_parser ('state extraction failed') — у нас
+# fallback на новый .concat() markup отрабатывает, warning нерелевантен.
+logging.getLogger("app.services.scrapers.cian").setLevel(logging.ERROR)
+
+# ── New-markup fallback parser ───────────────────────────────────────────────
+# Cian переключил MFE bootstrap с .push({...}) на .concat([{...},{...}]):
+#   window._cianConfig['frontend-serp'] = (window._cianConfig['frontend-serp'] || []).concat([…])
+# Существующий cian_state_parser.py не ловит этот шаблон → 0 lots на свежей выдаче.
+# Чтобы не трогать shared scraper module (правилами запрещено), парсим тут.
+_RE_CONCAT_START = re.compile(
+    r"window\._cianConfig\['(?P<mfe>[\w-]+)'\]\s*=\s*"
+    r"\(window\._cianConfig\['[\w-]+'\]\s*\|\|\s*\[\]\)\.concat\(\["
+)
+
+
+def _extract_balanced_array(text: str, start: int) -> str | None:
+    """Из позиции `start` (где '[') вернуть текст массива включая внешние '[' и ']'.
+
+    JSON-aware: учитывает strings и escapes, чтобы не сбить счётчик '[' ']' на
+    скобках внутри string literals.
+    """
+    if text[start] != "[":
+        return None
+    depth = 0
+    i = start
+    in_str = False
+    esc = False
+    while i < len(text):
+        ch = text[i]
+        if in_str:
+            if esc:
+                esc = False
+            elif ch == "\\":
+                esc = True
+            elif ch == '"':
+                in_str = False
+        else:
+            if ch == '"':
+                in_str = True
+            elif ch == "[":
+                depth += 1
+            elif ch == "]":
+                depth -= 1
+                if depth == 0:
+                    return text[start : i + 1]
+        i += 1
+    return None
+
+
+def _parse_serp_v2(scraper: CianScraper, html: str) -> list[ScrapedLot]:
+    """Fallback парсер для нового шаблона .concat([...]) — ekb.cian.ru/cat.php c 2026-05."""
+    state: dict | None = None
+    for m in _RE_CONCAT_START.finditer(html):
+        if m.group("mfe") != "frontend-serp":
+            continue
+        # Позиция '[' — это последний символ матча минус 1 (regex кончается на '[')
+        arr_start = m.end() - 1
+        arr_str = _extract_balanced_array(html, arr_start)
+        if arr_str is None:
+            continue
+        try:
+            entries = json.loads(arr_str)
+        except json.JSONDecodeError:
+            continue
+        for entry in entries:
+            if (
+                isinstance(entry, dict)
+                and entry.get("key") == "initialState"
+                and isinstance(entry.get("value"), dict)
+            ):
+                state = entry["value"]
+                break
+        if state is not None:
+            break
+    if state is None:
+        return []
+    offers = (state.get("results") or {}).get("offers") or []
+    lots: list[ScrapedLot] = []
+    for offer in offers:
+        # Cian свежий markup: descriptionMinhash = list[int] (multi-hash signatures).
+        # ScrapedLot ожидает str → сбрасываем чтобы _offer_to_lot вычислил SHA1 от description.
+        if isinstance(offer.get("descriptionMinhash"), list):
+            offer["descriptionMinhash"] = None
+        lot = scraper._offer_to_lot(offer)
+        if lot is not None:
+            lots.append(lot)
+    return lots
+
+
+def _load_cookies_file(path: Path) -> dict[str, str]:
+    """Загрузить cookies из JSON-файла.
+
+    Поддерживает два формата:
+    - browser export (list[{name,value,...}]) — DevTools / EditThisCookie / Cookie-Editor
+    - plain dict {name: value}
+    """
+    raw = json.loads(path.read_text(encoding="utf-8"))
+    if isinstance(raw, list):
+        return {c["name"]: c["value"] for c in raw if c.get("name") and c.get("value")}
+    if isinstance(raw, dict):
+        return {str(k): str(v) for k, v in raw.items()}
+    raise SystemExit(f"--cookies-file: unknown JSON shape (got {type(raw).__name__})")
+
+
+async def phase_citywide(
+    pages: int,
+    delay: float,
+    room_segments: tuple[tuple[int, ...] | None, ...],
+    cookies_file: Path | None = None,
+    start_page: int = 1,
+) -> int:
+    """Room-segmented citywide pagination — per-page save, incremental.
+
+    room_segments — кортеж из tuple (1,) / (2,) / ... / None (= no room filter).
+    None segment даёт смешанную выдачу (студии + всё подряд) — Cian
+    не различает их без room param.
+    """
+    seg_labels = [
+        ("r" + str(seg[0]) if seg else "all")
+        for seg in room_segments
+    ]
+    print(
+        f"\n=== PHASE CITYWIDE: Cian ЕКБ vtorichka "
+        f"({len(room_segments)} segments={seg_labels} × {pages} pages, delay {delay}s) ===\n",
+        flush=True,
+    )
+    total = 0
+    cum_ins = 0
+    cum_upd = 0
+    async with CianScraper() as s:
+        # Override per-request delay (in addition to between-anchors delay)
+        s.request_delay_sec = delay
+        # Inject cookies (если есть): прокидываем в curl_cffi session перед warm-up
+        if cookies_file is not None:
+            cookies = _load_cookies_file(cookies_file)
+            assert s._cffi is not None
+            s._cffi.cookies.update(cookies)
+            print(f"  loaded {len(cookies)} cookies from {cookies_file.name}", flush=True)
+        # Warm-up: посетить homepage чтобы получить _CIAN_GK / _yasc + не выглядеть как наглый бот
+        try:
+            assert s._cffi is not None
+            r_warm = await s._cffi.get(f"{s.base_url}/")
+            print(
+                f"  warmup GET / → HTTP {r_warm.status_code} "
+                f"(cookies={len(list(s._cffi.cookies.jar))})",
+                flush=True,
+            )
+            # Subsequent SERP requests should look like in-site navigation
+            s._cffi.headers.update(
+                {
+                    "Referer": f"{s.base_url}/",
+                    "Sec-Fetch-Site": "same-origin",
+                    "Sec-Fetch-Mode": "navigate",
+                    "Sec-Fetch-User": "?1",
+                    "Sec-Fetch-Dest": "document",
+                }
+            )
+            await asyncio.sleep(min(5.0, delay))
+        except Exception as e:  # noqa: BLE001
+            print(f"  warmup FAIL: {type(e).__name__}: {e}", flush=True)
+        for seg_idx, seg in enumerate(room_segments):
+            seg_label = "r" + str(seg[0]) if seg else "all"
+            # --start-page применяется только к первому segment'у — продолжение прерванного sweep
+            page_from = start_page if seg_idx == 0 else 1
+            for page in range(page_from, pages + 1):
+                # Manual fetch+parse чтобы можно было дампнуть HTML при empty results
+                try:
+                    url_ = s._build_url(rooms=seg, page=page)
+                    assert s._cffi is not None
+                    resp = await s._cffi.get(url_)
+                    if resp.status_code != 200:
+                        print(
+                            f"  seg={seg_label} page={page} HTTP {resp.status_code} → skip",
+                            flush=True,
+                        )
+                        await asyncio.sleep(delay)
+                        continue
+                    lots = s._parse_serp_html(resp.text)
+                    if not lots:
+                        # Fallback: новый markup .concat([...])
+                        lots = _parse_serp_v2(s, resp.text)
+                        if lots:
+                            print(
+                                f"    (parsed via v2 .concat fallback)",
+                                flush=True,
+                            )
+                    if not lots:
+                        # Дамп для разбора: captcha vs пустая страница vs изменение markup
+                        dump_path = Path(f"cian-empty-seg-{seg_label}-p{page}.html")
+                        dump_path.write_text(resp.text, encoding="utf-8")
+                        print(f"    DUMP: {dump_path} ({len(resp.text)} bytes)", flush=True)
+                    await s.sleep_between_requests()
+                except Exception as e:  # noqa: BLE001
+                    print(
+                        f"  seg={seg_label} page={page} FAIL fetch: "
+                        f"{type(e).__name__}: {e}",
+                        flush=True,
+                    )
+                    await asyncio.sleep(delay)
+                    continue
+                if not lots:
+                    print(
+                        f"  seg={seg_label} page={page}: 0 lots — end of segment "
+                        "(либо captcha)",
+                        flush=True,
+                    )
+                    break
+                db = SessionLocal()
+                try:
+                    ins, upd = save_listings(db, lots)
+                    cum_ins += ins
+                    cum_upd += upd
+                    total += len(lots)
+                    print(
+                        f"  seg={seg_label} page={page}/{pages}: {len(lots)} lots → "
+                        f"ins={ins} upd={upd} "
+                        f"[cum: total={total} new={cum_ins} upd={cum_upd}]",
+                        flush=True,
+                    )
+                except Exception as e:  # noqa: BLE001
+                    print(
+                        f"  seg={seg_label} page={page} FAIL save: "
+                        f"{type(e).__name__}: {e}",
+                        flush=True,
+                    )
+                finally:
+                    db.close()
+                # scraper.fetch_around() уже спит request_delay_sec * jitter — extra sleep не нужен
+    print(
+        f"  PHASE CITYWIDE TOTAL: fetched={total} ins={cum_ins} upd={cum_upd}",
+        flush=True,
+    )
+    return total
+
+
+async def main() -> None:
+    p = argparse.ArgumentParser()
+    p.add_argument(
+        "--pages",
+        type=int,
+        default=8,
+        help="страниц на room-segment (default 8 → ~28×4×8 = ~900 до дедупа)",
+    )
+    p.add_argument(
+        "--delay",
+        type=float,
+        default=15,
+        help="sec между HTTP requests (residential-safe ≥10, default 15)",
+    )
+    p.add_argument(
+        "--rooms",
+        type=str,
+        default="1,2,3,4",
+        help="room segments через запятую (1,2,3,4) или 'all' для одного citywide "
+        "запроса без room param (default '1,2,3,4')",
+    )
+    p.add_argument(
+        "--cookies-file",
+        type=Path,
+        default=None,
+        help="JSON-файл с Cian cookies (browser-export или {name:value}). "
+        "Опционально — для обхода captcha. См. scripts/.cian-cookies.json",
+    )
+    p.add_argument(
+        "--start-page",
+        type=int,
+        default=1,
+        help="С какой страницы начать ПЕРВЫЙ сегмент (для resume прерванного sweep). "
+        "Остальные segments идут с p=1. Default 1.",
+    )
+    args = p.parse_args()
+
+    # Parse rooms argument
+    if args.rooms.strip().lower() == "all":
+        segments: tuple[tuple[int, ...] | None, ...] = (None,)
+    else:
+        try:
+            nums = [int(x.strip()) for x in args.rooms.split(",") if x.strip()]
+        except ValueError as e:
+            raise SystemExit(f"--rooms must be comma-separated ints or 'all': {e}") from e
+        segments = tuple((n,) for n in nums)
+
+    cookies_file: Path | None = args.cookies_file
+    if cookies_file is not None and not cookies_file.exists():
+        raise SystemExit(f"--cookies-file not found: {cookies_file}")
+
+    t0 = time.perf_counter()
+    await phase_citywide(
+        args.pages,
+        args.delay,
+        segments,
+        cookies_file=cookies_file,
+        start_page=args.start_page,
+    )
+    elapsed = time.perf_counter() - t0
+    print(f"\n=== ALL DONE in {elapsed / 60:.1f} min ===", flush=True)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

tradein-mvp/scripts/local-sweep-ekb-yandex.py

@@ -0,0 +1,376 @@
+"""Yandex.Недвижимость ЕКБ vtorichka sweep — runs locally через residential IP.
+
+Why local: Yandex anti-bot (captcha redirect) включается быстрее на datacenter
+IP + httpx fingerprint. Запуск с residential IP + Chrome120 TLS impersonation
+(curl_cffi) + browser session cookies — проходит. Скрипт пишет в prod tradein
+DB через SSH tunnel (localhost:15433 → tradein-postgres docker container).
+
+Modes:
+  --mode citywide  (default): один общий /vtorichniy-rynok/ sweep, ~25 страниц
+                              × 23 cards = ~575 уникальных. Yandex SERP лимит.
+  --mode combos:    итерация 5 rooms × 6 price ranges = 30 combos × до 25 pages.
+                    DB-level дедуп → coverage близко к total (~4000+ vtorichka
+                    ЕКБ). ~75 min полный прогон.
+
+Setup:
+  ssh -f -N -L 15433:172.20.0.2:5432 gendesign   # tradein-postgres tunnel
+  cd tradein-mvp/backend
+  export DATABASE_URL='postgresql+psycopg://tradein:<PASS>@localhost:15433/tradein'
+  export ENVIRONMENT='local'
+  export GLITCHTIP_DSN=''
+
+Usage:
+  uv run python ../scripts/local-sweep-ekb-yandex.py                       # citywide
+  uv run python ../scripts/local-sweep-ekb-yandex.py --mode combos          # full ~4000
+  uv run python ../scripts/local-sweep-ekb-yandex.py --mode combos \\
+      --rooms 1,2 --price-ranges 5-7,7-10                                   # subset
+  uv run python ../scripts/local-sweep-ekb-yandex.py --pages 1              # smoke
+"""
+from __future__ import annotations
+
+# stdlib
+import argparse
+import asyncio
+import json
+import random
+import sys
+import time
+import types
+from pathlib import Path
+from urllib.parse import urlencode
+
+# Auto-inject backend/ в sys.path чтобы найти app.* package
+_BACKEND_DIR = Path(__file__).resolve().parent.parent / "backend"
+if str(_BACKEND_DIR) not in sys.path:
+    sys.path.insert(0, str(_BACKEND_DIR))
+
+# Stub weasyprint (Windows без GTK ломает import chain через trade_in.py)
+for _m in (
+    "weasyprint",
+    "weasyprint.text",
+    "weasyprint.text.ffi",
+    "weasyprint.css",
+    "weasyprint.css.targets",
+):
+    sys.modules[_m] = types.ModuleType(_m)
+sys.modules["weasyprint"].HTML = type("HTML", (), {"__init__": lambda self, *a, **kw: None})
+sys.modules["weasyprint"].CSS = type("CSS", (), {"__init__": lambda self, *a, **kw: None})
+
+# app imports после stubs
+from curl_cffi.requests import AsyncSession  # noqa: E402
+
+from app.core.db import SessionLocal  # noqa: E402
+from app.services.scrapers.base import save_listings  # noqa: E402
+from app.services.scrapers.yandex_realty import YandexRealtyScraper  # noqa: E402
+
+DEFAULT_CITY = "ekaterinburg"
+DEFAULT_COOKIES_FILE = Path(__file__).resolve().parent / ".yandex-cookies.json"
+
+# Path-сегменты Yandex для room buckets (после канонизации). Ключ — slug для
+# --rooms CLI; значение — segment в URL.
+ROOM_PATH: dict[str, str] = {
+    "studio": "studiya",
+    "1": "odnokomnatnaya",
+    "2": "dvuhkomnatnaya",
+    "3": "trehkomnatnaya",
+    "4+": "4-i-bolee",
+}
+
+# Price ranges (price_min, price_max) в рублях. None — open-ended boundary.
+PRICE_RANGES: list[tuple[int | None, int | None]] = [
+    (None, 5_000_000),
+    (5_000_000, 7_000_000),
+    (7_000_000, 10_000_000),
+    (10_000_000, 15_000_000),
+    (15_000_000, 25_000_000),
+    (25_000_000, None),
+]
+
+_CAPTCHA_MARKERS = ("/showcaptcha", "smartcaptcha", "checkboxcaptcha")
+
+
+def _load_cookies(path: Path) -> dict[str, str]:
+    if not path.exists():
+        return {}
+    raw = json.loads(path.read_text(encoding="utf-8"))
+    return {item["name"]: item["value"] for item in raw if "name" in item and "value" in item}
+
+
+def _is_captcha(html: str) -> bool:
+    head = html[:5000].lower()
+    return any(m in head for m in _CAPTCHA_MARKERS) or "докажите, что вы не робот" in head
+
+
+def _build_url(
+    city: str,
+    page: int = 0,
+    rooms: str | None = None,
+    price_min: int | None = None,
+    price_max: int | None = None,
+) -> str:
+    """Yandex SERP URL. Если rooms задан — path-based бакет; иначе общий vtorichka.
+
+    newFlat=NO_DEAL принудительно отфильтровывает новостройки (нужно при
+    combo, т.к. Yandex canonicalize иногда дропает /vtorichniy-rynok/).
+    """
+    if rooms:
+        path = f"/{city}/kupit/kvartira/{ROOM_PATH[rooms]}/vtorichniy-rynok/"
+    else:
+        path = f"/{city}/kupit/kvartira/vtorichniy-rynok/"
+    base = f"https://realty.yandex.ru{path}"
+
+    params: dict[str, str | int] = {}
+    if price_min is not None:
+        params["priceMin"] = price_min
+    if price_max is not None:
+        params["priceMax"] = price_max
+    if price_min is not None or price_max is not None:
+        # Гарантия vtorichka — Yandex иногда дропает /vtorichniy-rynok/ при canonicalize
+        params["newFlat"] = "NO_DEAL"
+    if page > 0:
+        params["page"] = page
+
+    if params:
+        return f"{base}?{urlencode(params)}"
+    return base
+
+
+def _combo_label(rooms: str | None, price_min: int | None, price_max: int | None) -> str:
+    r = rooms or "all-rooms"
+    if price_min is None and price_max is None:
+        return f"{r}/any-price"
+    lo = f"{price_min // 1_000_000}M" if price_min else "0"
+    hi = f"{price_max // 1_000_000}M" if price_max else "inf"
+    return f"{r}/{lo}-{hi}"
+
+
+async def sweep_one(
+    session: AsyncSession,
+    parser: YandexRealtyScraper,
+    city: str,
+    pages: int,
+    delay: float,
+    rooms: str | None = None,
+    price_min: int | None = None,
+    price_max: int | None = None,
+) -> tuple[int, int, int]:
+    """Один scan по фиксированному (rooms, price) — пагинация до pages или empty.
+
+    Returns: (fetched, inserted, updated)
+    """
+    label = _combo_label(rooms, price_min, price_max)
+    total = cum_ins = cum_upd = 0
+    for page in range(pages):
+        url = _build_url(city, page=page, rooms=rooms, price_min=price_min, price_max=price_max)
+        try:
+            resp = await session.get(url, timeout=20)
+        except Exception as e:  # noqa: BLE001
+            print(f"    [{label}] page={page} FAIL fetch: {type(e).__name__}: {e}", flush=True)
+            await asyncio.sleep(delay)
+            continue
+        if resp.status_code != 200:
+            print(f"    [{label}] page={page} HTTP {resp.status_code} — stop combo", flush=True)
+            break
+        if _is_captcha(resp.text):
+            print(
+                f"    [{label}] page={page} ⚠️ CAPTCHA — stop combo. "
+                f"Подожди 30-60min или refresh .yandex-cookies.json",
+                flush=True,
+            )
+            raise RuntimeError("captcha")
+        # NBSP → space: yandex_helpers.parse_rub() багует на \xa0 в group(1) → price None
+        html = resp.text.replace("\xa0", " ")
+        lots = parser._parse_html(html, page=page)
+        if not lots:
+            print(f"    [{label}] page={page}: 0 lots — end of pagination, stop", flush=True)
+            break
+        db = SessionLocal()
+        try:
+            ins, upd = save_listings(db, lots)
+            cum_ins += ins
+            cum_upd += upd
+            total += len(lots)
+            print(
+                f"    [{label}] page={page + 1}/{pages}: {len(lots)} lots → "
+                f"ins={ins} upd={upd} [combo: {total}/{cum_ins}/{cum_upd}]",
+                flush=True,
+            )
+        except Exception as e:  # noqa: BLE001
+            print(f"    [{label}] page={page} FAIL save: {type(e).__name__}: {e}", flush=True)
+        finally:
+            db.close()
+        if page + 1 < pages:
+            await asyncio.sleep(delay * random.uniform(0.85, 1.3))
+    return total, cum_ins, cum_upd
+
+
+async def phase_citywide(pages: int, delay: float, city: str, cookies: dict[str, str]) -> None:
+    print(
+        f"\n=== PHASE CITYWIDE: Yandex vtorichka (city={city}, {pages} pages, "
+        f"delay {delay}s) ===\n",
+        flush=True,
+    )
+    parser = YandexRealtyScraper(city=city)
+    async with AsyncSession(impersonate="chrome120", cookies=cookies or {}) as session:
+        total, ins, upd = await sweep_one(session, parser, city, pages, delay)
+    print(f"  CITYWIDE TOTAL: fetched={total} ins={ins} upd={upd}", flush=True)
+
+
+async def phase_combos(
+    pages: int,
+    delay: float,
+    city: str,
+    cookies: dict[str, str],
+    rooms_list: list[str],
+    price_ranges: list[tuple[int | None, int | None]],
+    combo_delay: float,
+) -> None:
+    combos = [(r, lo, hi) for r in rooms_list for lo, hi in price_ranges]
+    print(
+        f"\n=== PHASE COMBOS: {len(combos)} combos × ≤{pages} pages "
+        f"(city={city}, delay {delay}s, combo-delay {combo_delay}s) ===",
+        flush=True,
+    )
+    parser = YandexRealtyScraper(city=city)
+    grand_fetched = grand_ins = grand_upd = 0
+    async with AsyncSession(impersonate="chrome120", cookies=cookies or {}) as session:
+        for i, (rooms, lo, hi) in enumerate(combos, 1):
+            label = _combo_label(rooms, lo, hi)
+            print(f"\n  [{i}/{len(combos)}] combo={label}", flush=True)
+            try:
+                f, ins, upd = await sweep_one(
+                    session, parser, city, pages, delay,
+                    rooms=rooms, price_min=lo, price_max=hi,
+                )
+            except RuntimeError as e:
+                if str(e) == "captcha":
+                    print("  ⚠️ stopping combo run — captcha", flush=True)
+                    break
+                raise
+            grand_fetched += f
+            grand_ins += ins
+            grand_upd += upd
+            print(
+                f"  [{i}/{len(combos)}] {label} done: f={f} i={ins} u={upd} "
+                f"[grand: f={grand_fetched} i={grand_ins} u={grand_upd}]",
+                flush=True,
+            )
+            if i < len(combos):
+                await asyncio.sleep(combo_delay * random.uniform(0.85, 1.3))
+    print(
+        f"\n  COMBOS GRAND TOTAL: fetched={grand_fetched} "
+        f"ins={grand_ins} upd={grand_upd}",
+        flush=True,
+    )
+
+
+def _parse_rooms_arg(s: str) -> list[str]:
+    if s == "all":
+        return list(ROOM_PATH)
+    out: list[str] = []
+    for tok in s.split(","):
+        tok = tok.strip()
+        if tok not in ROOM_PATH:
+            raise argparse.ArgumentTypeError(
+                f"unknown room '{tok}'. Allowed: {sorted(ROOM_PATH)} or 'all'"
+            )
+        out.append(tok)
+    return out
+
+
+def _parse_price_ranges_arg(s: str) -> list[tuple[int | None, int | None]]:
+    if s == "all":
+        return list(PRICE_RANGES)
+    out: list[tuple[int | None, int | None]] = []
+    for tok in s.split(","):
+        tok = tok.strip()
+        if "-" not in tok:
+            raise argparse.ArgumentTypeError(f"range '{tok}' missing '-'")
+        lo_s, hi_s = tok.split("-", 1)
+        lo = int(float(lo_s) * 1_000_000) if lo_s else None
+        hi = int(float(hi_s) * 1_000_000) if hi_s else None
+        out.append((lo, hi))
+    return out
+
+
+async def main() -> None:
+    p = argparse.ArgumentParser(description="Yandex.Недвижимость ЕКБ sweep")
+    p.add_argument(
+        "--mode",
+        choices=["citywide", "combos"],
+        default="citywide",
+        help="citywide = один общий vtorichka sweep (~575 уникальных). "
+        "combos = rooms × price = ~4000+",
+    )
+    p.add_argument(
+        "--pages",
+        type=int,
+        default=25,
+        help="max pages per sweep (Yandex SERP cap ≈ 25; stops earlier on empty)",
+    )
+    p.add_argument(
+        "--delay",
+        type=float,
+        default=15,
+        help="sec между HTTP requests внутри combo (default 15, residential-safe)",
+    )
+    p.add_argument(
+        "--combo-delay",
+        type=float,
+        default=20,
+        help="sec между combos (default 20, дополнительный jitter combos mode)",
+    )
+    p.add_argument(
+        "--rooms",
+        type=_parse_rooms_arg,
+        default="all",
+        help=f"combos: comma-list из {sorted(ROOM_PATH)} или 'all' (default)",
+    )
+    p.add_argument(
+        "--price-ranges",
+        type=_parse_price_ranges_arg,
+        default="all",
+        help="combos: comma-list 'lo-hi' в млн руб ('-5,5-7,7-10,10-15,15-25,25-') "
+        "или 'all' (default). Empty side = open-ended.",
+    )
+    p.add_argument(
+        "--city",
+        type=str,
+        default=DEFAULT_CITY,
+        help=f"Yandex city slug (default '{DEFAULT_CITY}')",
+    )
+    p.add_argument(
+        "--cookies-file",
+        type=Path,
+        default=DEFAULT_COOKIES_FILE,
+        help=f"Browser cookies JSON (default '{DEFAULT_COOKIES_FILE.name}'). "
+        f"Empty/missing → start без cookies (риск captcha).",
+    )
+    args = p.parse_args()
+
+    cookies = _load_cookies(args.cookies_file)
+    if cookies:
+        print(f"loaded {len(cookies)} cookies from {args.cookies_file}", flush=True)
+    else:
+        print(
+            f"⚠️ no cookies loaded ({args.cookies_file}, exists={args.cookies_file.exists()}). "
+            f"Yandex anti-bot risk.",
+            flush=True,
+        )
+
+    t0 = time.perf_counter()
+    if args.mode == "citywide":
+        await phase_citywide(args.pages, args.delay, args.city, cookies)
+    else:
+        await phase_combos(
+            args.pages, args.delay, args.city, cookies,
+            rooms_list=args.rooms,
+            price_ranges=args.price_ranges,
+            combo_delay=args.combo_delay,
+        )
+    elapsed = time.perf_counter() - t0
+    print(f"\n=== ALL DONE in {elapsed / 60:.1f} min ===", flush=True)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

tradein-mvp/scripts/local-sweep-ekb.py

@@ -0,0 +1,443 @@
+"""Full ЕКБ sweep + houses enrichment — runs locally через residential IP.
+
+Why: prod VPS Beget IP заблокирован Avito (datacenter fingerprint → 429).
+Local IP (домашний ISP) = residential → проходит. Скрипт пишет в prod DB
+через SSH tunnel (localhost:15433 → tradein-postgres docker container).
+
+Setup:
+  ssh -f -N -L 15433:172.20.0.2:5432 gendesign   # tradein-postgres tunnel
+  cd tradein-mvp/backend
+  export DATABASE_URL='postgresql+psycopg://tradein:<PASS>@localhost:15433/tradein'
+
+Usage:
+  uv run python ../scripts/local-sweep-ekb.py                     # defaults
+  uv run python ../scripts/local-sweep-ekb.py --pages 3 --detail-top 20 --delay 30
+  uv run python ../scripts/local-sweep-ekb.py --skip-search       # только detail + houses
+  uv run python ../scripts/local-sweep-ekb.py --skip-houses       # без houses
+"""
+from __future__ import annotations
+
+# stdlib
+import argparse
+import asyncio
+import sys
+import time
+import types
+from pathlib import Path
+from urllib.parse import urlparse
+
+# Auto-inject backend/ в sys.path чтобы найти app.* package
+# (скрипт лежит в tradein-mvp/scripts/, app — в tradein-mvp/backend/)
+_BACKEND_DIR = Path(__file__).resolve().parent.parent / "backend"
+if str(_BACKEND_DIR) not in sys.path:
+    sys.path.insert(0, str(_BACKEND_DIR))
+
+# Stub weasyprint (Windows без GTK ломает import chain через trade_in.py)
+for _m in (
+    "weasyprint",
+    "weasyprint.text",
+    "weasyprint.text.ffi",
+    "weasyprint.css",
+    "weasyprint.css.targets",
+):
+    sys.modules[_m] = types.ModuleType(_m)
+sys.modules["weasyprint"].HTML = type("HTML", (), {"__init__": lambda self, *a, **kw: None})
+sys.modules["weasyprint"].CSS = type("CSS", (), {"__init__": lambda self, *a, **kw: None})
+
+# app imports после stubs
+from sqlalchemy import text  # noqa: E402
+
+from app.core.db import SessionLocal  # noqa: E402
+from app.services.scrapers.avito import AvitoScraper  # noqa: E402
+
+
+class CityWideAvitoScraper(AvitoScraper):
+    """Subclass без geo-фильтра — search всего города (ЕКБ) через pagination.
+
+    Override `_build_web_url` чтобы НЕ добавлять `geoCoords`/`radius` — Avito
+    отдаёт ВСЕ объявления в городе sorted by date, до 100 pages (cap ~5000).
+    """
+
+    def _build_web_url(self, lat: float, lon: float, radius_km: int, page: int = 1) -> str:  # noqa: ARG002
+        from urllib.parse import urlencode
+        params = {"s": 104, "p": page}  # s=104 sort by date
+        return f"{self.base_url}/ekaterinburg/kvartiry/prodam-ASgBAgICAUSSA8YQ?{urlencode(params)}"
+
+
+# Avito URL slugs для room-filter (extracted из real search page HTML 2026-05-23)
+ROOM_SLUGS: list[tuple[str, str]] = [
+    ("студии",       "studii-ASgBAgICAkSSA8YQygj~WA"),
+    ("1-комн.",      "1-komnatnye-ASgBAgICAkSSA8YQygiAWQ"),
+    ("2-комн.",      "2-komnatnye-ASgBAgICAkSSA8YQygiCWQ"),
+    ("3-комн.",      "3-komnatnye-ASgBAgICAkSSA8YQygiEWQ"),
+    ("4-комн.",      "4-komnatnye-ASgBAgICAkSSA8YQygiGWQ"),
+    ("5+комн.",      "5-komnatnye-ASgBAgICAkSSA8YQygiIWQ"),
+    ("своб.планир.", "svobodnaya_planirovka-ASgBAgICAkSSA8YQygj8zzI"),
+]
+
+
+class CityRoomsAvitoScraper(AvitoScraper):
+    """Subclass с URL slug filter по комнатности — для splitting когда total > 5000.
+
+    URL pattern: /ekaterinburg/kvartiry/prodam/<slug>?s=104&p=N
+    """
+
+    def __init__(self, slug: str) -> None:
+        super().__init__()
+        self.room_slug = slug
+
+    def _build_web_url(self, lat: float, lon: float, radius_km: int, page: int = 1) -> str:  # noqa: ARG002
+        from urllib.parse import urlencode
+        params = {"s": 104, "p": page}
+        return f"{self.base_url}/ekaterinburg/kvartiry/prodam/{self.room_slug}?{urlencode(params)}"
+from app.services.scrapers.avito_detail import (  # noqa: E402
+    fetch_detail,
+    save_detail_enrichment,
+)
+from app.services.scrapers.avito_houses import (  # noqa: E402
+    fetch_house_catalog,
+    save_house_catalog_enrichment,
+)
+from app.services.scrapers.base import save_listings  # noqa: E402
+
+# 10 anchors покрывающих центральные/северные/южные/восточные/западные районы ЕКБ
+ANCHORS: list[tuple[str, float, float]] = [
+    ("Центр",         56.8400, 60.6050),
+    ("ЮЗ",            56.7950, 60.5300),
+    ("Уралмаш",       56.8970, 60.6100),
+    ("Академический", 56.7700, 60.5500),
+    ("Пионерский",    56.8650, 60.6200),
+    ("Эльмаш",        56.8567, 60.6700),
+    ("Втузгородок",   56.8400, 60.6650),
+    ("ВИЗ",           56.8500, 60.5650),
+    ("Юго-Запад",     56.7770, 60.5800),
+    ("Сортировка",    56.8950, 60.5600),
+]
+
+
+async def phase_citywide(pages: int, delay: float) -> int:
+    """City-wide pagination — ВСЕ объявления ЕКБ (no geo filter), per-page save.
+
+    Per-page save важно: visible прогресс + не теряем данные если crash mid-way.
+    """
+    print(
+        f"\n=== PHASE CITYWIDE: full ЕКБ pagination ({pages} pages, delay {delay}s) ===\n",
+        flush=True,
+    )
+    total = 0
+    cum_ins = 0
+    cum_upd = 0
+    async with CityWideAvitoScraper() as s:
+        for page in range(1, pages + 1):
+            url = s._build_web_url(0.0, 0.0, 1, page=page)
+            try:
+                assert s._cffi is not None
+                response = await s._cffi.get(url)
+                if response.status_code != 200:
+                    print(f"  ⚠️ page={page} HTTP {response.status_code} — stop", flush=True)
+                    break
+                lots = s._parse_html(response.text, source_url_base=url)
+            except Exception as e:  # noqa: BLE001
+                print(f"  page={page} FAIL fetch: {type(e).__name__}: {e}", flush=True)
+                await asyncio.sleep(delay)
+                continue
+            if not lots:
+                print(f"  page={page}: 0 lots — end of pagination", flush=True)
+                break
+            db = SessionLocal()
+            try:
+                ins, upd = save_listings(db, lots)
+                cum_ins += ins
+                cum_upd += upd
+                total += len(lots)
+                print(
+                    f"  page={page}/{pages}: {len(lots)} lots → ins={ins} upd={upd} "
+                    f"[cum: total={total} new={cum_ins} upd={cum_upd}]",
+                    flush=True,
+                )
+            except Exception as e:  # noqa: BLE001
+                print(f"  page={page} FAIL save: {type(e).__name__}: {e}", flush=True)
+            finally:
+                db.close()
+            if page < pages:
+                await asyncio.sleep(delay)
+    print(f"  PHASE CITYWIDE TOTAL: fetched={total} ins={cum_ins} upd={cum_upd}", flush=True)
+    return total
+
+
+async def phase_byrooms(pages: int, delay: float) -> int:
+    """Rooms-split sweep — 7 categories × pages, per-page save.
+
+    Для total > 5000 (Avito search cap) split по комнатности позволяет получить
+    все listings (каждый segment <5000).
+    """
+    print(
+        f"\n=== PHASE BYROOMS: 7 категорий × {pages} pages, delay {delay}s ===\n",
+        flush=True,
+    )
+    grand_total = 0
+    grand_ins = 0
+    grand_upd = 0
+    import random
+    rate_limit_hits = 0
+    for ci, (name, slug) in enumerate(ROOM_SLUGS, 1):
+        print(f"\n--- [{ci}/{len(ROOM_SLUGS)}] Категория: {name} (slug={slug[:30]}...) ---", flush=True)
+        cat_total = 0
+        cat_ins = 0
+        cat_upd = 0
+        async with CityRoomsAvitoScraper(slug) as s:
+            for page in range(1, pages + 1):
+                url = s._build_web_url(0.0, 0.0, 1, page=page)
+                try:
+                    assert s._cffi is not None
+                    response = await s._cffi.get(url)
+                    if response.status_code == 429:
+                        # Exponential backoff на rate-limit: 60s × 2^N (max 30 min)
+                        rate_limit_hits += 1
+                        backoff = min(60 * (2 ** (rate_limit_hits - 1)), 1800)
+                        print(f"  {name} p={page} HTTP 429 (hit #{rate_limit_hits}) → backoff {backoff}s", flush=True)
+                        await asyncio.sleep(backoff)
+                        # retry same page after backoff
+                        try:
+                            response = await s._cffi.get(url)
+                            if response.status_code != 200:
+                                print(f"  {name} p={page} retry HTTP {response.status_code} — skip page", flush=True)
+                                await asyncio.sleep(delay * 2)
+                                continue
+                        except Exception as e:  # noqa: BLE001
+                            print(f"  {name} p={page} retry FAIL: {type(e).__name__}: {e}", flush=True)
+                            continue
+                    elif response.status_code != 200:
+                        print(f"  {name} p={page} HTTP {response.status_code} — skip page", flush=True)
+                        await asyncio.sleep(delay)
+                        continue
+                    lots = s._parse_html(response.text, source_url_base=url)
+                except Exception as e:  # noqa: BLE001
+                    print(f"  {name} p={page} FAIL fetch: {type(e).__name__}: {e}", flush=True)
+                    await asyncio.sleep(delay)
+                    continue
+                if not lots:
+                    print(f"  {name} page={page}: 0 lots — end of category", flush=True)
+                    break
+                db = SessionLocal()
+                try:
+                    ins, upd = save_listings(db, lots)
+                    cat_ins += ins
+                    cat_upd += upd
+                    cat_total += len(lots)
+                    print(
+                        f"  {name} p={page}/{pages}: {len(lots)} lots → ins={ins} upd={upd} "
+                        f"[cat: tot={cat_total} new={cat_ins}]",
+                        flush=True,
+                    )
+                except Exception as e:  # noqa: BLE001
+                    print(f"  {name} page={page} FAIL save: {type(e).__name__}: {e}", flush=True)
+                finally:
+                    db.close()
+                if page < pages:
+                    # Jitter: ±30% от delay чтобы выглядеть менее robotic
+                    jittered = delay * random.uniform(0.7, 1.3)
+                    await asyncio.sleep(jittered)
+        print(
+            f"  --- {name} TOTAL: fetched={cat_total} ins={cat_ins} upd={cat_upd} ---",
+            flush=True,
+        )
+        grand_total += cat_total
+        grand_ins += cat_ins
+        grand_upd += cat_upd
+    print(
+        f"\n  PHASE BYROOMS GRAND TOTAL: fetched={grand_total} new={grand_ins} updated={grand_upd}",
+        flush=True,
+    )
+    return grand_total
+
+
+async def phase_a_search(pages: int, delay: float) -> int:
+    """Search всех anchors × pages, save listings."""
+    print(
+        f"\n=== PHASE A: search ({len(ANCHORS)} anchors × {pages} pages, "
+        f"delay {delay}s) ===\n",
+        flush=True,
+    )
+    total = 0
+    for i, (name, lat, lon) in enumerate(ANCHORS, 1):
+        print(f"[{i}/{len(ANCHORS)}] {name} ({lat},{lon})", flush=True)
+        try:
+            async with AvitoScraper() as s:
+                lots = await s.fetch_around(
+                    lat, lon, 1500, pages=pages, delay_override_sec=delay
+                )
+        except Exception as e:  # noqa: BLE001
+            print(f"  FAIL fetch: {type(e).__name__}: {e}", flush=True)
+            await asyncio.sleep(delay)
+            continue
+        print(f"  fetched {len(lots)} lots", flush=True)
+        if not lots:
+            print("  ⚠️ 0 lots — possible 429, stop phase A", flush=True)
+            break
+        db = SessionLocal()
+        try:
+            ins, upd = save_listings(db, lots)
+            print(f"  saved: ins={ins} upd={upd}", flush=True)
+            total += len(lots)
+        except Exception as e:  # noqa: BLE001
+            print(f"  FAIL save: {type(e).__name__}: {e}", flush=True)
+        finally:
+            db.close()
+        if i < len(ANCHORS):
+            await asyncio.sleep(delay)
+    return total
+
+
+async def phase_b_detail(top_n: int, delay: float) -> tuple[int, set[str]]:
+    """Detail enrichment для top-N recent listings без detail.
+
+    Returns: (enriched_count, set of unique house_catalog_url paths extracted)
+    """
+    print(
+        f"\n=== PHASE B: detail enrichment (top {top_n}, delay {delay}s) ===\n",
+        flush=True,
+    )
+    db = SessionLocal()
+    try:
+        rows = db.execute(
+            text(
+                """
+                SELECT source_url, source_id FROM listings
+                WHERE source='avito' AND detail_enriched_at IS NULL
+                  AND source_url IS NOT NULL AND price_rub > 0
+                ORDER BY scraped_at DESC NULLS LAST
+                LIMIT :limit
+                """
+            ),
+            {"limit": top_n},
+        ).mappings().all()
+    finally:
+        db.close()
+
+    print(f"  queue: {len(rows)} listings to enrich", flush=True)
+    enriched = 0
+    failed = 0
+    house_urls: set[str] = set()
+    for i, row in enumerate(rows, 1):
+        url = row["source_url"]
+        path = urlparse(url).path if url.startswith("http") else url
+        print(f"[{i}/{len(rows)}] detail {row['source_id']}", flush=True)
+        try:
+            enr = await fetch_detail(path)
+            db = SessionLocal()
+            try:
+                ok = save_detail_enrichment(db, enr)
+                print(
+                    f"  enriched: ok={ok} "
+                    f"rooms={enr.rooms} area={enr.area_m2} "
+                    f"kitchen={enr.kitchen_area_m2} owners={enr.owners_count} "
+                    f"house_url={enr.house_catalog_url}",
+                    flush=True,
+                )
+                if ok:
+                    enriched += 1
+                # Извлекаем house path (strip query string + filter только /catalog/houses/)
+                if enr.house_catalog_url:
+                    p = urlparse(enr.house_catalog_url).path
+                    if p.startswith("/catalog/houses/"):
+                        house_urls.add(p)
+            finally:
+                db.close()
+        except Exception as e:  # noqa: BLE001
+            print(f"  FAIL: {type(e).__name__}: {e}", flush=True)
+            failed += 1
+        if i < len(rows):
+            await asyncio.sleep(delay)
+    print(f"  total: enriched={enriched} failed={failed} unique_houses_extracted={len(house_urls)}", flush=True)
+    return enriched, house_urls
+
+
+async def phase_c_houses(delay: float, extra_urls: set[str] | None = None) -> int:
+    """Houses enrichment — unique house_url из listings (DB) + extra_urls (из Phase B in-memory).
+
+    save_detail_enrichment не пишет house_url в listings (баг Stage 2b),
+    поэтому Phase C принимает extra_urls собранные Phase B вручную.
+    """
+    print("\n=== PHASE C: houses enrichment ===\n", flush=True)
+    db = SessionLocal()
+    try:
+        rows = db.execute(
+            text(
+                """
+                SELECT DISTINCT house_url FROM listings
+                WHERE source='avito' AND house_url IS NOT NULL
+                  AND house_id_fk IS NULL
+                ORDER BY house_url
+                """
+            )
+        ).fetchall()
+    finally:
+        db.close()
+
+    urls_from_db = {r[0] for r in rows if r[0]}
+    urls = urls_from_db | (extra_urls or set())
+    print(f"  queue: {len(urls)} unique houses (db={len(urls_from_db)} +extras={len(extra_urls or set())})", flush=True)
+    if not urls:
+        print("  no houses to enrich (skip)", flush=True)
+        return 0
+
+    enriched = 0
+    failed = 0
+    for i, url in enumerate(sorted(urls), 1):
+        path = urlparse(url).path if url.startswith("http") else url
+        print(f"[{i}/{len(urls)}] {path}", flush=True)
+        try:
+            enr = await fetch_house_catalog(path)
+            db = SessionLocal()
+            try:
+                counters = save_house_catalog_enrichment(db, enr)
+                print(f"  saved: {counters}", flush=True)
+                enriched += 1
+            finally:
+                db.close()
+        except Exception as e:  # noqa: BLE001
+            print(f"  FAIL: {type(e).__name__}: {e}", flush=True)
+            failed += 1
+        if i < len(urls):
+            await asyncio.sleep(delay)
+    print(f"  total: enriched={enriched} failed={failed}", flush=True)
+    return enriched
+
+
+async def main() -> None:
+    p = argparse.ArgumentParser()
+    p.add_argument(
+        "--mode",
+        choices=["anchors", "citywide", "byrooms", "both"],
+        default="anchors",
+        help="anchors=10 anchors; citywide=full pagination без geo; byrooms=7 категорий комнат (для total>5000); both=anchors+citywide",
+    )
+    p.add_argument("--pages", type=int, default=5, help="pages per anchor (anchors mode) или total pages (citywide)")
+    p.add_argument("--detail-top", type=int, default=50, help="top-N для detail enrichment")
+    p.add_argument("--delay", type=float, default=30, help="sec между HTTP requests")
+    p.add_argument("--skip-search", action="store_true")
+    p.add_argument("--skip-detail", action="store_true")
+    p.add_argument("--skip-houses", action="store_true")
+    args = p.parse_args()
+
+    t0 = time.perf_counter()
+    if not args.skip_search:
+        if args.mode in ("anchors", "both"):
+            await phase_a_search(args.pages, args.delay)
+        if args.mode in ("citywide", "both"):
+            cw_pages = args.pages if args.mode == "citywide" else 100
+            await phase_citywide(cw_pages, args.delay)
+        if args.mode == "byrooms":
+            await phase_byrooms(args.pages, args.delay)
+    extracted_house_urls: set[str] = set()
+    if not args.skip_detail:
+        _, extracted_house_urls = await phase_b_detail(args.detail_top, args.delay)
+    if not args.skip_houses:
+        await phase_c_houses(args.delay, extra_urls=extracted_house_urls)
+    elapsed = time.perf_counter() - t0
+    print(f"\n=== ALL DONE in {elapsed / 60:.1f} min ===", flush=True)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

tradein-mvp/scripts/probe-avito-1anchor.py

@@ -0,0 +1,103 @@
+"""Avito probe: 1 anchor × 1 page с настраиваемой задержкой.
+
+Минимальный dry-run чтобы проверить bans / detection после anti-bot
+hardening (PR #487) + global delay setting (PR #485).
+
+Setup:
+  ssh -f -N -L 15433:172.20.0.2:5432 gendesign
+  cd tradein-mvp/backend
+  $env:DATABASE_URL = 'postgresql+psycopg://tradein:<PASS>@localhost:15433/tradein'
+
+Usage:
+  uv run python ../scripts/probe-avito-1anchor.py
+  uv run python ../scripts/probe-avito-1anchor.py --delay 5 --radius 1000
+"""
+from __future__ import annotations
+
+import argparse
+import asyncio
+import sys
+import time
+import types
+from pathlib import Path
+
+_BACKEND_DIR = Path(__file__).resolve().parent.parent / "backend"
+if str(_BACKEND_DIR) not in sys.path:
+    sys.path.insert(0, str(_BACKEND_DIR))
+
+for _m in (
+    "weasyprint",
+    "weasyprint.text",
+    "weasyprint.text.ffi",
+    "weasyprint.css",
+    "weasyprint.css.targets",
+):
+    sys.modules[_m] = types.ModuleType(_m)
+sys.modules["weasyprint"].HTML = type("HTML", (), {"__init__": lambda self, *a, **kw: None})
+sys.modules["weasyprint"].CSS = type("CSS", (), {"__init__": lambda self, *a, **kw: None})
+
+from app.core.db import SessionLocal  # noqa: E402
+from app.services.scrapers.avito import AvitoScraper  # noqa: E402
+from app.services.scrapers.base import save_listings  # noqa: E402
+
+ANCHOR_NAME = "Центр ЕКБ"
+ANCHOR_LAT = 56.8400
+ANCHOR_LON = 60.6050
+
+
+async def main() -> None:
+    p = argparse.ArgumentParser()
+    p.add_argument("--delay", type=float, default=3.0, help="sec между HTTP requests")
+    p.add_argument("--radius", type=int, default=1500, help="метры")
+    p.add_argument("--pages", type=int, default=1)
+    args = p.parse_args()
+
+    print(
+        f"=== Avito probe ===\n"
+        f"  anchor:  {ANCHOR_NAME} ({ANCHOR_LAT}, {ANCHOR_LON})\n"
+        f"  radius:  {args.radius}m\n"
+        f"  pages:   {args.pages}\n"
+        f"  delay:   {args.delay}s\n",
+        flush=True,
+    )
+
+    t0 = time.perf_counter()
+    try:
+        async with AvitoScraper() as s:
+            lots = await s.fetch_around(
+                ANCHOR_LAT, ANCHOR_LON, args.radius,
+                pages=args.pages, delay_override_sec=args.delay,
+            )
+    except Exception as e:  # noqa: BLE001
+        print(f"FETCH FAIL: {type(e).__name__}: {e}", flush=True)
+        return
+
+    fetched = len(lots)
+    print(f"fetched: {fetched} lots in {time.perf_counter() - t0:.1f}s", flush=True)
+    if fetched == 0:
+        print("⚠️  0 lots — вероятно 429/ban. Смотри backend logs.", flush=True)
+        return
+
+    db = SessionLocal()
+    try:
+        ins, upd = save_listings(db, lots)
+        print(f"saved: ins={ins} upd={upd}", flush=True)
+    except Exception as e:  # noqa: BLE001
+        print(f"SAVE FAIL: {type(e).__name__}: {e}", flush=True)
+    finally:
+        db.close()
+
+    sample = lots[:3]
+    print("\nsample listings:", flush=True)
+    for lot in sample:
+        addr = (lot.address or "")[:60]
+        print(
+            f"  source_id={lot.source_id} price={lot.price_rub}₽ "
+            f"rooms={lot.rooms} area={lot.area_m2}м² floor={lot.floor}/{lot.total_floors} "
+            f"addr={addr!r}",
+            flush=True,
+        )
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

tradein-mvp/scripts/probe-db-counts.py

@@ -0,0 +1,58 @@
+"""Quick DB diagnostics: Avito listings count + recent insert rate."""
+from __future__ import annotations
+
+import os
+import psycopg
+
+dsn = os.environ["DATABASE_URL"].replace("postgresql+psycopg://", "postgresql://")
+conn = psycopg.connect(dsn)
+cur = conn.cursor()
+
+cur.execute("SELECT count(*) FROM listings WHERE source = 'avito'")
+print("avito total:", cur.fetchone()[0])
+
+cur.execute(
+    "SELECT count(*) FROM listings "
+    "WHERE source='avito' AND scraped_at > NOW() - interval '20 minutes'"
+)
+print("avito scraped last 20min:", cur.fetchone()[0])
+
+cur.execute(
+    """
+    SELECT column_name FROM information_schema.columns
+    WHERE table_name='listings' AND column_name IN
+      ('created_at','first_seen_at','first_seen','inserted_at')
+    """
+)
+cols = [r[0] for r in cur.fetchall()]
+print(f"timestamp cols available: {cols}")
+if "first_seen_at" in cols:
+    cur.execute(
+        "SELECT count(*) FROM listings "
+        "WHERE source='avito' AND first_seen_at > NOW() - interval '20 minutes'"
+    )
+    print("avito first_seen last 20min (new):", cur.fetchone()[0])
+
+cur.execute(
+    """
+    SELECT date_trunc('minute', scraped_at) AS m, count(*)
+    FROM listings
+    WHERE source='avito' AND scraped_at > NOW() - interval '30 minutes'
+    GROUP BY 1 ORDER BY 1 DESC
+    """
+)
+print("\nper-minute scraped (last 30min):")
+for row in cur.fetchall():
+    print(f"  {row[0]}  {row[1]}")
+
+cur.execute(
+    """
+    SELECT rooms, count(*)
+    FROM listings
+    WHERE source='avito' AND scraped_at > NOW() - interval '30 minutes'
+    GROUP BY 1 ORDER BY 1
+    """
+)
+print("\nby rooms (last 30min):")
+for row in cur.fetchall():
+    print(f"  rooms={row[0]}  count={row[1]}")

tradein-mvp/scripts/probe-db-leak-full.py

@@ -0,0 +1,129 @@
+"""Полная проверка региональной утечки в БД Avito listings.
+
+Угол 1: source_url slug (canonical, надёжный)
+Угол 2: address с маркером области (квартиры в ЕКБ не содержат "обл.")
+Угол 3: address contains имя пригорода/соседнего города
+Угол 4: top самых частых первых слов address (sanity check)
+"""
+from __future__ import annotations
+
+import os
+from collections import Counter
+
+import psycopg
+
+dsn = os.environ["DATABASE_URL"].replace("postgresql+psycopg://", "postgresql://")
+conn = psycopg.connect(dsn)
+cur = conn.cursor()
+
+cur.execute("SELECT count(*) FROM listings WHERE source='avito'")
+total = cur.fetchone()[0]
+print(f"=== Avito total: {total} ===\n")
+
+# Угол 1: URL slug (canonical)
+print("[1] URL slug check:")
+cur.execute(
+    """
+    SELECT
+        count(*) FILTER (WHERE source_url ~ 'avito\\.ru/ekaterinburg/') AS ekb,
+        count(*) FILTER (WHERE source_url IS NOT NULL AND source_url !~ 'avito\\.ru/ekaterinburg/') AS not_ekb,
+        count(*) FILTER (WHERE source_url IS NULL) AS no_url
+    FROM listings WHERE source='avito'
+    """
+)
+ekb, not_ekb, no_url = cur.fetchone()
+print(f"  ekaterinburg URL:    {ekb}")
+print(f"  NOT ekaterinburg:    {not_ekb}  {'← LEAK' if not_ekb else '✓ clean'}")
+print(f"  NULL url:            {no_url}")
+
+# Угол 2: address с "область" / "обл."
+print("\n[2] Address contains region marker (область/обл.):")
+cur.execute(
+    """
+    SELECT count(*) FROM listings
+    WHERE source='avito'
+      AND address IS NOT NULL
+      AND (address ILIKE '%область%' OR address ~* '\\bобл\\b' OR address ~* '\\bобл\\.')
+    """
+)
+n = cur.fetchone()[0]
+print(f"  count: {n}  {'← suspicious' if n > 5 else '✓ ok'}")
+if n > 0:
+    cur.execute(
+        """
+        SELECT source_url, address FROM listings
+        WHERE source='avito' AND address IS NOT NULL
+          AND (address ILIKE '%область%' OR address ~* '\\bобл\\b' OR address ~* '\\bобл\\.')
+        LIMIT 10
+        """
+    )
+    for url, addr in cur.fetchall():
+        slug = url.split('avito.ru/')[1].split('/')[0] if url and 'avito.ru/' in url else '?'
+        print(f"  [{slug}] {addr[:90]!r}")
+
+# Угол 3: address contains имя другого города/пригорода
+print("\n[3] Address contains satellite-city marker:")
+satellites = [
+    "Верхняя Пышма", "Арамиль", "Сысерть", "Дегтярск", "Первоуральск",
+    "Заречный", "Невьянск", "Нижний Тагил", "Кыштым", "Челябинск",
+    "Алапаевск", "Краснофимск", "Озёрск", "Озерск", "Снежинск", "Касли",
+    "Рефтинский", "Свободный", "Большой Исток", "Пышма", "Покровское",
+    "Реж", "Лесной", "Полевской", "Берёзовск", "Березовск", "Михайловск",
+    "Туринск", "Североуральск", "Тавда", "Богданович", "Артёмовский",
+    "Артемовский", "Качканар", "Ивдель", "Серов", "Ирбит", "Новоуральск",
+    "Камышлов", "Сухой Лог",
+]
+for city in satellites:
+    cur.execute(
+        "SELECT count(*) FROM listings WHERE source='avito' AND address ILIKE %s",
+        (f"%{city}%",),
+    )
+    n = cur.fetchone()[0]
+    if n > 0:
+        print(f"  {city}: {n}")
+
+cur.execute(
+    """
+    SELECT count(*) FROM listings WHERE source='avito' AND (
+        address ILIKE '%Верхняя Пышма%' OR address ILIKE '%Арамиль%' OR
+        address ILIKE '%Сысерть%' OR address ILIKE '%Дегтярск%' OR
+        address ILIKE '%Первоуральск%' OR address ILIKE '%Заречный%' OR
+        address ILIKE '%Невьянск%' OR address ILIKE '%Нижний Тагил%' OR
+        address ILIKE '%Кыштым%' OR address ILIKE '%Челябинск%' OR
+        address ILIKE '%Алапаевск%' OR address ILIKE '%Краснофимск%' OR
+        address ILIKE '%Озёрск%' OR address ILIKE '%Озерск%' OR
+        address ILIKE '%Снежинск%' OR address ILIKE '%Касли%' OR
+        address ILIKE '%Рефтинский%' OR address ILIKE '%Большой Исток%' OR
+        address ILIKE '%Реж%' OR address ILIKE '%Лесной%' OR address ILIKE '%Полевской%' OR
+        address ILIKE '%Берёзовск%' OR address ILIKE '%Березовск%'
+    )
+    """
+)
+sat_total = cur.fetchone()[0]
+print(f"  TOTAL satellite-city leaks: {sat_total}")
+
+# Угол 4: top first-words of address
+print("\n[4] Top 20 first-words in address (sanity check):")
+cur.execute(
+    """
+    SELECT split_part(trim(address), ' ', 1) AS first_word, count(*) AS n
+    FROM listings WHERE source='avito' AND COALESCE(address, '') != ''
+    GROUP BY 1 ORDER BY 2 DESC LIMIT 20
+    """
+)
+for word, n in cur.fetchall():
+    print(f"  {n:>5}  {word!r}")
+
+# Угол 5: address NULL — сколько неopределённых
+print("\n[5] Coverage:")
+cur.execute(
+    """
+    SELECT
+        count(*) FILTER (WHERE COALESCE(address, '') = '') AS no_addr,
+        count(*) FILTER (WHERE COALESCE(address, '') != '') AS has_addr
+    FROM listings WHERE source='avito'
+    """
+)
+no_a, has_a = cur.fetchone()
+print(f"  has address: {has_a}")
+print(f"  no address:  {no_a}")

tradein-mvp/scripts/probe-db-region-leak.py

@@ -0,0 +1,75 @@
+"""Проверка: есть ли в DB Avito листинги из других городов (region leak)."""
+from __future__ import annotations
+
+import os
+import psycopg
+
+dsn = os.environ["DATABASE_URL"].replace("postgresql+psycopg://", "postgresql://")
+conn = psycopg.connect(dsn)
+cur = conn.cursor()
+
+cur.execute("SELECT count(*) FROM listings WHERE source='avito'")
+total = cur.fetchone()[0]
+print(f"avito total: {total}")
+
+# Все с lat/lon — проверим bounding box ЕКБ (56.7..57.0, 60.4..60.8)
+cur.execute(
+    """
+    SELECT
+        count(*) FILTER (WHERE lat BETWEEN 56.5 AND 57.0 AND lon BETWEEN 60.4 AND 61.0) as in_ekb,
+        count(*) FILTER (WHERE lat IS NOT NULL AND (lat < 56.5 OR lat > 57.0 OR lon < 60.4 OR lon > 61.0)) as outside_ekb,
+        count(*) FILTER (WHERE lat IS NULL) as no_coords
+    FROM listings WHERE source='avito'
+    """
+)
+in_ekb, outside, no_coords = cur.fetchone()
+print(f"  in EKB bbox:        {in_ekb}")
+print(f"  outside EKB bbox:   {outside}  ← out-of-region leak!")
+print(f"  no coords yet:      {no_coords}")
+
+# Sample outside
+cur.execute(
+    """
+    SELECT source_id, lat, lon, address, price_rub, rooms
+    FROM listings
+    WHERE source='avito' AND lat IS NOT NULL
+      AND (lat < 56.5 OR lat > 57.0 OR lon < 60.4 OR lon > 61.0)
+    ORDER BY scraped_at DESC NULLS LAST
+    LIMIT 10
+    """
+)
+print("\nsample non-EKB listings:")
+for r in cur.fetchall():
+    sid, lat, lon, addr, price, rooms = r
+    print(f"  {sid} ({lat:.3f},{lon:.3f}) rooms={rooms} {price:,}₽ — {(addr or '')[:80]!r}")
+
+# Address-based leak (без lat/lon) — поиск явных маркеров других городов
+cur.execute(
+    """
+    SELECT count(*) FROM listings
+    WHERE source='avito'
+      AND lat IS NULL
+      AND address IS NOT NULL
+      AND (address ILIKE '%москв%' OR address ILIKE '%омск%' OR address ILIKE '%чебок%'
+        OR address ILIKE '%петерб%' OR address ILIKE '%казан%' OR address ILIKE '%новосиб%')
+    """
+)
+print(f"\naddress-based leak (no coords, other-city keyword): {cur.fetchone()[0]}")
+
+# По комнатам — где leak вероятнее всего
+cur.execute(
+    """
+    SELECT rooms,
+        count(*) AS total,
+        count(*) FILTER (WHERE lat IS NOT NULL AND (lat < 56.5 OR lat > 57.0 OR lon < 60.4 OR lon > 61.0)) AS leak
+    FROM listings WHERE source='avito'
+    GROUP BY rooms
+    ORDER BY rooms NULLS LAST
+    """
+)
+print("\nleak by rooms:")
+print(f"  {'rooms':<8} {'total':>8} {'leak':>8}")
+for r in cur.fetchall():
+    rooms, total, leak = r
+    pct = (leak * 100.0 / total) if total else 0
+    print(f"  {str(rooms):<8} {total:>8} {leak:>8}  ({pct:.0f}%)")

tradein-mvp/scripts/probe-db-url-leak.py

@@ -0,0 +1,51 @@
+"""URL-based region leak check для Avito listings."""
+from __future__ import annotations
+
+import os
+import re
+from collections import Counter
+
+import psycopg
+
+dsn = os.environ["DATABASE_URL"].replace("postgresql+psycopg://", "postgresql://")
+conn = psycopg.connect(dsn)
+cur = conn.cursor()
+
+cur.execute("SELECT source_url FROM listings WHERE source='avito'")
+slug_re = re.compile(r"avito\.ru/([a-z_]+)/")
+
+cities = Counter()
+no_match = 0
+for (url,) in cur.fetchall():
+    if not url:
+        no_match += 1
+        continue
+    m = slug_re.search(url)
+    if m:
+        cities[m.group(1)] += 1
+    else:
+        no_match += 1
+
+print("city slug distribution in source_url:")
+for city, n in cities.most_common():
+    flag = "  ← EKB" if city == "ekaterinburg" else ("  ← OTHER CITY" if city not in ("ekaterinburg",) else "")
+    print(f"  {city:<30} {n}{flag}")
+print(f"  (no slug parse): {no_match}")
+
+# Дальше — sample non-EKB
+cur.execute(
+    """
+    SELECT source_id, source_url, address, price_rub
+    FROM listings
+    WHERE source='avito'
+      AND source_url IS NOT NULL
+      AND source_url !~ 'avito\\.ru/ekaterinburg/'
+    ORDER BY scraped_at DESC NULLS LAST
+    LIMIT 10
+    """
+)
+print("\nsample non-EKB listings (by URL):")
+for sid, url, addr, price in cur.fetchall():
+    print(f"  {sid} {price:,}₽")
+    print(f"    URL: {url}")
+    print(f"    addr: {(addr or '')[:80]!r}")

tradein-mvp/scripts/probe-migration-028.py

@@ -0,0 +1,74 @@
+"""Проверка состояния миграции 028_matching_tables.sql на прод tradein-postgres."""
+from __future__ import annotations
+
+import os
+import psycopg
+
+dsn = os.environ["DATABASE_URL"].replace("postgresql+psycopg://", "postgresql://")
+conn = psycopg.connect(dsn)
+cur = conn.cursor()
+
+EXPECTED_COLUMNS = [
+    ("houses", "address_fingerprint"),
+    ("listings", "canonical"),
+    ("listings", "merged_into"),
+]
+EXPECTED_INDEXES = [
+    "houses_addr_fp_idx",
+    "listings_merged_into_idx",
+    "house_sources_house_idx",
+    "house_sources_low_conf_idx",
+    "listing_sources_listing_idx",
+    "listing_sources_low_conf_idx",
+    "listing_sources_scraped_idx",
+    "haa_house_idx",
+    "haa_fingerprint_idx",
+]
+EXPECTED_TABLES = ["house_sources", "listing_sources", "house_address_aliases"]
+
+print("=== Columns ===")
+for tbl, col in EXPECTED_COLUMNS:
+    cur.execute(
+        "SELECT 1 FROM information_schema.columns WHERE table_name=%s AND column_name=%s",
+        (tbl, col),
+    )
+    status = "✓" if cur.fetchone() else "✗ MISSING"
+    print(f"  {tbl}.{col:<25} {status}")
+
+print("\n=== Tables ===")
+for tbl in EXPECTED_TABLES:
+    cur.execute(
+        "SELECT 1 FROM information_schema.tables WHERE table_name=%s AND table_schema='public'",
+        (tbl,),
+    )
+    status = "✓" if cur.fetchone() else "✗ MISSING"
+    print(f"  {tbl:<30} {status}")
+
+print("\n=== Indexes ===")
+for idx in EXPECTED_INDEXES:
+    cur.execute(
+        "SELECT 1 FROM pg_indexes WHERE indexname=%s",
+        (idx,),
+    )
+    status = "✓" if cur.fetchone() else "✗ MISSING"
+    print(f"  {idx:<35} {status}")
+
+# Проверим есть ли активные транзакции/locks от deploy которые ещё висят
+print("\n=== Active transactions (locks on relevant tables) ===")
+cur.execute(
+    """
+    SELECT pid, usename, application_name, state, wait_event_type, wait_event,
+           query_start, substring(query, 1, 80) AS query_head
+    FROM pg_stat_activity
+    WHERE state IS DISTINCT FROM 'idle'
+      AND pid != pg_backend_pid()
+    ORDER BY query_start NULLS LAST
+    """
+)
+rows = cur.fetchall()
+if not rows:
+    print("  (none — clean)")
+else:
+    for r in rows:
+        print(f"  pid={r[0]} user={r[1]} app={r[2]} state={r[3]} wait={r[4]}/{r[5]} started={r[6]}")
+        print(f"    query: {r[7]!r}")

tradein-mvp/scripts/probe-migration-status.py

@@ -0,0 +1,90 @@
+"""Проверка какие миграции 028..054 НЕ применены."""
+from __future__ import annotations
+
+import os
+import psycopg
+
+dsn = os.environ["DATABASE_URL"].replace("postgresql+psycopg://", "postgresql://")
+conn = psycopg.connect(dsn)
+cur = conn.cursor()
+
+# Маркеры по миграциям (table/column/index/etc)
+CHECKS = [
+    ("028 matching_tables", "table", "house_sources"),
+    ("029 extend matching/valuation/dynamics", "column", ("listing_sources", "price_divergence_pct")),
+    ("030 avito_imv cache_key unique", "index_unique", ("avito_imv_evaluations", "cache_key")),
+    ("030 listings_alter_yandex", "column", ("listings", "yandex_id")),
+    ("031 houses_alter_yandex", "column", ("houses", "yandex_id")),
+    ("032 yandex_history", "table", "yandex_house_history"),
+    ("040 houses_extend", "column", ("houses", "year_renovated")),
+    ("041 house_sources noop", "noop", None),
+    ("042 listing_sources price_divergence_idx", "index", "listing_sources_price_divergence_idx"),
+    ("043 house_reviews_extend", "column", ("house_reviews", "review_text")),
+    ("044 external_valuations_link", "column", ("external_valuations", "link_url")),
+    ("045 house_placement_history_extend", "column", ("house_placement_history", "first_seen_at")),
+    ("046 views", "view", "v_house_canonical"),
+    ("050 search_optimization", "matview", "listings_search_mv"),
+    ("051 scrape_runs_extend", "column", ("scrape_runs", "heartbeat_at")),
+    ("052 scrape_schedules", "table", "scrape_schedules"),
+    ("053 scraper_settings", "table", "scraper_settings"),
+    ("054 scraper_settings_global", "row", ("scraper_settings", "source='global'")),
+]
+
+
+def check(kind: str, target) -> bool:
+    if kind == "noop":
+        return True
+    if kind == "table":
+        cur.execute(
+            "SELECT 1 FROM information_schema.tables WHERE table_schema='public' AND table_name=%s",
+            (target,),
+        )
+    elif kind == "view":
+        cur.execute(
+            "SELECT 1 FROM information_schema.views WHERE table_schema='public' AND table_name=%s",
+            (target,),
+        )
+    elif kind == "matview":
+        cur.execute("SELECT 1 FROM pg_matviews WHERE matviewname=%s", (target,))
+    elif kind == "column":
+        tbl, col = target
+        cur.execute(
+            "SELECT 1 FROM information_schema.columns WHERE table_name=%s AND column_name=%s",
+            (tbl, col),
+        )
+    elif kind == "index":
+        cur.execute("SELECT 1 FROM pg_indexes WHERE indexname=%s", (target,))
+    elif kind == "index_unique":
+        tbl, col = target
+        cur.execute(
+            """SELECT 1 FROM pg_indexes WHERE tablename=%s AND indexdef ~ %s""",
+            (tbl, f"UNIQUE.*{col}"),
+        )
+    elif kind == "row":
+        tbl, where = target
+        cur.execute(f"SELECT 1 FROM {tbl} WHERE {where}")
+    else:
+        return False
+    return cur.fetchone() is not None
+
+
+print(f"{'Migration':<48}  Status")
+print("-" * 60)
+missing = []
+for name, kind, target in CHECKS:
+    try:
+        ok = check(kind, target)
+    except Exception as e:  # noqa: BLE001
+        ok = False
+        print(f"{name:<48}  ✗ ERROR: {type(e).__name__}: {str(e)[:50]}")
+        conn.rollback()
+        missing.append(name)
+        continue
+    status = "✓" if ok else "✗ MISSING"
+    print(f"{name:<48}  {status}")
+    if not ok:
+        missing.append(name)
+
+print(f"\nMissing: {len(missing)}/{len(CHECKS)}")
+for m in missing:
+    print(f"  - {m}")

tradein-mvp/scripts/probe-parser-region-filter.py

@@ -0,0 +1,49 @@
+"""Прогнать current parser на HTML из Playwright debug-страницы (своб.планир p=10).
+
+Проверяет что URL-filter работает: до фикса parser возвращал ~30 карточек других
+городов, после — должен 0 или только ЕКБ.
+"""
+from __future__ import annotations
+
+import asyncio
+import sys
+import types
+from pathlib import Path
+
+_BACKEND = Path(__file__).resolve().parent.parent / "backend"
+if str(_BACKEND) not in sys.path:
+    sys.path.insert(0, str(_BACKEND))
+for _m in ("weasyprint", "weasyprint.text", "weasyprint.text.ffi", "weasyprint.css", "weasyprint.css.targets"):
+    sys.modules[_m] = types.ModuleType(_m)
+sys.modules["weasyprint"].HTML = type("HTML", (), {"__init__": lambda self, *a, **kw: None})
+sys.modules["weasyprint"].CSS = type("CSS", (), {"__init__": lambda self, *a, **kw: None})
+
+from app.services.scrapers.avito import AvitoScraper  # noqa: E402
+
+URL = "https://www.avito.ru/ekaterinburg/kvartiry/prodam/svobodnaya_planirovka-ASgBAgICAkSSA8YQygj8zzI?p=10&s=104"
+
+
+async def main() -> None:
+    async with AvitoScraper() as s:
+        assert s._cffi is not None
+        # Use scraper's own session — может попасть в 403 (curl_cffi), это OK,
+        # тогда переключимся на reading from Playwright MCP page.
+        print(f"Fetching {URL[:80]}...")
+        try:
+            r = await s._cffi.get(URL)
+            print(f"  HTTP {r.status_code}, html size: {len(r.text)}")
+            if r.status_code != 200:
+                print("  curl_cffi 403 — Avito не отдал. Пробни через MCP / browser save.")
+                return
+        except Exception as e:  # noqa: BLE001
+            print(f"  fetch fail: {type(e).__name__}: {e}")
+            return
+        lots = s._parse_html(r.text, source_url_base=URL)
+        print(f"\nParsed {len(lots)} EKB lots (после фильтра).")
+        for lot in lots[:5]:
+            print(f"  {lot.source_id} {lot.source_url[:80]}")
+            print(f"    {lot.address or '-'!r}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

tradein-mvp/scripts/test-playwright.py

@@ -0,0 +1,34 @@
+"""Minimal Playwright test — открывает Chrome окно с Avito на 30 sec."""
+import asyncio
+from playwright.async_api import async_playwright
+
+
+async def go():
+    async with async_playwright() as p:
+        print("launching browser...", flush=True)
+        browser = await p.chromium.launch(headless=False)
+        print("browser launched, opening page...", flush=True)
+        page = await browser.new_page()
+        try:
+            await page.goto(
+                "https://www.avito.ru/ekaterinburg/kvartiry/prodam-ASgBAgICAUSSA8YQ?s=104&p=1",
+                timeout=30000,
+            )
+        except Exception as e:
+            print(f"goto FAIL: {type(e).__name__}: {e}", flush=True)
+            await asyncio.sleep(15)
+            await browser.close()
+            return
+        print("page loaded", flush=True)
+        try:
+            title = await page.title()
+            print(f"title: {title}", flush=True)
+        except Exception as e:
+            print(f"title FAIL: {e}", flush=True)
+        print("окно должно быть видно ~30 сек", flush=True)
+        await asyncio.sleep(30)
+        await browser.close()
+
+
+if __name__ == "__main__":
+    asyncio.run(go())