From 70e45ce5e9d7919154b2acd796084209d0f9b00c Mon Sep 17 00:00:00 2001 From: lekss361 Date: Sun, 24 May 2026 22:49:20 +0300 Subject: [PATCH] feat(tradein-scripts): scrapers, probes, Phase C improvements MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New scripts: - avito-playwright-sweep.py — Playwright-based Avito sweep (bypasses Qrator) - backfill-yandex-addresses-ekb.py — fill street-only listings.address с house number - local-sweep-ekb-{cian,yandex,}.py — full ЕКБ sweep through residential IP - probe-* — quick diagnostics (region leak, migration status, parser filter, db counts) - inspect-{houses,search}-html.py — Avito DOM structure analysis - test-playwright.py — minimal Playwright smoke Phase C script updates (backfill-house-imv.py): - Pipe avito_imv module logs to avito-raw.log file (env AVITO_LOG_PATH override) - Clamp rooms=0 (studio) → 1 — Avito IMV API rejects rooms=0 with HTTP 400 --- tradein-mvp/scripts/avito-playwright-sweep.py | 371 +++++++++++++++ tradein-mvp/scripts/backfill-house-imv.py | 17 +- .../scripts/backfill-yandex-addresses-ekb.py | 173 +++++++ tradein-mvp/scripts/inspect-houses-html.py | 90 ++++ tradein-mvp/scripts/inspect-search-html.py | 55 +++ tradein-mvp/scripts/local-sweep-ekb-cian.py | 359 ++++++++++++++ tradein-mvp/scripts/local-sweep-ekb-yandex.py | 376 +++++++++++++++ tradein-mvp/scripts/local-sweep-ekb.py | 443 ++++++++++++++++++ tradein-mvp/scripts/probe-avito-1anchor.py | 103 ++++ tradein-mvp/scripts/probe-db-counts.py | 58 +++ tradein-mvp/scripts/probe-db-leak-full.py | 129 +++++ tradein-mvp/scripts/probe-db-region-leak.py | 75 +++ tradein-mvp/scripts/probe-db-url-leak.py | 51 ++ tradein-mvp/scripts/probe-migration-028.py | 74 +++ tradein-mvp/scripts/probe-migration-status.py | 90 ++++ .../scripts/probe-parser-region-filter.py | 49 ++ tradein-mvp/scripts/test-playwright.py | 34 ++ 17 files changed, 2546 insertions(+), 1 deletion(-) create mode 100644 tradein-mvp/scripts/avito-playwright-sweep.py create mode 100644 tradein-mvp/scripts/backfill-yandex-addresses-ekb.py create mode 100644 tradein-mvp/scripts/inspect-houses-html.py create mode 100644 tradein-mvp/scripts/inspect-search-html.py create mode 100644 tradein-mvp/scripts/local-sweep-ekb-cian.py create mode 100644 tradein-mvp/scripts/local-sweep-ekb-yandex.py create mode 100644 tradein-mvp/scripts/local-sweep-ekb.py create mode 100644 tradein-mvp/scripts/probe-avito-1anchor.py create mode 100644 tradein-mvp/scripts/probe-db-counts.py create mode 100644 tradein-mvp/scripts/probe-db-leak-full.py create mode 100644 tradein-mvp/scripts/probe-db-region-leak.py create mode 100644 tradein-mvp/scripts/probe-db-url-leak.py create mode 100644 tradein-mvp/scripts/probe-migration-028.py create mode 100644 tradein-mvp/scripts/probe-migration-status.py create mode 100644 tradein-mvp/scripts/probe-parser-region-filter.py create mode 100644 tradein-mvp/scripts/test-playwright.py diff --git a/tradein-mvp/scripts/avito-playwright-sweep.py b/tradein-mvp/scripts/avito-playwright-sweep.py new file mode 100644 index 00000000..66344d89 --- /dev/null +++ b/tradein-mvp/scripts/avito-playwright-sweep.py @@ -0,0 +1,371 @@ +"""Playwright-based Avito sweep — обходит Qrator JS challenge через real headless Chromium. + +Use case: curl_cffi заблочен Qrator anti-DDoS (429 + captcha challenge). Real +browser proходит JS challenge автоматически. + +Setup: + cd tradein-mvp/backend + uv pip install playwright + uv run playwright install chromium + +Usage: + export DATABASE_URL='postgresql+psycopg://tradein:@localhost:15433/tradein' + uv run --project tradein-mvp/backend python tradein-mvp/scripts/avito-playwright-sweep.py + # options: --pages N --delay S --mode {citywide|byrooms} --headless +""" +from __future__ import annotations + +# Banner FIRST — даже до imports чтобы видно что скрипт стартанул: +import sys as _sys + +print("=" * 60, flush=True) +print("AVITO PLAYWRIGHT SWEEP — starting...", flush=True) +print("=" * 60, flush=True) +_sys.stdout.flush() + +import argparse +import asyncio +import json +import os +import random +import sys +import time +import types +from pathlib import Path +from urllib.parse import urlencode + +# Force unbuffered output +os.environ["PYTHONUNBUFFERED"] = "1" + +_BACKEND = Path(__file__).resolve().parent.parent / "backend" +if str(_BACKEND) not in sys.path: + sys.path.insert(0, str(_BACKEND)) + +# WeasyPrint stub (Windows без GTK) +for _m in ("weasyprint", "weasyprint.text", "weasyprint.text.ffi", "weasyprint.css", "weasyprint.css.targets"): + sys.modules[_m] = types.ModuleType(_m) +sys.modules["weasyprint"].HTML = type("HTML", (), {"__init__": lambda self, *a, **kw: None}) +sys.modules["weasyprint"].CSS = type("CSS", (), {"__init__": lambda self, *a, **kw: None}) + +from playwright.async_api import Browser, BrowserContext, async_playwright # noqa: E402 + +try: + from playwright_stealth import Stealth # noqa: E402 + _STEALTH_AVAILABLE = True +except ImportError: + _STEALTH_AVAILABLE = False + print("[warn] playwright-stealth not installed — без stealth, может triggered больше CAPTCHA", flush=True) + +from app.core.db import SessionLocal # noqa: E402 +from app.services.scrapers.avito import AvitoScraper # noqa: E402 +from app.services.scrapers.base import save_listings # noqa: E402 + +COOKIES_FILE = Path(__file__).resolve().parent / ".avito-cookies.json" +AVITO_BASE = "https://www.avito.ru" +UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36" + +# Reuse AvitoScraper's parser (DOM cards extraction) +_parser_instance = AvitoScraper() +_parser_instance._anchor_lat = 0.0 # noqa: SLF001 +_parser_instance._anchor_lon = 0.0 # noqa: SLF001 +_parser_instance._anchor_radius_m = 1000 # noqa: SLF001 + + +def load_cookies() -> list[dict]: + """Load DevTools-exported cookies → Playwright format.""" + if not COOKIES_FILE.exists(): + print(f"WARN: cookies file not found at {COOKIES_FILE} — будем без cookies") + return [] + raw = json.loads(COOKIES_FILE.read_text(encoding="utf-8")) + out = [] + for c in raw: + ck = { + "name": c["name"], + "value": c["value"], + "domain": c["domain"], + "path": c.get("path", "/"), + "secure": c.get("secure", True), + "httpOnly": c.get("httpOnly", False), + } + if c.get("expirationDate"): + ck["expires"] = int(c["expirationDate"]) + ss = c.get("sameSite") + if ss == "no_restriction": + ck["sameSite"] = "None" + elif ss == "lax": + ck["sameSite"] = "Lax" + elif ss == "strict": + ck["sameSite"] = "Strict" + out.append(ck) + return out + + +def build_citywide_url(page: int) -> str: + return f"{AVITO_BASE}/ekaterinburg/kvartiry/prodam-ASgBAgICAUSSA8YQ?{urlencode({'s':104,'p':page})}" + + +ROOM_SLUGS: list[tuple[str, str]] = [ + ("студии", "studii-ASgBAgICAkSSA8YQygj~WA"), + ("1-комн.", "1-komnatnye-ASgBAgICAkSSA8YQygiAWQ"), + ("2-комн.", "2-komnatnye-ASgBAgICAkSSA8YQygiCWQ"), + ("3-комн.", "3-komnatnye-ASgBAgICAkSSA8YQygiEWQ"), + ("4-комн.", "4-komnatnye-ASgBAgICAkSSA8YQygiGWQ"), + ("5+комн.", "5-komnatnye-ASgBAgICAkSSA8YQygiIWQ"), + ("своб.планир.", "svobodnaya_planirovka-ASgBAgICAkSSA8YQygj8zzI"), +] + + +def build_rooms_url(slug: str, page: int) -> str: + return f"{AVITO_BASE}/ekaterinburg/kvartiry/prodam/{slug}?{urlencode({'s':104,'p':page})}" + + +async def fetch_page(context: BrowserContext, url: str) -> tuple[int, str | None]: + """Open page in Playwright, wait for content, return (status, html or None). + + Strategy: + 1. goto с wait_until='domcontentloaded' (быстро, не ждём всех scripts) + 2. Если status 200 → wait_for_selector data-marker=item (Qrator challenge должен resolve пока) + 3. Иначе если 429/403 — challenge не прошёл, дать ещё 5s + retry content (maybe challenge resolves async) + """ + page = await context.new_page() + try: + try: + response = await page.goto(url, wait_until="domcontentloaded", timeout=45000) + except Exception as e: # noqa: BLE001 + print(f" goto FAIL: {type(e).__name__}: {str(e)[:80]}") + return 0, None + status = response.status if response else 0 + if status in (429, 403): + # Qrator challenge — wait для JS auto-resolve, потом reload + await asyncio.sleep(5) + try: + response = await page.reload(wait_until="domcontentloaded", timeout=30000) + status = response.status if response else status + except Exception: # noqa: BLE001 + pass + if status != 200: + return status, None + # Wait for item cards + try: + await page.wait_for_selector('[data-marker="item"]', timeout=15000) + except Exception: # noqa: BLE001 + pass + html = await page.content() + return status, html + finally: + await page.close() + + +async def sweep_urls( + urls_with_label: list[tuple[str, str]], + delay: float, + headless: bool, +) -> int: + """Generic sweep: list of (label, url) → fetch + parse + save per URL.""" + cookies = load_cookies() + print(f"loaded {len(cookies)} cookies") + total_fetched = 0 + total_ins = 0 + total_upd = 0 + challenge_hits = 0 + print("=== Launching Playwright browser ===", flush=True) + async with async_playwright() as p: + print(f" headless={headless}", flush=True) + browser: Browser = await p.chromium.launch( + headless=headless, + args=[ + "--disable-blink-features=AutomationControlled", + "--no-sandbox", + ], + ) + print(" browser launched, creating context...", flush=True) + context: BrowserContext = await browser.new_context( + user_agent=UA, + locale="ru-RU", + timezone_id="Asia/Yekaterinburg", + viewport={"width": 1366, "height": 900}, + ) + # Stealth — optional, skip on error + if _STEALTH_AVAILABLE: + try: + stealth = Stealth() + await stealth.apply_stealth_async(context) + print(" stealth applied", flush=True) + except Exception as e: # noqa: BLE001 + print(f" stealth skipped (err): {type(e).__name__}: {e}", flush=True) + if cookies: + await context.add_cookies(cookies) + print(f" cookies added ({len(cookies)})", flush=True) + + # Initial warmup: open main Avito + wait для user pass CAPTCHA если нужно + print("\n=== WARMUP: opening Avito initial page ===", flush=True) + print(" creating new page tab...", flush=True) + warmup_page = await context.new_page() + print(" navigating to Avito (timeout 30s)...", flush=True) + try: + await warmup_page.goto( + "https://www.avito.ru/ekaterinburg/kvartiry/prodam-ASgBAgICAUSSA8YQ?s=104&p=1", + wait_until="domcontentloaded", + timeout=30000, + ) + print(" goto done, reading content...", flush=True) + except Exception as e: # noqa: BLE001 + print(f" goto FAIL: {type(e).__name__}: {e}", flush=True) + # Check if challenge page — reliable detector: NO listing cards в HTML. + # Слово "captcha" может быть в JS embed страницы normally (false positive). + warmup_html = await warmup_page.content() + print(f" content read: {len(warmup_html)} bytes", flush=True) + try: + await warmup_page.wait_for_selector('[data-marker="item"]', timeout=5000) + has_items = True + except Exception: # noqa: BLE001 + has_items = False + is_qrator = "qrator" in warmup_html.lower() and len(warmup_html) < 100000 + challenge_present = (not has_items) and (is_qrator or len(warmup_html) < 50000) + print(f" has_items={has_items} qrator={is_qrator} size={len(warmup_html)} challenge={challenge_present}", flush=True) + if challenge_present: + print("\n🚨 CAPTCHA detected на initial page!") + print("👉 Окно браузера открыто — пройди капчу руками,") + print("👉 затем нажми Enter здесь чтобы продолжить sweep...") + try: + input(">>> Press ENTER when ready: ") + except (EOFError, KeyboardInterrupt): + print("aborted") + await browser.close() + return 0 + # Verify challenge passed + await warmup_page.reload(wait_until="domcontentloaded", timeout=30000) + new_html = await warmup_page.content() + if any(m in new_html.lower() for m in ("captcha", "qrator")): + print("⚠️ всё ещё CAPTCHA — повторно пройди или Ctrl+C") + input(">>> Retry ENTER: ") + else: + print("✅ CAPTCHA пройдена, продолжаю sweep") + else: + print("✅ Initial page OK (no challenge)") + await warmup_page.close() + + try: + # Per-category skip flag: 0 lots на странице категории = end of pagination + # ДЛЯ ТЕКУЩЕЙ КАТЕГОРИИ, а не для всего sweep'а (был bug: break ломал всё). + current_cat: str | None = None + skip_until_next_cat = False + for i, (label, url) in enumerate(urls_with_label, 1): + # Категория = всё до " page=" в label ("1-комн. page=5/60" → "1-комн.") + cat = label.split(" page=")[0] if " page=" in label else label + if cat != current_cat: + current_cat = cat + skip_until_next_cat = False + challenge_hits = 0 # reset 429-counter на границе категорий + if skip_until_next_cat: + continue + + print(f"[{i}/{len(urls_with_label)}] {label}", flush=True) + status, html = await fetch_page(context, url) + if status != 200 or not html: + print(f" HTTP {status} → skip", flush=True) + if status in (429, 403): + challenge_hits += 1 + if challenge_hits >= 3: + print(f" ⚠️ {challenge_hits}x challenge hits — sleep 60s extra", flush=True) + print(" если CAPTCHA в окне — пройди и нажми ENTER:", flush=True) + try: + input(" >>> ") + except (EOFError, KeyboardInterrupt): + break + challenge_hits = 0 + await asyncio.sleep(delay) + continue + # Parse + save + lots = _parser_instance._parse_html(html, source_url_base=url) # noqa: SLF001 + if not lots: + print(f" 0 lots — end of pagination for {cat!r}", flush=True) + if " page=" in label: + # 0 lots на пагинированной странице = end of category, не sweep + skip_until_next_cat = True + continue + db = SessionLocal() + try: + ins, upd = save_listings(db, lots) + total_fetched += len(lots) + total_ins += ins + total_upd += upd + print( + f" {len(lots)} lots → ins={ins} upd={upd} " + f"[cum: tot={total_fetched} new={total_ins}]", + flush=True, + ) + except Exception as e: # noqa: BLE001 + print(f" save FAIL: {type(e).__name__}: {e}", flush=True) + finally: + db.close() + # Jittered delay + if i < len(urls_with_label): + j = delay * random.uniform(0.7, 1.3) + await asyncio.sleep(j) + finally: + await context.close() + await browser.close() + print( + f"\n=== TOTAL: fetched={total_fetched} ins={total_ins} upd={total_upd} ===", + flush=True, + ) + return total_fetched + + +async def main() -> None: + # Diagnostic — show env vars used: + db_url = os.environ.get("DATABASE_URL", "") + # Mask password for logging + if "://" in db_url and "@" in db_url: + before_at, after_at = db_url.rsplit("@", 1) + scheme_user = before_at.split("://", 1) + if len(scheme_user) == 2 and ":" in scheme_user[1]: + user = scheme_user[1].split(":", 1)[0] + masked = f"{scheme_user[0]}://{user}:***@{after_at}" + else: + masked = db_url[:50] + "..." + else: + masked = db_url + print(f"DATABASE_URL: {masked}", flush=True) + print(f"ENVIRONMENT: {os.environ.get('ENVIRONMENT', '')}", flush=True) + + p = argparse.ArgumentParser() + p.add_argument("--mode", choices=["citywide", "byrooms"], default="citywide") + p.add_argument("--pages", type=int, default=100, help="pages per query") + p.add_argument("--delay", type=float, default=7, help="sec между requests") + p.add_argument("--headless", action="store_true", default=True) + p.add_argument("--no-headless", dest="headless", action="store_false") + p.add_argument( + "--start-cat", + type=int, + default=1, + help="byrooms: с какой категории начать (1-based). 1=студии, 2=1-комн, " + "3=2-комн, 4=3-комн, 5=4-комн, 6=5+комн, 7=своб.планир.", + ) + args = p.parse_args() + + urls: list[tuple[str, str]] = [] + if args.mode == "citywide": + for page in range(1, args.pages + 1): + urls.append((f"citywide page={page}/{args.pages}", build_citywide_url(page))) + else: # byrooms + cats = ROOM_SLUGS[args.start_cat - 1:] + if not cats: + print(f"ERROR: --start-cat={args.start_cat} > {len(ROOM_SLUGS)} cats", flush=True) + return + print(f"byrooms: starting from cat #{args.start_cat} ({cats[0][0]!r}), " + f"{len(cats)} cats total", flush=True) + for cat_name, slug in cats: + for page in range(1, args.pages + 1): + urls.append((f"{cat_name} page={page}/{args.pages}", build_rooms_url(slug, page))) + + print(f"Mode: {args.mode}, pages: {args.pages}, delay: {args.delay}s, total URLs: {len(urls)}") + print(f"Estimated time: {len(urls) * args.delay / 60:.0f} min (без challenge)") + t0 = time.perf_counter() + await sweep_urls(urls, args.delay, args.headless) + print(f"\nDONE in {(time.perf_counter() - t0) / 60:.1f} min") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/tradein-mvp/scripts/backfill-house-imv.py b/tradein-mvp/scripts/backfill-house-imv.py index a9da11d3..83aabccd 100644 --- a/tradein-mvp/scripts/backfill-house-imv.py +++ b/tradein-mvp/scripts/backfill-house-imv.py @@ -51,6 +51,19 @@ logging.basicConfig( ) logger = logging.getLogger("phase-c") +# Avito raw log — pipe everything from app.services.scrapers.avito_imv to a file +# so we keep full request/response history for diagnosing contract changes. +_AVITO_LOG_PATH = os.environ.get("AVITO_LOG_PATH", "avito-raw.log") +_avito_fh = logging.FileHandler(_AVITO_LOG_PATH, encoding="utf-8") +_avito_fh.setFormatter(logging.Formatter("%(asctime)s %(levelname)s [%(name)s] %(message)s")) +_avito_fh.setLevel(logging.DEBUG) +_avito_logger = logging.getLogger("app.services.scrapers.avito_imv") +_avito_logger.addHandler(_avito_fh) +_avito_logger.setLevel(logging.DEBUG) +# Also pipe phase-c progress lines to the file +logger.addHandler(_avito_fh) +logger.info("avito raw log writing to %s", os.path.abspath(_AVITO_LOG_PATH)) + # Avito IMV API accepts only these house_type values (verified 2026-05-24). # Our `listings.house_type` may contain pre-2026 values like 'monolith' / 'monolith_brick'. _HOUSE_TYPE_TO_IMV = { @@ -147,8 +160,10 @@ def pick_lot_params(db: Session, house_id: int) -> dict: {"hid": house_id}, ).mappings().first() + # Avito IMV: rooms=0 (студия) rejected with HTTP 400. Clamp to 1. + rooms_raw = int(row["rooms"]) return { - "rooms": int(row["rooms"]), + "rooms": max(rooms_raw, 1), "area_m2": float(row["area_m2"]), "floor": int(row["floor"] or 1), "floor_at_home": int(row["total_floors"] or (house and house["total_floors"]) or 9), diff --git a/tradein-mvp/scripts/backfill-yandex-addresses-ekb.py b/tradein-mvp/scripts/backfill-yandex-addresses-ekb.py new file mode 100644 index 00000000..9c9e7db4 --- /dev/null +++ b/tradein-mvp/scripts/backfill-yandex-addresses-ekb.py @@ -0,0 +1,173 @@ +"""Backfill listings.address with full address (street + house number) for +active EKB yandex listings that currently have street-only addresses. + +Why: Yandex SERP returns only street (e.g. 'улица Горького') in card snippet. +Detail page has the full address ('Екатеринбург, улица Горького, 36'). +With house number we can call /otsenka-kvartiry-po-adresu-onlayn/ and get the +building's history. + +Usage: + cd tradein-mvp/backend + uv run python ../scripts/backfill-yandex-addresses-ekb.py --limit 5 # smoke + uv run python ../scripts/backfill-yandex-addresses-ekb.py # full + uv run python ../scripts/backfill-yandex-addresses-ekb.py --dry-run # no DB write + +Requires SSH tunnel + DATABASE_URL env (see sweep-yandex-valuation-ekb.py docstring). +""" + +from __future__ import annotations + +import argparse +import asyncio +import random +import re +import sys +import time +import types +from pathlib import Path + +# sys.path injection +_BACKEND_DIR = Path(__file__).resolve().parent.parent / "backend" +if str(_BACKEND_DIR) not in sys.path: + sys.path.insert(0, str(_BACKEND_DIR)) + +# Stub weasyprint (Windows GTK-less) +for _m in ( + "weasyprint", + "weasyprint.text", + "weasyprint.text.ffi", + "weasyprint.css", + "weasyprint.css.targets", +): + sys.modules[_m] = types.ModuleType(_m) +sys.modules["weasyprint"].HTML = type("HTML", (), {"__init__": lambda self, *a, **kw: None}) +sys.modules["weasyprint"].CSS = type("CSS", (), {"__init__": lambda self, *a, **kw: None}) + +from curl_cffi.requests import AsyncSession # noqa: E402 +from sqlalchemy import text # noqa: E402 + +from app.core.db import SessionLocal # noqa: E402 + +# Extract '<City>, <addr> — id <number>' from <title> +RE_TITLE_ADDRESS = re.compile( + r"<title>[^<]*?—\s*" + r"(?:[^—,]*?,\s*)?" # optional ЖК/жилой комплекс prefix + r"(Екатеринбург,\s*[^<]+?)" + r"\s*—\s*id\s*\d+\s*", + re.IGNORECASE, +) + + +def _eligible_listings(limit: int | None) -> list[dict]: + """Active EKB yandex listings without house number in address.""" + db = SessionLocal() + try: + sql = text( + """ + SELECT id, source_id, address, source_url + FROM listings + WHERE source = 'yandex' + AND region_code = 66 + AND is_active = TRUE + AND address IS NOT NULL + AND NOT (address ~ ',\\s*\\d+') + AND source_url IS NOT NULL + ORDER BY scraped_at DESC + """ + ) + rows = db.execute(sql).mappings().all() + out = [dict(r) for r in rows] + finally: + db.close() + if limit is not None and limit > 0: + out = out[:limit] + return out + + +def _update_listing_address(listing_id: int, new_address: str) -> None: + db = SessionLocal() + try: + db.execute( + text("UPDATE listings SET address = :addr WHERE id = :id"), + {"addr": new_address, "id": listing_id}, + ) + db.commit() + finally: + db.close() + + +async def _fetch_address(session: AsyncSession, url: str) -> str | None: + """Fetch detail page, extract full address from .""" + try: + r = await session.get(url, timeout=30) + except Exception as e: + print(f" fetch FAIL: {type(e).__name__}: {e}", flush=True) + return None + if r.status_code != 200: + print(f" HTTP {r.status_code}", flush=True) + return None + html = r.text.replace("\xa0", " ") + m = RE_TITLE_ADDRESS.search(html) + if not m: + return None + addr = m.group(1).strip().rstrip(",").strip() + return addr if addr else None + + +async def main() -> None: + p = argparse.ArgumentParser( + description="Backfill yandex listings.address with house number from detail page" + ) + p.add_argument("--limit", type=int, default=None, help="Cap items (smoke testing)") + p.add_argument("--delay", type=float, default=3.0, help="Sec between requests") + p.add_argument("--dry-run", action="store_true", help="Don't UPDATE DB") + args = p.parse_args() + + items = _eligible_listings(args.limit) + if not items: + print("No eligible listings.", flush=True) + return + + print( + f"=== Backfill {len(items)} listings (delay={args.delay}s dry_run={args.dry_run}) ===", + flush=True, + ) + + t0 = time.perf_counter() + enriched = 0 + skipped = 0 + failed = 0 + + async with AsyncSession(impersonate="chrome120") as session: + for i, item in enumerate(items, 1): + url = item["source_url"] + old_addr = item["address"] + print( + f"[{i}/{len(items)}] id={item['id']} old={old_addr!r}", + flush=True, + ) + new_addr = await _fetch_address(session, url) + if new_addr is None: + print(" -> no address extracted", flush=True) + failed += 1 + elif new_addr == old_addr: + print(" -> unchanged", flush=True) + skipped += 1 + else: + print(f" -> {new_addr!r}", flush=True) + if not args.dry_run: + _update_listing_address(item["id"], new_addr) + enriched += 1 + if i < len(items): + await asyncio.sleep(args.delay * random.uniform(0.85, 1.3)) + + elapsed = time.perf_counter() - t0 + print( + f"\n=== DONE in {elapsed / 60:.1f} min. " + f"enriched={enriched} skipped={skipped} failed={failed} ===", + flush=True, + ) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/tradein-mvp/scripts/inspect-houses-html.py b/tradein-mvp/scripts/inspect-houses-html.py new file mode 100644 index 00000000..a839e030 --- /dev/null +++ b/tradein-mvp/scripts/inspect-houses-html.py @@ -0,0 +1,90 @@ +"""One-off: fetch Avito houses HTML и анализ markers для понимания новой структуры. + +После 2026-05-23 Avito поменяли rendering — старый `window.__preloadedState__` +больше не работает (Stage 2c parser fails). Этот скрипт fetch'ит один houses +catalog page локально через residential IP и сохраняет HTML + проверяет markers +чтобы понять new format (__NEXT_DATA__? React SSR? иной). +""" +import sys +import types +from pathlib import Path + +_BACKEND = Path(__file__).resolve().parent.parent / "backend" +if str(_BACKEND) not in sys.path: + sys.path.insert(0, str(_BACKEND)) + +for _m in ("weasyprint", "weasyprint.text", "weasyprint.text.ffi", "weasyprint.css", "weasyprint.css.targets"): + sys.modules[_m] = types.ModuleType(_m) +sys.modules["weasyprint"].HTML = type("HTML", (), {"__init__": lambda self, *a, **kw: None}) +sys.modules["weasyprint"].CSS = type("CSS", (), {"__init__": lambda self, *a, **kw: None}) + +import asyncio # noqa: E402 + +from curl_cffi.requests import AsyncSession # noqa: E402 + +URLS = [ + "https://www.avito.ru/catalog/houses/ekaterinburg/ul_yumasheva_6/115188", + "https://www.avito.ru/catalog/houses/ekaterinburg/ul_akademika_postovskogo_17a/3171365", +] + + +async def main() -> None: + async with AsyncSession( + impersonate="chrome120", + timeout=25, + headers={ + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "ru-RU,ru;q=0.9", + "Sec-Fetch-Dest": "document", + "Sec-Fetch-Mode": "navigate", + "Sec-Fetch-Site": "none", + }, + ) as s: + for url in URLS: + print(f"\n=== {url} ===") + try: + r = await s.get(url) + except Exception as e: # noqa: BLE001 + print(f" FAIL fetch: {type(e).__name__}: {e}") + continue + print(f" status={r.status_code} size={len(r.content)}") + if r.status_code != 200: + continue + text = r.text + # Save first one для inspection + if "yumasheva" in url: + with Path("/tmp/avito-house-sample.html").open("wb") as f: + f.write(r.content) + print(" saved → /tmp/avito-house-sample.html") + markers = { + "__preloadedState__": "__preloadedState__" in text, + "__NEXT_DATA__": "__NEXT_DATA__" in text, + "__INITIAL_STATE__": "__INITIAL_STATE__" in text, + "datadome": "datadome" in text.lower(), + "captcha": "captcha" in text.lower(), + "data-marker": "data-marker" in text, + "data-app-state": "data-app-state" in text, + "developmentData": "developmentData" in text, + "avitoId": "avitoId" in text, + "window.__": "window.__" in text, + "<script id=": "<script id=" in text, + "ssr-state": "ssr-state" in text, + "__NUXT__": "__NUXT__" in text, + "Не найдено": "Не найдено" in text, + "недоступна": "недоступна" in text, + } + for k, v in markers.items(): + print(f" {'✓' if v else '✗'} {k}") + # Look for script id="..." patterns (Next.js typical) + import re + script_ids = re.findall(r'<script[^>]*id="([^"]+)"', text) + if script_ids: + print(f" script IDs found: {script_ids[:10]}") + data_attrs = re.findall(r'<\w+\s+(data-[\w-]+)=', text) + if data_attrs: + unique = sorted(set(data_attrs))[:15] + print(f" data attrs sample: {unique}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/tradein-mvp/scripts/inspect-search-html.py b/tradein-mvp/scripts/inspect-search-html.py new file mode 100644 index 00000000..c6525dee --- /dev/null +++ b/tradein-mvp/scripts/inspect-search-html.py @@ -0,0 +1,55 @@ +"""Fetch Avito search page и анализ URL slugs для room-filter.""" +import re +import sys +import types +from pathlib import Path + +_BACKEND = Path(__file__).resolve().parent.parent / "backend" +if str(_BACKEND) not in sys.path: + sys.path.insert(0, str(_BACKEND)) + +for _m in ("weasyprint", "weasyprint.text", "weasyprint.text.ffi", "weasyprint.css", "weasyprint.css.targets"): + sys.modules[_m] = types.ModuleType(_m) +sys.modules["weasyprint"].HTML = type("HTML", (), {"__init__": lambda self, *a, **kw: None}) +sys.modules["weasyprint"].CSS = type("CSS", (), {"__init__": lambda self, *a, **kw: None}) + +import asyncio # noqa: E402 + +from curl_cffi.requests import AsyncSession # noqa: E402 + + +async def main() -> None: + url = "https://www.avito.ru/ekaterinburg/kvartiry/prodam-ASgBAgICAUSSA8YQ?s=104&p=1" + async with AsyncSession( + impersonate="chrome120", + headers={ + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "ru-RU,ru;q=0.9", + "Sec-Fetch-Dest": "document", + "Sec-Fetch-Mode": "navigate", + "Sec-Fetch-Site": "none", + }, + ) as s: + r = await s.get(url) + print(f"status={r.status_code} size={len(r.content)}") + text = r.text + with Path("/tmp/avito-search-sample.html").open("wb") as f: + f.write(r.content) + print("saved → /tmp/avito-search-sample.html") + # Find filter links — pattern /ekaterinburg/kvartiry/prodam/<slug>-ASg... + links = re.findall(r'href="(/ekaterinburg/kvartiry/prodam/[^"]+)"', text) + unique = sorted(set(links)) + print(f"\nUnique filter links ({len(unique)} found):") + for link in unique[:30]: + # Decode shortened + if "komnatn" in link.lower() or "studii" in link.lower() or "svobodnaya" in link.lower(): + print(f" ROOMS: {link}") + else: + print(f" other: {link[:120]}") + # Also check direct headings + room_headings = re.findall(r'>(Студии?|[0-9]-[Кк]омнатн\w+|Свободная планировка)<', text) + print(f"\nRoom headings in HTML: {sorted(set(room_headings))}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/tradein-mvp/scripts/local-sweep-ekb-cian.py b/tradein-mvp/scripts/local-sweep-ekb-cian.py new file mode 100644 index 00000000..6250e8ec --- /dev/null +++ b/tradein-mvp/scripts/local-sweep-ekb-cian.py @@ -0,0 +1,359 @@ +"""Cian ЕКБ vtorichka sweep — local residential IP → prod tradein DB через SSH tunnel. + +Why: ekb.cian.ru SERP отдаёт ВЕСЬ город (без geo-фильтра по lat/lon), поэтому +"10 anchors" из Avito-эталона тут бесполезны — одинаковый ответ × 10. +Вместо anchors — proper room-segmented citywide pagination +(rooms 1/2/3/4 × N pages × ~28 listings), per-page save (visible progress + +не теряем данные при крэше). + +Cian реально менее агрессивен чем Avito, но **residential IP всё равно может попасть +в blacklist при слишком частых запросах** — default delay 15s. + +Setup (один раз): + ssh -f -N -L 15433:172.20.0.2:5432 gendesign + cd tradein-mvp/backend + $env:DATABASE_URL='postgresql+psycopg://tradein:<PASS>@localhost:15433/tradein' + $env:ENVIRONMENT='local'; $env:GLITCHTIP_DSN='' + +Usage: + uv run python ../scripts/local-sweep-ekb-cian.py # default: 4 rooms × 8 pages + uv run python ../scripts/local-sweep-ekb-cian.py --pages 20 # больше глубины + uv run python ../scripts/local-sweep-ekb-cian.py --delay 10 # быстрее (risk) + uv run python ../scripts/local-sweep-ekb-cian.py --rooms 0 # всё (включая студии — no room param) +""" +from __future__ import annotations + +# stdlib +import argparse +import asyncio +import json +import sys +import time +import types +from pathlib import Path + +# Auto-inject backend/ в sys.path чтобы найти app.* package +_BACKEND_DIR = Path(__file__).resolve().parent.parent / "backend" +if str(_BACKEND_DIR) not in sys.path: + sys.path.insert(0, str(_BACKEND_DIR)) + +# Stub weasyprint (Windows без GTK ломает import chain через trade_in.py) +for _m in ( + "weasyprint", + "weasyprint.text", + "weasyprint.text.ffi", + "weasyprint.css", + "weasyprint.css.targets", +): + sys.modules[_m] = types.ModuleType(_m) +sys.modules["weasyprint"].HTML = type("HTML", (), {"__init__": lambda self, *a, **kw: None}) +sys.modules["weasyprint"].CSS = type("CSS", (), {"__init__": lambda self, *a, **kw: None}) + +# app imports после stubs +import logging # noqa: E402 +import re # noqa: E402 + +from app.core.db import SessionLocal # noqa: E402 +from app.services.scrapers.base import ScrapedLot, save_listings # noqa: E402 +from app.services.scrapers.cian import CianScraper # noqa: E402 + +# Шумный WARNING из upstream cian_state_parser ('state extraction failed') — у нас +# fallback на новый .concat() markup отрабатывает, warning нерелевантен. +logging.getLogger("app.services.scrapers.cian").setLevel(logging.ERROR) + +# ── New-markup fallback parser ─────────────────────────────────────────────── +# Cian переключил MFE bootstrap с .push({...}) на .concat([{...},{...}]): +# window._cianConfig['frontend-serp'] = (window._cianConfig['frontend-serp'] || []).concat([…]) +# Существующий cian_state_parser.py не ловит этот шаблон → 0 lots на свежей выдаче. +# Чтобы не трогать shared scraper module (правилами запрещено), парсим тут. +_RE_CONCAT_START = re.compile( + r"window\._cianConfig\['(?P<mfe>[\w-]+)'\]\s*=\s*" + r"\(window\._cianConfig\['[\w-]+'\]\s*\|\|\s*\[\]\)\.concat\(\[" +) + + +def _extract_balanced_array(text: str, start: int) -> str | None: + """Из позиции `start` (где '[') вернуть текст массива включая внешние '[' и ']'. + + JSON-aware: учитывает strings и escapes, чтобы не сбить счётчик '[' ']' на + скобках внутри string literals. + """ + if text[start] != "[": + return None + depth = 0 + i = start + in_str = False + esc = False + while i < len(text): + ch = text[i] + if in_str: + if esc: + esc = False + elif ch == "\\": + esc = True + elif ch == '"': + in_str = False + else: + if ch == '"': + in_str = True + elif ch == "[": + depth += 1 + elif ch == "]": + depth -= 1 + if depth == 0: + return text[start : i + 1] + i += 1 + return None + + +def _parse_serp_v2(scraper: CianScraper, html: str) -> list[ScrapedLot]: + """Fallback парсер для нового шаблона .concat([...]) — ekb.cian.ru/cat.php c 2026-05.""" + state: dict | None = None + for m in _RE_CONCAT_START.finditer(html): + if m.group("mfe") != "frontend-serp": + continue + # Позиция '[' — это последний символ матча минус 1 (regex кончается на '[') + arr_start = m.end() - 1 + arr_str = _extract_balanced_array(html, arr_start) + if arr_str is None: + continue + try: + entries = json.loads(arr_str) + except json.JSONDecodeError: + continue + for entry in entries: + if ( + isinstance(entry, dict) + and entry.get("key") == "initialState" + and isinstance(entry.get("value"), dict) + ): + state = entry["value"] + break + if state is not None: + break + if state is None: + return [] + offers = (state.get("results") or {}).get("offers") or [] + lots: list[ScrapedLot] = [] + for offer in offers: + # Cian свежий markup: descriptionMinhash = list[int] (multi-hash signatures). + # ScrapedLot ожидает str → сбрасываем чтобы _offer_to_lot вычислил SHA1 от description. + if isinstance(offer.get("descriptionMinhash"), list): + offer["descriptionMinhash"] = None + lot = scraper._offer_to_lot(offer) + if lot is not None: + lots.append(lot) + return lots + + +def _load_cookies_file(path: Path) -> dict[str, str]: + """Загрузить cookies из JSON-файла. + + Поддерживает два формата: + - browser export (list[{name,value,...}]) — DevTools / EditThisCookie / Cookie-Editor + - plain dict {name: value} + """ + raw = json.loads(path.read_text(encoding="utf-8")) + if isinstance(raw, list): + return {c["name"]: c["value"] for c in raw if c.get("name") and c.get("value")} + if isinstance(raw, dict): + return {str(k): str(v) for k, v in raw.items()} + raise SystemExit(f"--cookies-file: unknown JSON shape (got {type(raw).__name__})") + + +async def phase_citywide( + pages: int, + delay: float, + room_segments: tuple[tuple[int, ...] | None, ...], + cookies_file: Path | None = None, + start_page: int = 1, +) -> int: + """Room-segmented citywide pagination — per-page save, incremental. + + room_segments — кортеж из tuple (1,) / (2,) / ... / None (= no room filter). + None segment даёт смешанную выдачу (студии + всё подряд) — Cian + не различает их без room param. + """ + seg_labels = [ + ("r" + str(seg[0]) if seg else "all") + for seg in room_segments + ] + print( + f"\n=== PHASE CITYWIDE: Cian ЕКБ vtorichka " + f"({len(room_segments)} segments={seg_labels} × {pages} pages, delay {delay}s) ===\n", + flush=True, + ) + total = 0 + cum_ins = 0 + cum_upd = 0 + async with CianScraper() as s: + # Override per-request delay (in addition to between-anchors delay) + s.request_delay_sec = delay + # Inject cookies (если есть): прокидываем в curl_cffi session перед warm-up + if cookies_file is not None: + cookies = _load_cookies_file(cookies_file) + assert s._cffi is not None + s._cffi.cookies.update(cookies) + print(f" loaded {len(cookies)} cookies from {cookies_file.name}", flush=True) + # Warm-up: посетить homepage чтобы получить _CIAN_GK / _yasc + не выглядеть как наглый бот + try: + assert s._cffi is not None + r_warm = await s._cffi.get(f"{s.base_url}/") + print( + f" warmup GET / → HTTP {r_warm.status_code} " + f"(cookies={len(list(s._cffi.cookies.jar))})", + flush=True, + ) + # Subsequent SERP requests should look like in-site navigation + s._cffi.headers.update( + { + "Referer": f"{s.base_url}/", + "Sec-Fetch-Site": "same-origin", + "Sec-Fetch-Mode": "navigate", + "Sec-Fetch-User": "?1", + "Sec-Fetch-Dest": "document", + } + ) + await asyncio.sleep(min(5.0, delay)) + except Exception as e: # noqa: BLE001 + print(f" warmup FAIL: {type(e).__name__}: {e}", flush=True) + for seg_idx, seg in enumerate(room_segments): + seg_label = "r" + str(seg[0]) if seg else "all" + # --start-page применяется только к первому segment'у — продолжение прерванного sweep + page_from = start_page if seg_idx == 0 else 1 + for page in range(page_from, pages + 1): + # Manual fetch+parse чтобы можно было дампнуть HTML при empty results + try: + url_ = s._build_url(rooms=seg, page=page) + assert s._cffi is not None + resp = await s._cffi.get(url_) + if resp.status_code != 200: + print( + f" seg={seg_label} page={page} HTTP {resp.status_code} → skip", + flush=True, + ) + await asyncio.sleep(delay) + continue + lots = s._parse_serp_html(resp.text) + if not lots: + # Fallback: новый markup .concat([...]) + lots = _parse_serp_v2(s, resp.text) + if lots: + print( + f" (parsed via v2 .concat fallback)", + flush=True, + ) + if not lots: + # Дамп для разбора: captcha vs пустая страница vs изменение markup + dump_path = Path(f"cian-empty-seg-{seg_label}-p{page}.html") + dump_path.write_text(resp.text, encoding="utf-8") + print(f" DUMP: {dump_path} ({len(resp.text)} bytes)", flush=True) + await s.sleep_between_requests() + except Exception as e: # noqa: BLE001 + print( + f" seg={seg_label} page={page} FAIL fetch: " + f"{type(e).__name__}: {e}", + flush=True, + ) + await asyncio.sleep(delay) + continue + if not lots: + print( + f" seg={seg_label} page={page}: 0 lots — end of segment " + "(либо captcha)", + flush=True, + ) + break + db = SessionLocal() + try: + ins, upd = save_listings(db, lots) + cum_ins += ins + cum_upd += upd + total += len(lots) + print( + f" seg={seg_label} page={page}/{pages}: {len(lots)} lots → " + f"ins={ins} upd={upd} " + f"[cum: total={total} new={cum_ins} upd={cum_upd}]", + flush=True, + ) + except Exception as e: # noqa: BLE001 + print( + f" seg={seg_label} page={page} FAIL save: " + f"{type(e).__name__}: {e}", + flush=True, + ) + finally: + db.close() + # scraper.fetch_around() уже спит request_delay_sec * jitter — extra sleep не нужен + print( + f" PHASE CITYWIDE TOTAL: fetched={total} ins={cum_ins} upd={cum_upd}", + flush=True, + ) + return total + + +async def main() -> None: + p = argparse.ArgumentParser() + p.add_argument( + "--pages", + type=int, + default=8, + help="страниц на room-segment (default 8 → ~28×4×8 = ~900 до дедупа)", + ) + p.add_argument( + "--delay", + type=float, + default=15, + help="sec между HTTP requests (residential-safe ≥10, default 15)", + ) + p.add_argument( + "--rooms", + type=str, + default="1,2,3,4", + help="room segments через запятую (1,2,3,4) или 'all' для одного citywide " + "запроса без room param (default '1,2,3,4')", + ) + p.add_argument( + "--cookies-file", + type=Path, + default=None, + help="JSON-файл с Cian cookies (browser-export или {name:value}). " + "Опционально — для обхода captcha. См. scripts/.cian-cookies.json", + ) + p.add_argument( + "--start-page", + type=int, + default=1, + help="С какой страницы начать ПЕРВЫЙ сегмент (для resume прерванного sweep). " + "Остальные segments идут с p=1. Default 1.", + ) + args = p.parse_args() + + # Parse rooms argument + if args.rooms.strip().lower() == "all": + segments: tuple[tuple[int, ...] | None, ...] = (None,) + else: + try: + nums = [int(x.strip()) for x in args.rooms.split(",") if x.strip()] + except ValueError as e: + raise SystemExit(f"--rooms must be comma-separated ints or 'all': {e}") from e + segments = tuple((n,) for n in nums) + + cookies_file: Path | None = args.cookies_file + if cookies_file is not None and not cookies_file.exists(): + raise SystemExit(f"--cookies-file not found: {cookies_file}") + + t0 = time.perf_counter() + await phase_citywide( + args.pages, + args.delay, + segments, + cookies_file=cookies_file, + start_page=args.start_page, + ) + elapsed = time.perf_counter() - t0 + print(f"\n=== ALL DONE in {elapsed / 60:.1f} min ===", flush=True) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/tradein-mvp/scripts/local-sweep-ekb-yandex.py b/tradein-mvp/scripts/local-sweep-ekb-yandex.py new file mode 100644 index 00000000..9fc87cf9 --- /dev/null +++ b/tradein-mvp/scripts/local-sweep-ekb-yandex.py @@ -0,0 +1,376 @@ +"""Yandex.Недвижимость ЕКБ vtorichka sweep — runs locally через residential IP. + +Why local: Yandex anti-bot (captcha redirect) включается быстрее на datacenter +IP + httpx fingerprint. Запуск с residential IP + Chrome120 TLS impersonation +(curl_cffi) + browser session cookies — проходит. Скрипт пишет в prod tradein +DB через SSH tunnel (localhost:15433 → tradein-postgres docker container). + +Modes: + --mode citywide (default): один общий /vtorichniy-rynok/ sweep, ~25 страниц + × 23 cards = ~575 уникальных. Yandex SERP лимит. + --mode combos: итерация 5 rooms × 6 price ranges = 30 combos × до 25 pages. + DB-level дедуп → coverage близко к total (~4000+ vtorichka + ЕКБ). ~75 min полный прогон. + +Setup: + ssh -f -N -L 15433:172.20.0.2:5432 gendesign # tradein-postgres tunnel + cd tradein-mvp/backend + export DATABASE_URL='postgresql+psycopg://tradein:<PASS>@localhost:15433/tradein' + export ENVIRONMENT='local' + export GLITCHTIP_DSN='' + +Usage: + uv run python ../scripts/local-sweep-ekb-yandex.py # citywide + uv run python ../scripts/local-sweep-ekb-yandex.py --mode combos # full ~4000 + uv run python ../scripts/local-sweep-ekb-yandex.py --mode combos \\ + --rooms 1,2 --price-ranges 5-7,7-10 # subset + uv run python ../scripts/local-sweep-ekb-yandex.py --pages 1 # smoke +""" +from __future__ import annotations + +# stdlib +import argparse +import asyncio +import json +import random +import sys +import time +import types +from pathlib import Path +from urllib.parse import urlencode + +# Auto-inject backend/ в sys.path чтобы найти app.* package +_BACKEND_DIR = Path(__file__).resolve().parent.parent / "backend" +if str(_BACKEND_DIR) not in sys.path: + sys.path.insert(0, str(_BACKEND_DIR)) + +# Stub weasyprint (Windows без GTK ломает import chain через trade_in.py) +for _m in ( + "weasyprint", + "weasyprint.text", + "weasyprint.text.ffi", + "weasyprint.css", + "weasyprint.css.targets", +): + sys.modules[_m] = types.ModuleType(_m) +sys.modules["weasyprint"].HTML = type("HTML", (), {"__init__": lambda self, *a, **kw: None}) +sys.modules["weasyprint"].CSS = type("CSS", (), {"__init__": lambda self, *a, **kw: None}) + +# app imports после stubs +from curl_cffi.requests import AsyncSession # noqa: E402 + +from app.core.db import SessionLocal # noqa: E402 +from app.services.scrapers.base import save_listings # noqa: E402 +from app.services.scrapers.yandex_realty import YandexRealtyScraper # noqa: E402 + +DEFAULT_CITY = "ekaterinburg" +DEFAULT_COOKIES_FILE = Path(__file__).resolve().parent / ".yandex-cookies.json" + +# Path-сегменты Yandex для room buckets (после канонизации). Ключ — slug для +# --rooms CLI; значение — segment в URL. +ROOM_PATH: dict[str, str] = { + "studio": "studiya", + "1": "odnokomnatnaya", + "2": "dvuhkomnatnaya", + "3": "trehkomnatnaya", + "4+": "4-i-bolee", +} + +# Price ranges (price_min, price_max) в рублях. None — open-ended boundary. +PRICE_RANGES: list[tuple[int | None, int | None]] = [ + (None, 5_000_000), + (5_000_000, 7_000_000), + (7_000_000, 10_000_000), + (10_000_000, 15_000_000), + (15_000_000, 25_000_000), + (25_000_000, None), +] + +_CAPTCHA_MARKERS = ("/showcaptcha", "smartcaptcha", "checkboxcaptcha") + + +def _load_cookies(path: Path) -> dict[str, str]: + if not path.exists(): + return {} + raw = json.loads(path.read_text(encoding="utf-8")) + return {item["name"]: item["value"] for item in raw if "name" in item and "value" in item} + + +def _is_captcha(html: str) -> bool: + head = html[:5000].lower() + return any(m in head for m in _CAPTCHA_MARKERS) or "докажите, что вы не робот" in head + + +def _build_url( + city: str, + page: int = 0, + rooms: str | None = None, + price_min: int | None = None, + price_max: int | None = None, +) -> str: + """Yandex SERP URL. Если rooms задан — path-based бакет; иначе общий vtorichka. + + newFlat=NO_DEAL принудительно отфильтровывает новостройки (нужно при + combo, т.к. Yandex canonicalize иногда дропает /vtorichniy-rynok/). + """ + if rooms: + path = f"/{city}/kupit/kvartira/{ROOM_PATH[rooms]}/vtorichniy-rynok/" + else: + path = f"/{city}/kupit/kvartira/vtorichniy-rynok/" + base = f"https://realty.yandex.ru{path}" + + params: dict[str, str | int] = {} + if price_min is not None: + params["priceMin"] = price_min + if price_max is not None: + params["priceMax"] = price_max + if price_min is not None or price_max is not None: + # Гарантия vtorichka — Yandex иногда дропает /vtorichniy-rynok/ при canonicalize + params["newFlat"] = "NO_DEAL" + if page > 0: + params["page"] = page + + if params: + return f"{base}?{urlencode(params)}" + return base + + +def _combo_label(rooms: str | None, price_min: int | None, price_max: int | None) -> str: + r = rooms or "all-rooms" + if price_min is None and price_max is None: + return f"{r}/any-price" + lo = f"{price_min // 1_000_000}M" if price_min else "0" + hi = f"{price_max // 1_000_000}M" if price_max else "inf" + return f"{r}/{lo}-{hi}" + + +async def sweep_one( + session: AsyncSession, + parser: YandexRealtyScraper, + city: str, + pages: int, + delay: float, + rooms: str | None = None, + price_min: int | None = None, + price_max: int | None = None, +) -> tuple[int, int, int]: + """Один scan по фиксированному (rooms, price) — пагинация до pages или empty. + + Returns: (fetched, inserted, updated) + """ + label = _combo_label(rooms, price_min, price_max) + total = cum_ins = cum_upd = 0 + for page in range(pages): + url = _build_url(city, page=page, rooms=rooms, price_min=price_min, price_max=price_max) + try: + resp = await session.get(url, timeout=20) + except Exception as e: # noqa: BLE001 + print(f" [{label}] page={page} FAIL fetch: {type(e).__name__}: {e}", flush=True) + await asyncio.sleep(delay) + continue + if resp.status_code != 200: + print(f" [{label}] page={page} HTTP {resp.status_code} — stop combo", flush=True) + break + if _is_captcha(resp.text): + print( + f" [{label}] page={page} ⚠️ CAPTCHA — stop combo. " + f"Подожди 30-60min или refresh .yandex-cookies.json", + flush=True, + ) + raise RuntimeError("captcha") + # NBSP → space: yandex_helpers.parse_rub() багует на \xa0 в group(1) → price None + html = resp.text.replace("\xa0", " ") + lots = parser._parse_html(html, page=page) + if not lots: + print(f" [{label}] page={page}: 0 lots — end of pagination, stop", flush=True) + break + db = SessionLocal() + try: + ins, upd = save_listings(db, lots) + cum_ins += ins + cum_upd += upd + total += len(lots) + print( + f" [{label}] page={page + 1}/{pages}: {len(lots)} lots → " + f"ins={ins} upd={upd} [combo: {total}/{cum_ins}/{cum_upd}]", + flush=True, + ) + except Exception as e: # noqa: BLE001 + print(f" [{label}] page={page} FAIL save: {type(e).__name__}: {e}", flush=True) + finally: + db.close() + if page + 1 < pages: + await asyncio.sleep(delay * random.uniform(0.85, 1.3)) + return total, cum_ins, cum_upd + + +async def phase_citywide(pages: int, delay: float, city: str, cookies: dict[str, str]) -> None: + print( + f"\n=== PHASE CITYWIDE: Yandex vtorichka (city={city}, {pages} pages, " + f"delay {delay}s) ===\n", + flush=True, + ) + parser = YandexRealtyScraper(city=city) + async with AsyncSession(impersonate="chrome120", cookies=cookies or {}) as session: + total, ins, upd = await sweep_one(session, parser, city, pages, delay) + print(f" CITYWIDE TOTAL: fetched={total} ins={ins} upd={upd}", flush=True) + + +async def phase_combos( + pages: int, + delay: float, + city: str, + cookies: dict[str, str], + rooms_list: list[str], + price_ranges: list[tuple[int | None, int | None]], + combo_delay: float, +) -> None: + combos = [(r, lo, hi) for r in rooms_list for lo, hi in price_ranges] + print( + f"\n=== PHASE COMBOS: {len(combos)} combos × ≤{pages} pages " + f"(city={city}, delay {delay}s, combo-delay {combo_delay}s) ===", + flush=True, + ) + parser = YandexRealtyScraper(city=city) + grand_fetched = grand_ins = grand_upd = 0 + async with AsyncSession(impersonate="chrome120", cookies=cookies or {}) as session: + for i, (rooms, lo, hi) in enumerate(combos, 1): + label = _combo_label(rooms, lo, hi) + print(f"\n [{i}/{len(combos)}] combo={label}", flush=True) + try: + f, ins, upd = await sweep_one( + session, parser, city, pages, delay, + rooms=rooms, price_min=lo, price_max=hi, + ) + except RuntimeError as e: + if str(e) == "captcha": + print(" ⚠️ stopping combo run — captcha", flush=True) + break + raise + grand_fetched += f + grand_ins += ins + grand_upd += upd + print( + f" [{i}/{len(combos)}] {label} done: f={f} i={ins} u={upd} " + f"[grand: f={grand_fetched} i={grand_ins} u={grand_upd}]", + flush=True, + ) + if i < len(combos): + await asyncio.sleep(combo_delay * random.uniform(0.85, 1.3)) + print( + f"\n COMBOS GRAND TOTAL: fetched={grand_fetched} " + f"ins={grand_ins} upd={grand_upd}", + flush=True, + ) + + +def _parse_rooms_arg(s: str) -> list[str]: + if s == "all": + return list(ROOM_PATH) + out: list[str] = [] + for tok in s.split(","): + tok = tok.strip() + if tok not in ROOM_PATH: + raise argparse.ArgumentTypeError( + f"unknown room '{tok}'. Allowed: {sorted(ROOM_PATH)} or 'all'" + ) + out.append(tok) + return out + + +def _parse_price_ranges_arg(s: str) -> list[tuple[int | None, int | None]]: + if s == "all": + return list(PRICE_RANGES) + out: list[tuple[int | None, int | None]] = [] + for tok in s.split(","): + tok = tok.strip() + if "-" not in tok: + raise argparse.ArgumentTypeError(f"range '{tok}' missing '-'") + lo_s, hi_s = tok.split("-", 1) + lo = int(float(lo_s) * 1_000_000) if lo_s else None + hi = int(float(hi_s) * 1_000_000) if hi_s else None + out.append((lo, hi)) + return out + + +async def main() -> None: + p = argparse.ArgumentParser(description="Yandex.Недвижимость ЕКБ sweep") + p.add_argument( + "--mode", + choices=["citywide", "combos"], + default="citywide", + help="citywide = один общий vtorichka sweep (~575 уникальных). " + "combos = rooms × price = ~4000+", + ) + p.add_argument( + "--pages", + type=int, + default=25, + help="max pages per sweep (Yandex SERP cap ≈ 25; stops earlier on empty)", + ) + p.add_argument( + "--delay", + type=float, + default=15, + help="sec между HTTP requests внутри combo (default 15, residential-safe)", + ) + p.add_argument( + "--combo-delay", + type=float, + default=20, + help="sec между combos (default 20, дополнительный jitter combos mode)", + ) + p.add_argument( + "--rooms", + type=_parse_rooms_arg, + default="all", + help=f"combos: comma-list из {sorted(ROOM_PATH)} или 'all' (default)", + ) + p.add_argument( + "--price-ranges", + type=_parse_price_ranges_arg, + default="all", + help="combos: comma-list 'lo-hi' в млн руб ('-5,5-7,7-10,10-15,15-25,25-') " + "или 'all' (default). Empty side = open-ended.", + ) + p.add_argument( + "--city", + type=str, + default=DEFAULT_CITY, + help=f"Yandex city slug (default '{DEFAULT_CITY}')", + ) + p.add_argument( + "--cookies-file", + type=Path, + default=DEFAULT_COOKIES_FILE, + help=f"Browser cookies JSON (default '{DEFAULT_COOKIES_FILE.name}'). " + f"Empty/missing → start без cookies (риск captcha).", + ) + args = p.parse_args() + + cookies = _load_cookies(args.cookies_file) + if cookies: + print(f"loaded {len(cookies)} cookies from {args.cookies_file}", flush=True) + else: + print( + f"⚠️ no cookies loaded ({args.cookies_file}, exists={args.cookies_file.exists()}). " + f"Yandex anti-bot risk.", + flush=True, + ) + + t0 = time.perf_counter() + if args.mode == "citywide": + await phase_citywide(args.pages, args.delay, args.city, cookies) + else: + await phase_combos( + args.pages, args.delay, args.city, cookies, + rooms_list=args.rooms, + price_ranges=args.price_ranges, + combo_delay=args.combo_delay, + ) + elapsed = time.perf_counter() - t0 + print(f"\n=== ALL DONE in {elapsed / 60:.1f} min ===", flush=True) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/tradein-mvp/scripts/local-sweep-ekb.py b/tradein-mvp/scripts/local-sweep-ekb.py new file mode 100644 index 00000000..35c2dd98 --- /dev/null +++ b/tradein-mvp/scripts/local-sweep-ekb.py @@ -0,0 +1,443 @@ +"""Full ЕКБ sweep + houses enrichment — runs locally через residential IP. + +Why: prod VPS Beget IP заблокирован Avito (datacenter fingerprint → 429). +Local IP (домашний ISP) = residential → проходит. Скрипт пишет в prod DB +через SSH tunnel (localhost:15433 → tradein-postgres docker container). + +Setup: + ssh -f -N -L 15433:172.20.0.2:5432 gendesign # tradein-postgres tunnel + cd tradein-mvp/backend + export DATABASE_URL='postgresql+psycopg://tradein:<PASS>@localhost:15433/tradein' + +Usage: + uv run python ../scripts/local-sweep-ekb.py # defaults + uv run python ../scripts/local-sweep-ekb.py --pages 3 --detail-top 20 --delay 30 + uv run python ../scripts/local-sweep-ekb.py --skip-search # только detail + houses + uv run python ../scripts/local-sweep-ekb.py --skip-houses # без houses +""" +from __future__ import annotations + +# stdlib +import argparse +import asyncio +import sys +import time +import types +from pathlib import Path +from urllib.parse import urlparse + +# Auto-inject backend/ в sys.path чтобы найти app.* package +# (скрипт лежит в tradein-mvp/scripts/, app — в tradein-mvp/backend/) +_BACKEND_DIR = Path(__file__).resolve().parent.parent / "backend" +if str(_BACKEND_DIR) not in sys.path: + sys.path.insert(0, str(_BACKEND_DIR)) + +# Stub weasyprint (Windows без GTK ломает import chain через trade_in.py) +for _m in ( + "weasyprint", + "weasyprint.text", + "weasyprint.text.ffi", + "weasyprint.css", + "weasyprint.css.targets", +): + sys.modules[_m] = types.ModuleType(_m) +sys.modules["weasyprint"].HTML = type("HTML", (), {"__init__": lambda self, *a, **kw: None}) +sys.modules["weasyprint"].CSS = type("CSS", (), {"__init__": lambda self, *a, **kw: None}) + +# app imports после stubs +from sqlalchemy import text # noqa: E402 + +from app.core.db import SessionLocal # noqa: E402 +from app.services.scrapers.avito import AvitoScraper # noqa: E402 + + +class CityWideAvitoScraper(AvitoScraper): + """Subclass без geo-фильтра — search всего города (ЕКБ) через pagination. + + Override `_build_web_url` чтобы НЕ добавлять `geoCoords`/`radius` — Avito + отдаёт ВСЕ объявления в городе sorted by date, до 100 pages (cap ~5000). + """ + + def _build_web_url(self, lat: float, lon: float, radius_km: int, page: int = 1) -> str: # noqa: ARG002 + from urllib.parse import urlencode + params = {"s": 104, "p": page} # s=104 sort by date + return f"{self.base_url}/ekaterinburg/kvartiry/prodam-ASgBAgICAUSSA8YQ?{urlencode(params)}" + + +# Avito URL slugs для room-filter (extracted из real search page HTML 2026-05-23) +ROOM_SLUGS: list[tuple[str, str]] = [ + ("студии", "studii-ASgBAgICAkSSA8YQygj~WA"), + ("1-комн.", "1-komnatnye-ASgBAgICAkSSA8YQygiAWQ"), + ("2-комн.", "2-komnatnye-ASgBAgICAkSSA8YQygiCWQ"), + ("3-комн.", "3-komnatnye-ASgBAgICAkSSA8YQygiEWQ"), + ("4-комн.", "4-komnatnye-ASgBAgICAkSSA8YQygiGWQ"), + ("5+комн.", "5-komnatnye-ASgBAgICAkSSA8YQygiIWQ"), + ("своб.планир.", "svobodnaya_planirovka-ASgBAgICAkSSA8YQygj8zzI"), +] + + +class CityRoomsAvitoScraper(AvitoScraper): + """Subclass с URL slug filter по комнатности — для splitting когда total > 5000. + + URL pattern: /ekaterinburg/kvartiry/prodam/<slug>?s=104&p=N + """ + + def __init__(self, slug: str) -> None: + super().__init__() + self.room_slug = slug + + def _build_web_url(self, lat: float, lon: float, radius_km: int, page: int = 1) -> str: # noqa: ARG002 + from urllib.parse import urlencode + params = {"s": 104, "p": page} + return f"{self.base_url}/ekaterinburg/kvartiry/prodam/{self.room_slug}?{urlencode(params)}" +from app.services.scrapers.avito_detail import ( # noqa: E402 + fetch_detail, + save_detail_enrichment, +) +from app.services.scrapers.avito_houses import ( # noqa: E402 + fetch_house_catalog, + save_house_catalog_enrichment, +) +from app.services.scrapers.base import save_listings # noqa: E402 + +# 10 anchors покрывающих центральные/северные/южные/восточные/западные районы ЕКБ +ANCHORS: list[tuple[str, float, float]] = [ + ("Центр", 56.8400, 60.6050), + ("ЮЗ", 56.7950, 60.5300), + ("Уралмаш", 56.8970, 60.6100), + ("Академический", 56.7700, 60.5500), + ("Пионерский", 56.8650, 60.6200), + ("Эльмаш", 56.8567, 60.6700), + ("Втузгородок", 56.8400, 60.6650), + ("ВИЗ", 56.8500, 60.5650), + ("Юго-Запад", 56.7770, 60.5800), + ("Сортировка", 56.8950, 60.5600), +] + + +async def phase_citywide(pages: int, delay: float) -> int: + """City-wide pagination — ВСЕ объявления ЕКБ (no geo filter), per-page save. + + Per-page save важно: visible прогресс + не теряем данные если crash mid-way. + """ + print( + f"\n=== PHASE CITYWIDE: full ЕКБ pagination ({pages} pages, delay {delay}s) ===\n", + flush=True, + ) + total = 0 + cum_ins = 0 + cum_upd = 0 + async with CityWideAvitoScraper() as s: + for page in range(1, pages + 1): + url = s._build_web_url(0.0, 0.0, 1, page=page) + try: + assert s._cffi is not None + response = await s._cffi.get(url) + if response.status_code != 200: + print(f" ⚠️ page={page} HTTP {response.status_code} — stop", flush=True) + break + lots = s._parse_html(response.text, source_url_base=url) + except Exception as e: # noqa: BLE001 + print(f" page={page} FAIL fetch: {type(e).__name__}: {e}", flush=True) + await asyncio.sleep(delay) + continue + if not lots: + print(f" page={page}: 0 lots — end of pagination", flush=True) + break + db = SessionLocal() + try: + ins, upd = save_listings(db, lots) + cum_ins += ins + cum_upd += upd + total += len(lots) + print( + f" page={page}/{pages}: {len(lots)} lots → ins={ins} upd={upd} " + f"[cum: total={total} new={cum_ins} upd={cum_upd}]", + flush=True, + ) + except Exception as e: # noqa: BLE001 + print(f" page={page} FAIL save: {type(e).__name__}: {e}", flush=True) + finally: + db.close() + if page < pages: + await asyncio.sleep(delay) + print(f" PHASE CITYWIDE TOTAL: fetched={total} ins={cum_ins} upd={cum_upd}", flush=True) + return total + + +async def phase_byrooms(pages: int, delay: float) -> int: + """Rooms-split sweep — 7 categories × pages, per-page save. + + Для total > 5000 (Avito search cap) split по комнатности позволяет получить + все listings (каждый segment <5000). + """ + print( + f"\n=== PHASE BYROOMS: 7 категорий × {pages} pages, delay {delay}s ===\n", + flush=True, + ) + grand_total = 0 + grand_ins = 0 + grand_upd = 0 + import random + rate_limit_hits = 0 + for ci, (name, slug) in enumerate(ROOM_SLUGS, 1): + print(f"\n--- [{ci}/{len(ROOM_SLUGS)}] Категория: {name} (slug={slug[:30]}...) ---", flush=True) + cat_total = 0 + cat_ins = 0 + cat_upd = 0 + async with CityRoomsAvitoScraper(slug) as s: + for page in range(1, pages + 1): + url = s._build_web_url(0.0, 0.0, 1, page=page) + try: + assert s._cffi is not None + response = await s._cffi.get(url) + if response.status_code == 429: + # Exponential backoff на rate-limit: 60s × 2^N (max 30 min) + rate_limit_hits += 1 + backoff = min(60 * (2 ** (rate_limit_hits - 1)), 1800) + print(f" {name} p={page} HTTP 429 (hit #{rate_limit_hits}) → backoff {backoff}s", flush=True) + await asyncio.sleep(backoff) + # retry same page after backoff + try: + response = await s._cffi.get(url) + if response.status_code != 200: + print(f" {name} p={page} retry HTTP {response.status_code} — skip page", flush=True) + await asyncio.sleep(delay * 2) + continue + except Exception as e: # noqa: BLE001 + print(f" {name} p={page} retry FAIL: {type(e).__name__}: {e}", flush=True) + continue + elif response.status_code != 200: + print(f" {name} p={page} HTTP {response.status_code} — skip page", flush=True) + await asyncio.sleep(delay) + continue + lots = s._parse_html(response.text, source_url_base=url) + except Exception as e: # noqa: BLE001 + print(f" {name} p={page} FAIL fetch: {type(e).__name__}: {e}", flush=True) + await asyncio.sleep(delay) + continue + if not lots: + print(f" {name} page={page}: 0 lots — end of category", flush=True) + break + db = SessionLocal() + try: + ins, upd = save_listings(db, lots) + cat_ins += ins + cat_upd += upd + cat_total += len(lots) + print( + f" {name} p={page}/{pages}: {len(lots)} lots → ins={ins} upd={upd} " + f"[cat: tot={cat_total} new={cat_ins}]", + flush=True, + ) + except Exception as e: # noqa: BLE001 + print(f" {name} page={page} FAIL save: {type(e).__name__}: {e}", flush=True) + finally: + db.close() + if page < pages: + # Jitter: ±30% от delay чтобы выглядеть менее robotic + jittered = delay * random.uniform(0.7, 1.3) + await asyncio.sleep(jittered) + print( + f" --- {name} TOTAL: fetched={cat_total} ins={cat_ins} upd={cat_upd} ---", + flush=True, + ) + grand_total += cat_total + grand_ins += cat_ins + grand_upd += cat_upd + print( + f"\n PHASE BYROOMS GRAND TOTAL: fetched={grand_total} new={grand_ins} updated={grand_upd}", + flush=True, + ) + return grand_total + + +async def phase_a_search(pages: int, delay: float) -> int: + """Search всех anchors × pages, save listings.""" + print( + f"\n=== PHASE A: search ({len(ANCHORS)} anchors × {pages} pages, " + f"delay {delay}s) ===\n", + flush=True, + ) + total = 0 + for i, (name, lat, lon) in enumerate(ANCHORS, 1): + print(f"[{i}/{len(ANCHORS)}] {name} ({lat},{lon})", flush=True) + try: + async with AvitoScraper() as s: + lots = await s.fetch_around( + lat, lon, 1500, pages=pages, delay_override_sec=delay + ) + except Exception as e: # noqa: BLE001 + print(f" FAIL fetch: {type(e).__name__}: {e}", flush=True) + await asyncio.sleep(delay) + continue + print(f" fetched {len(lots)} lots", flush=True) + if not lots: + print(" ⚠️ 0 lots — possible 429, stop phase A", flush=True) + break + db = SessionLocal() + try: + ins, upd = save_listings(db, lots) + print(f" saved: ins={ins} upd={upd}", flush=True) + total += len(lots) + except Exception as e: # noqa: BLE001 + print(f" FAIL save: {type(e).__name__}: {e}", flush=True) + finally: + db.close() + if i < len(ANCHORS): + await asyncio.sleep(delay) + return total + + +async def phase_b_detail(top_n: int, delay: float) -> tuple[int, set[str]]: + """Detail enrichment для top-N recent listings без detail. + + Returns: (enriched_count, set of unique house_catalog_url paths extracted) + """ + print( + f"\n=== PHASE B: detail enrichment (top {top_n}, delay {delay}s) ===\n", + flush=True, + ) + db = SessionLocal() + try: + rows = db.execute( + text( + """ + SELECT source_url, source_id FROM listings + WHERE source='avito' AND detail_enriched_at IS NULL + AND source_url IS NOT NULL AND price_rub > 0 + ORDER BY scraped_at DESC NULLS LAST + LIMIT :limit + """ + ), + {"limit": top_n}, + ).mappings().all() + finally: + db.close() + + print(f" queue: {len(rows)} listings to enrich", flush=True) + enriched = 0 + failed = 0 + house_urls: set[str] = set() + for i, row in enumerate(rows, 1): + url = row["source_url"] + path = urlparse(url).path if url.startswith("http") else url + print(f"[{i}/{len(rows)}] detail {row['source_id']}", flush=True) + try: + enr = await fetch_detail(path) + db = SessionLocal() + try: + ok = save_detail_enrichment(db, enr) + print( + f" enriched: ok={ok} " + f"rooms={enr.rooms} area={enr.area_m2} " + f"kitchen={enr.kitchen_area_m2} owners={enr.owners_count} " + f"house_url={enr.house_catalog_url}", + flush=True, + ) + if ok: + enriched += 1 + # Извлекаем house path (strip query string + filter только /catalog/houses/) + if enr.house_catalog_url: + p = urlparse(enr.house_catalog_url).path + if p.startswith("/catalog/houses/"): + house_urls.add(p) + finally: + db.close() + except Exception as e: # noqa: BLE001 + print(f" FAIL: {type(e).__name__}: {e}", flush=True) + failed += 1 + if i < len(rows): + await asyncio.sleep(delay) + print(f" total: enriched={enriched} failed={failed} unique_houses_extracted={len(house_urls)}", flush=True) + return enriched, house_urls + + +async def phase_c_houses(delay: float, extra_urls: set[str] | None = None) -> int: + """Houses enrichment — unique house_url из listings (DB) + extra_urls (из Phase B in-memory). + + save_detail_enrichment не пишет house_url в listings (баг Stage 2b), + поэтому Phase C принимает extra_urls собранные Phase B вручную. + """ + print("\n=== PHASE C: houses enrichment ===\n", flush=True) + db = SessionLocal() + try: + rows = db.execute( + text( + """ + SELECT DISTINCT house_url FROM listings + WHERE source='avito' AND house_url IS NOT NULL + AND house_id_fk IS NULL + ORDER BY house_url + """ + ) + ).fetchall() + finally: + db.close() + + urls_from_db = {r[0] for r in rows if r[0]} + urls = urls_from_db | (extra_urls or set()) + print(f" queue: {len(urls)} unique houses (db={len(urls_from_db)} +extras={len(extra_urls or set())})", flush=True) + if not urls: + print(" no houses to enrich (skip)", flush=True) + return 0 + + enriched = 0 + failed = 0 + for i, url in enumerate(sorted(urls), 1): + path = urlparse(url).path if url.startswith("http") else url + print(f"[{i}/{len(urls)}] {path}", flush=True) + try: + enr = await fetch_house_catalog(path) + db = SessionLocal() + try: + counters = save_house_catalog_enrichment(db, enr) + print(f" saved: {counters}", flush=True) + enriched += 1 + finally: + db.close() + except Exception as e: # noqa: BLE001 + print(f" FAIL: {type(e).__name__}: {e}", flush=True) + failed += 1 + if i < len(urls): + await asyncio.sleep(delay) + print(f" total: enriched={enriched} failed={failed}", flush=True) + return enriched + + +async def main() -> None: + p = argparse.ArgumentParser() + p.add_argument( + "--mode", + choices=["anchors", "citywide", "byrooms", "both"], + default="anchors", + help="anchors=10 anchors; citywide=full pagination без geo; byrooms=7 категорий комнат (для total>5000); both=anchors+citywide", + ) + p.add_argument("--pages", type=int, default=5, help="pages per anchor (anchors mode) или total pages (citywide)") + p.add_argument("--detail-top", type=int, default=50, help="top-N для detail enrichment") + p.add_argument("--delay", type=float, default=30, help="sec между HTTP requests") + p.add_argument("--skip-search", action="store_true") + p.add_argument("--skip-detail", action="store_true") + p.add_argument("--skip-houses", action="store_true") + args = p.parse_args() + + t0 = time.perf_counter() + if not args.skip_search: + if args.mode in ("anchors", "both"): + await phase_a_search(args.pages, args.delay) + if args.mode in ("citywide", "both"): + cw_pages = args.pages if args.mode == "citywide" else 100 + await phase_citywide(cw_pages, args.delay) + if args.mode == "byrooms": + await phase_byrooms(args.pages, args.delay) + extracted_house_urls: set[str] = set() + if not args.skip_detail: + _, extracted_house_urls = await phase_b_detail(args.detail_top, args.delay) + if not args.skip_houses: + await phase_c_houses(args.delay, extra_urls=extracted_house_urls) + elapsed = time.perf_counter() - t0 + print(f"\n=== ALL DONE in {elapsed / 60:.1f} min ===", flush=True) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/tradein-mvp/scripts/probe-avito-1anchor.py b/tradein-mvp/scripts/probe-avito-1anchor.py new file mode 100644 index 00000000..cdcbbbbc --- /dev/null +++ b/tradein-mvp/scripts/probe-avito-1anchor.py @@ -0,0 +1,103 @@ +"""Avito probe: 1 anchor × 1 page с настраиваемой задержкой. + +Минимальный dry-run чтобы проверить bans / detection после anti-bot +hardening (PR #487) + global delay setting (PR #485). + +Setup: + ssh -f -N -L 15433:172.20.0.2:5432 gendesign + cd tradein-mvp/backend + $env:DATABASE_URL = 'postgresql+psycopg://tradein:<PASS>@localhost:15433/tradein' + +Usage: + uv run python ../scripts/probe-avito-1anchor.py + uv run python ../scripts/probe-avito-1anchor.py --delay 5 --radius 1000 +""" +from __future__ import annotations + +import argparse +import asyncio +import sys +import time +import types +from pathlib import Path + +_BACKEND_DIR = Path(__file__).resolve().parent.parent / "backend" +if str(_BACKEND_DIR) not in sys.path: + sys.path.insert(0, str(_BACKEND_DIR)) + +for _m in ( + "weasyprint", + "weasyprint.text", + "weasyprint.text.ffi", + "weasyprint.css", + "weasyprint.css.targets", +): + sys.modules[_m] = types.ModuleType(_m) +sys.modules["weasyprint"].HTML = type("HTML", (), {"__init__": lambda self, *a, **kw: None}) +sys.modules["weasyprint"].CSS = type("CSS", (), {"__init__": lambda self, *a, **kw: None}) + +from app.core.db import SessionLocal # noqa: E402 +from app.services.scrapers.avito import AvitoScraper # noqa: E402 +from app.services.scrapers.base import save_listings # noqa: E402 + +ANCHOR_NAME = "Центр ЕКБ" +ANCHOR_LAT = 56.8400 +ANCHOR_LON = 60.6050 + + +async def main() -> None: + p = argparse.ArgumentParser() + p.add_argument("--delay", type=float, default=3.0, help="sec между HTTP requests") + p.add_argument("--radius", type=int, default=1500, help="метры") + p.add_argument("--pages", type=int, default=1) + args = p.parse_args() + + print( + f"=== Avito probe ===\n" + f" anchor: {ANCHOR_NAME} ({ANCHOR_LAT}, {ANCHOR_LON})\n" + f" radius: {args.radius}m\n" + f" pages: {args.pages}\n" + f" delay: {args.delay}s\n", + flush=True, + ) + + t0 = time.perf_counter() + try: + async with AvitoScraper() as s: + lots = await s.fetch_around( + ANCHOR_LAT, ANCHOR_LON, args.radius, + pages=args.pages, delay_override_sec=args.delay, + ) + except Exception as e: # noqa: BLE001 + print(f"FETCH FAIL: {type(e).__name__}: {e}", flush=True) + return + + fetched = len(lots) + print(f"fetched: {fetched} lots in {time.perf_counter() - t0:.1f}s", flush=True) + if fetched == 0: + print("⚠️ 0 lots — вероятно 429/ban. Смотри backend logs.", flush=True) + return + + db = SessionLocal() + try: + ins, upd = save_listings(db, lots) + print(f"saved: ins={ins} upd={upd}", flush=True) + except Exception as e: # noqa: BLE001 + print(f"SAVE FAIL: {type(e).__name__}: {e}", flush=True) + finally: + db.close() + + sample = lots[:3] + print("\nsample listings:", flush=True) + for lot in sample: + addr = (lot.address or "")[:60] + print( + f" source_id={lot.source_id} price={lot.price_rub}₽ " + f"rooms={lot.rooms} area={lot.area_m2}м² floor={lot.floor}/{lot.total_floors} " + f"addr={addr!r}", + flush=True, + ) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/tradein-mvp/scripts/probe-db-counts.py b/tradein-mvp/scripts/probe-db-counts.py new file mode 100644 index 00000000..9c9d0f0c --- /dev/null +++ b/tradein-mvp/scripts/probe-db-counts.py @@ -0,0 +1,58 @@ +"""Quick DB diagnostics: Avito listings count + recent insert rate.""" +from __future__ import annotations + +import os +import psycopg + +dsn = os.environ["DATABASE_URL"].replace("postgresql+psycopg://", "postgresql://") +conn = psycopg.connect(dsn) +cur = conn.cursor() + +cur.execute("SELECT count(*) FROM listings WHERE source = 'avito'") +print("avito total:", cur.fetchone()[0]) + +cur.execute( + "SELECT count(*) FROM listings " + "WHERE source='avito' AND scraped_at > NOW() - interval '20 minutes'" +) +print("avito scraped last 20min:", cur.fetchone()[0]) + +cur.execute( + """ + SELECT column_name FROM information_schema.columns + WHERE table_name='listings' AND column_name IN + ('created_at','first_seen_at','first_seen','inserted_at') + """ +) +cols = [r[0] for r in cur.fetchall()] +print(f"timestamp cols available: {cols}") +if "first_seen_at" in cols: + cur.execute( + "SELECT count(*) FROM listings " + "WHERE source='avito' AND first_seen_at > NOW() - interval '20 minutes'" + ) + print("avito first_seen last 20min (new):", cur.fetchone()[0]) + +cur.execute( + """ + SELECT date_trunc('minute', scraped_at) AS m, count(*) + FROM listings + WHERE source='avito' AND scraped_at > NOW() - interval '30 minutes' + GROUP BY 1 ORDER BY 1 DESC + """ +) +print("\nper-minute scraped (last 30min):") +for row in cur.fetchall(): + print(f" {row[0]} {row[1]}") + +cur.execute( + """ + SELECT rooms, count(*) + FROM listings + WHERE source='avito' AND scraped_at > NOW() - interval '30 minutes' + GROUP BY 1 ORDER BY 1 + """ +) +print("\nby rooms (last 30min):") +for row in cur.fetchall(): + print(f" rooms={row[0]} count={row[1]}") diff --git a/tradein-mvp/scripts/probe-db-leak-full.py b/tradein-mvp/scripts/probe-db-leak-full.py new file mode 100644 index 00000000..366236c7 --- /dev/null +++ b/tradein-mvp/scripts/probe-db-leak-full.py @@ -0,0 +1,129 @@ +"""Полная проверка региональной утечки в БД Avito listings. + +Угол 1: source_url slug (canonical, надёжный) +Угол 2: address с маркером области (квартиры в ЕКБ не содержат "обл.") +Угол 3: address contains имя пригорода/соседнего города +Угол 4: top самых частых первых слов address (sanity check) +""" +from __future__ import annotations + +import os +from collections import Counter + +import psycopg + +dsn = os.environ["DATABASE_URL"].replace("postgresql+psycopg://", "postgresql://") +conn = psycopg.connect(dsn) +cur = conn.cursor() + +cur.execute("SELECT count(*) FROM listings WHERE source='avito'") +total = cur.fetchone()[0] +print(f"=== Avito total: {total} ===\n") + +# Угол 1: URL slug (canonical) +print("[1] URL slug check:") +cur.execute( + """ + SELECT + count(*) FILTER (WHERE source_url ~ 'avito\\.ru/ekaterinburg/') AS ekb, + count(*) FILTER (WHERE source_url IS NOT NULL AND source_url !~ 'avito\\.ru/ekaterinburg/') AS not_ekb, + count(*) FILTER (WHERE source_url IS NULL) AS no_url + FROM listings WHERE source='avito' + """ +) +ekb, not_ekb, no_url = cur.fetchone() +print(f" ekaterinburg URL: {ekb}") +print(f" NOT ekaterinburg: {not_ekb} {'← LEAK' if not_ekb else '✓ clean'}") +print(f" NULL url: {no_url}") + +# Угол 2: address с "область" / "обл." +print("\n[2] Address contains region marker (область/обл.):") +cur.execute( + """ + SELECT count(*) FROM listings + WHERE source='avito' + AND address IS NOT NULL + AND (address ILIKE '%область%' OR address ~* '\\bобл\\b' OR address ~* '\\bобл\\.') + """ +) +n = cur.fetchone()[0] +print(f" count: {n} {'← suspicious' if n > 5 else '✓ ok'}") +if n > 0: + cur.execute( + """ + SELECT source_url, address FROM listings + WHERE source='avito' AND address IS NOT NULL + AND (address ILIKE '%область%' OR address ~* '\\bобл\\b' OR address ~* '\\bобл\\.') + LIMIT 10 + """ + ) + for url, addr in cur.fetchall(): + slug = url.split('avito.ru/')[1].split('/')[0] if url and 'avito.ru/' in url else '?' + print(f" [{slug}] {addr[:90]!r}") + +# Угол 3: address contains имя другого города/пригорода +print("\n[3] Address contains satellite-city marker:") +satellites = [ + "Верхняя Пышма", "Арамиль", "Сысерть", "Дегтярск", "Первоуральск", + "Заречный", "Невьянск", "Нижний Тагил", "Кыштым", "Челябинск", + "Алапаевск", "Краснофимск", "Озёрск", "Озерск", "Снежинск", "Касли", + "Рефтинский", "Свободный", "Большой Исток", "Пышма", "Покровское", + "Реж", "Лесной", "Полевской", "Берёзовск", "Березовск", "Михайловск", + "Туринск", "Североуральск", "Тавда", "Богданович", "Артёмовский", + "Артемовский", "Качканар", "Ивдель", "Серов", "Ирбит", "Новоуральск", + "Камышлов", "Сухой Лог", +] +for city in satellites: + cur.execute( + "SELECT count(*) FROM listings WHERE source='avito' AND address ILIKE %s", + (f"%{city}%",), + ) + n = cur.fetchone()[0] + if n > 0: + print(f" {city}: {n}") + +cur.execute( + """ + SELECT count(*) FROM listings WHERE source='avito' AND ( + address ILIKE '%Верхняя Пышма%' OR address ILIKE '%Арамиль%' OR + address ILIKE '%Сысерть%' OR address ILIKE '%Дегтярск%' OR + address ILIKE '%Первоуральск%' OR address ILIKE '%Заречный%' OR + address ILIKE '%Невьянск%' OR address ILIKE '%Нижний Тагил%' OR + address ILIKE '%Кыштым%' OR address ILIKE '%Челябинск%' OR + address ILIKE '%Алапаевск%' OR address ILIKE '%Краснофимск%' OR + address ILIKE '%Озёрск%' OR address ILIKE '%Озерск%' OR + address ILIKE '%Снежинск%' OR address ILIKE '%Касли%' OR + address ILIKE '%Рефтинский%' OR address ILIKE '%Большой Исток%' OR + address ILIKE '%Реж%' OR address ILIKE '%Лесной%' OR address ILIKE '%Полевской%' OR + address ILIKE '%Берёзовск%' OR address ILIKE '%Березовск%' + ) + """ +) +sat_total = cur.fetchone()[0] +print(f" TOTAL satellite-city leaks: {sat_total}") + +# Угол 4: top first-words of address +print("\n[4] Top 20 first-words in address (sanity check):") +cur.execute( + """ + SELECT split_part(trim(address), ' ', 1) AS first_word, count(*) AS n + FROM listings WHERE source='avito' AND COALESCE(address, '') != '' + GROUP BY 1 ORDER BY 2 DESC LIMIT 20 + """ +) +for word, n in cur.fetchall(): + print(f" {n:>5} {word!r}") + +# Угол 5: address NULL — сколько неopределённых +print("\n[5] Coverage:") +cur.execute( + """ + SELECT + count(*) FILTER (WHERE COALESCE(address, '') = '') AS no_addr, + count(*) FILTER (WHERE COALESCE(address, '') != '') AS has_addr + FROM listings WHERE source='avito' + """ +) +no_a, has_a = cur.fetchone() +print(f" has address: {has_a}") +print(f" no address: {no_a}") diff --git a/tradein-mvp/scripts/probe-db-region-leak.py b/tradein-mvp/scripts/probe-db-region-leak.py new file mode 100644 index 00000000..8abb2039 --- /dev/null +++ b/tradein-mvp/scripts/probe-db-region-leak.py @@ -0,0 +1,75 @@ +"""Проверка: есть ли в DB Avito листинги из других городов (region leak).""" +from __future__ import annotations + +import os +import psycopg + +dsn = os.environ["DATABASE_URL"].replace("postgresql+psycopg://", "postgresql://") +conn = psycopg.connect(dsn) +cur = conn.cursor() + +cur.execute("SELECT count(*) FROM listings WHERE source='avito'") +total = cur.fetchone()[0] +print(f"avito total: {total}") + +# Все с lat/lon — проверим bounding box ЕКБ (56.7..57.0, 60.4..60.8) +cur.execute( + """ + SELECT + count(*) FILTER (WHERE lat BETWEEN 56.5 AND 57.0 AND lon BETWEEN 60.4 AND 61.0) as in_ekb, + count(*) FILTER (WHERE lat IS NOT NULL AND (lat < 56.5 OR lat > 57.0 OR lon < 60.4 OR lon > 61.0)) as outside_ekb, + count(*) FILTER (WHERE lat IS NULL) as no_coords + FROM listings WHERE source='avito' + """ +) +in_ekb, outside, no_coords = cur.fetchone() +print(f" in EKB bbox: {in_ekb}") +print(f" outside EKB bbox: {outside} ← out-of-region leak!") +print(f" no coords yet: {no_coords}") + +# Sample outside +cur.execute( + """ + SELECT source_id, lat, lon, address, price_rub, rooms + FROM listings + WHERE source='avito' AND lat IS NOT NULL + AND (lat < 56.5 OR lat > 57.0 OR lon < 60.4 OR lon > 61.0) + ORDER BY scraped_at DESC NULLS LAST + LIMIT 10 + """ +) +print("\nsample non-EKB listings:") +for r in cur.fetchall(): + sid, lat, lon, addr, price, rooms = r + print(f" {sid} ({lat:.3f},{lon:.3f}) rooms={rooms} {price:,}₽ — {(addr or '')[:80]!r}") + +# Address-based leak (без lat/lon) — поиск явных маркеров других городов +cur.execute( + """ + SELECT count(*) FROM listings + WHERE source='avito' + AND lat IS NULL + AND address IS NOT NULL + AND (address ILIKE '%москв%' OR address ILIKE '%омск%' OR address ILIKE '%чебок%' + OR address ILIKE '%петерб%' OR address ILIKE '%казан%' OR address ILIKE '%новосиб%') + """ +) +print(f"\naddress-based leak (no coords, other-city keyword): {cur.fetchone()[0]}") + +# По комнатам — где leak вероятнее всего +cur.execute( + """ + SELECT rooms, + count(*) AS total, + count(*) FILTER (WHERE lat IS NOT NULL AND (lat < 56.5 OR lat > 57.0 OR lon < 60.4 OR lon > 61.0)) AS leak + FROM listings WHERE source='avito' + GROUP BY rooms + ORDER BY rooms NULLS LAST + """ +) +print("\nleak by rooms:") +print(f" {'rooms':<8} {'total':>8} {'leak':>8}") +for r in cur.fetchall(): + rooms, total, leak = r + pct = (leak * 100.0 / total) if total else 0 + print(f" {str(rooms):<8} {total:>8} {leak:>8} ({pct:.0f}%)") diff --git a/tradein-mvp/scripts/probe-db-url-leak.py b/tradein-mvp/scripts/probe-db-url-leak.py new file mode 100644 index 00000000..086c2478 --- /dev/null +++ b/tradein-mvp/scripts/probe-db-url-leak.py @@ -0,0 +1,51 @@ +"""URL-based region leak check для Avito listings.""" +from __future__ import annotations + +import os +import re +from collections import Counter + +import psycopg + +dsn = os.environ["DATABASE_URL"].replace("postgresql+psycopg://", "postgresql://") +conn = psycopg.connect(dsn) +cur = conn.cursor() + +cur.execute("SELECT source_url FROM listings WHERE source='avito'") +slug_re = re.compile(r"avito\.ru/([a-z_]+)/") + +cities = Counter() +no_match = 0 +for (url,) in cur.fetchall(): + if not url: + no_match += 1 + continue + m = slug_re.search(url) + if m: + cities[m.group(1)] += 1 + else: + no_match += 1 + +print("city slug distribution in source_url:") +for city, n in cities.most_common(): + flag = " ← EKB" if city == "ekaterinburg" else (" ← OTHER CITY" if city not in ("ekaterinburg",) else "") + print(f" {city:<30} {n}{flag}") +print(f" (no slug parse): {no_match}") + +# Дальше — sample non-EKB +cur.execute( + """ + SELECT source_id, source_url, address, price_rub + FROM listings + WHERE source='avito' + AND source_url IS NOT NULL + AND source_url !~ 'avito\\.ru/ekaterinburg/' + ORDER BY scraped_at DESC NULLS LAST + LIMIT 10 + """ +) +print("\nsample non-EKB listings (by URL):") +for sid, url, addr, price in cur.fetchall(): + print(f" {sid} {price:,}₽") + print(f" URL: {url}") + print(f" addr: {(addr or '')[:80]!r}") diff --git a/tradein-mvp/scripts/probe-migration-028.py b/tradein-mvp/scripts/probe-migration-028.py new file mode 100644 index 00000000..6f27fa71 --- /dev/null +++ b/tradein-mvp/scripts/probe-migration-028.py @@ -0,0 +1,74 @@ +"""Проверка состояния миграции 028_matching_tables.sql на прод tradein-postgres.""" +from __future__ import annotations + +import os +import psycopg + +dsn = os.environ["DATABASE_URL"].replace("postgresql+psycopg://", "postgresql://") +conn = psycopg.connect(dsn) +cur = conn.cursor() + +EXPECTED_COLUMNS = [ + ("houses", "address_fingerprint"), + ("listings", "canonical"), + ("listings", "merged_into"), +] +EXPECTED_INDEXES = [ + "houses_addr_fp_idx", + "listings_merged_into_idx", + "house_sources_house_idx", + "house_sources_low_conf_idx", + "listing_sources_listing_idx", + "listing_sources_low_conf_idx", + "listing_sources_scraped_idx", + "haa_house_idx", + "haa_fingerprint_idx", +] +EXPECTED_TABLES = ["house_sources", "listing_sources", "house_address_aliases"] + +print("=== Columns ===") +for tbl, col in EXPECTED_COLUMNS: + cur.execute( + "SELECT 1 FROM information_schema.columns WHERE table_name=%s AND column_name=%s", + (tbl, col), + ) + status = "✓" if cur.fetchone() else "✗ MISSING" + print(f" {tbl}.{col:<25} {status}") + +print("\n=== Tables ===") +for tbl in EXPECTED_TABLES: + cur.execute( + "SELECT 1 FROM information_schema.tables WHERE table_name=%s AND table_schema='public'", + (tbl,), + ) + status = "✓" if cur.fetchone() else "✗ MISSING" + print(f" {tbl:<30} {status}") + +print("\n=== Indexes ===") +for idx in EXPECTED_INDEXES: + cur.execute( + "SELECT 1 FROM pg_indexes WHERE indexname=%s", + (idx,), + ) + status = "✓" if cur.fetchone() else "✗ MISSING" + print(f" {idx:<35} {status}") + +# Проверим есть ли активные транзакции/locks от deploy которые ещё висят +print("\n=== Active transactions (locks on relevant tables) ===") +cur.execute( + """ + SELECT pid, usename, application_name, state, wait_event_type, wait_event, + query_start, substring(query, 1, 80) AS query_head + FROM pg_stat_activity + WHERE state IS DISTINCT FROM 'idle' + AND pid != pg_backend_pid() + ORDER BY query_start NULLS LAST + """ +) +rows = cur.fetchall() +if not rows: + print(" (none — clean)") +else: + for r in rows: + print(f" pid={r[0]} user={r[1]} app={r[2]} state={r[3]} wait={r[4]}/{r[5]} started={r[6]}") + print(f" query: {r[7]!r}") diff --git a/tradein-mvp/scripts/probe-migration-status.py b/tradein-mvp/scripts/probe-migration-status.py new file mode 100644 index 00000000..2bc3e097 --- /dev/null +++ b/tradein-mvp/scripts/probe-migration-status.py @@ -0,0 +1,90 @@ +"""Проверка какие миграции 028..054 НЕ применены.""" +from __future__ import annotations + +import os +import psycopg + +dsn = os.environ["DATABASE_URL"].replace("postgresql+psycopg://", "postgresql://") +conn = psycopg.connect(dsn) +cur = conn.cursor() + +# Маркеры по миграциям (table/column/index/etc) +CHECKS = [ + ("028 matching_tables", "table", "house_sources"), + ("029 extend matching/valuation/dynamics", "column", ("listing_sources", "price_divergence_pct")), + ("030 avito_imv cache_key unique", "index_unique", ("avito_imv_evaluations", "cache_key")), + ("030 listings_alter_yandex", "column", ("listings", "yandex_id")), + ("031 houses_alter_yandex", "column", ("houses", "yandex_id")), + ("032 yandex_history", "table", "yandex_house_history"), + ("040 houses_extend", "column", ("houses", "year_renovated")), + ("041 house_sources noop", "noop", None), + ("042 listing_sources price_divergence_idx", "index", "listing_sources_price_divergence_idx"), + ("043 house_reviews_extend", "column", ("house_reviews", "review_text")), + ("044 external_valuations_link", "column", ("external_valuations", "link_url")), + ("045 house_placement_history_extend", "column", ("house_placement_history", "first_seen_at")), + ("046 views", "view", "v_house_canonical"), + ("050 search_optimization", "matview", "listings_search_mv"), + ("051 scrape_runs_extend", "column", ("scrape_runs", "heartbeat_at")), + ("052 scrape_schedules", "table", "scrape_schedules"), + ("053 scraper_settings", "table", "scraper_settings"), + ("054 scraper_settings_global", "row", ("scraper_settings", "source='global'")), +] + + +def check(kind: str, target) -> bool: + if kind == "noop": + return True + if kind == "table": + cur.execute( + "SELECT 1 FROM information_schema.tables WHERE table_schema='public' AND table_name=%s", + (target,), + ) + elif kind == "view": + cur.execute( + "SELECT 1 FROM information_schema.views WHERE table_schema='public' AND table_name=%s", + (target,), + ) + elif kind == "matview": + cur.execute("SELECT 1 FROM pg_matviews WHERE matviewname=%s", (target,)) + elif kind == "column": + tbl, col = target + cur.execute( + "SELECT 1 FROM information_schema.columns WHERE table_name=%s AND column_name=%s", + (tbl, col), + ) + elif kind == "index": + cur.execute("SELECT 1 FROM pg_indexes WHERE indexname=%s", (target,)) + elif kind == "index_unique": + tbl, col = target + cur.execute( + """SELECT 1 FROM pg_indexes WHERE tablename=%s AND indexdef ~ %s""", + (tbl, f"UNIQUE.*{col}"), + ) + elif kind == "row": + tbl, where = target + cur.execute(f"SELECT 1 FROM {tbl} WHERE {where}") + else: + return False + return cur.fetchone() is not None + + +print(f"{'Migration':<48} Status") +print("-" * 60) +missing = [] +for name, kind, target in CHECKS: + try: + ok = check(kind, target) + except Exception as e: # noqa: BLE001 + ok = False + print(f"{name:<48} ✗ ERROR: {type(e).__name__}: {str(e)[:50]}") + conn.rollback() + missing.append(name) + continue + status = "✓" if ok else "✗ MISSING" + print(f"{name:<48} {status}") + if not ok: + missing.append(name) + +print(f"\nMissing: {len(missing)}/{len(CHECKS)}") +for m in missing: + print(f" - {m}") diff --git a/tradein-mvp/scripts/probe-parser-region-filter.py b/tradein-mvp/scripts/probe-parser-region-filter.py new file mode 100644 index 00000000..e0ef87b5 --- /dev/null +++ b/tradein-mvp/scripts/probe-parser-region-filter.py @@ -0,0 +1,49 @@ +"""Прогнать current parser на HTML из Playwright debug-страницы (своб.планир p=10). + +Проверяет что URL-filter работает: до фикса parser возвращал ~30 карточек других +городов, после — должен 0 или только ЕКБ. +""" +from __future__ import annotations + +import asyncio +import sys +import types +from pathlib import Path + +_BACKEND = Path(__file__).resolve().parent.parent / "backend" +if str(_BACKEND) not in sys.path: + sys.path.insert(0, str(_BACKEND)) +for _m in ("weasyprint", "weasyprint.text", "weasyprint.text.ffi", "weasyprint.css", "weasyprint.css.targets"): + sys.modules[_m] = types.ModuleType(_m) +sys.modules["weasyprint"].HTML = type("HTML", (), {"__init__": lambda self, *a, **kw: None}) +sys.modules["weasyprint"].CSS = type("CSS", (), {"__init__": lambda self, *a, **kw: None}) + +from app.services.scrapers.avito import AvitoScraper # noqa: E402 + +URL = "https://www.avito.ru/ekaterinburg/kvartiry/prodam/svobodnaya_planirovka-ASgBAgICAkSSA8YQygj8zzI?p=10&s=104" + + +async def main() -> None: + async with AvitoScraper() as s: + assert s._cffi is not None + # Use scraper's own session — может попасть в 403 (curl_cffi), это OK, + # тогда переключимся на reading from Playwright MCP page. + print(f"Fetching {URL[:80]}...") + try: + r = await s._cffi.get(URL) + print(f" HTTP {r.status_code}, html size: {len(r.text)}") + if r.status_code != 200: + print(" curl_cffi 403 — Avito не отдал. Пробни через MCP / browser save.") + return + except Exception as e: # noqa: BLE001 + print(f" fetch fail: {type(e).__name__}: {e}") + return + lots = s._parse_html(r.text, source_url_base=URL) + print(f"\nParsed {len(lots)} EKB lots (после фильтра).") + for lot in lots[:5]: + print(f" {lot.source_id} {lot.source_url[:80]}") + print(f" {lot.address or '-'!r}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/tradein-mvp/scripts/test-playwright.py b/tradein-mvp/scripts/test-playwright.py new file mode 100644 index 00000000..fd14afed --- /dev/null +++ b/tradein-mvp/scripts/test-playwright.py @@ -0,0 +1,34 @@ +"""Minimal Playwright test — открывает Chrome окно с Avito на 30 sec.""" +import asyncio +from playwright.async_api import async_playwright + + +async def go(): + async with async_playwright() as p: + print("launching browser...", flush=True) + browser = await p.chromium.launch(headless=False) + print("browser launched, opening page...", flush=True) + page = await browser.new_page() + try: + await page.goto( + "https://www.avito.ru/ekaterinburg/kvartiry/prodam-ASgBAgICAUSSA8YQ?s=104&p=1", + timeout=30000, + ) + except Exception as e: + print(f"goto FAIL: {type(e).__name__}: {e}", flush=True) + await asyncio.sleep(15) + await browser.close() + return + print("page loaded", flush=True) + try: + title = await page.title() + print(f"title: {title}", flush=True) + except Exception as e: + print(f"title FAIL: {e}", flush=True) + print("окно должно быть видно ~30 сек", flush=True) + await asyncio.sleep(30) + await browser.close() + + +if __name__ == "__main__": + asyncio.run(go())