feat(tradein-scripts): scrapers, probes, Phase C improvements
New scripts:
- avito-playwright-sweep.py — Playwright-based Avito sweep (bypasses Qrator)
- backfill-yandex-addresses-ekb.py — fill street-only listings.address с house number
- local-sweep-ekb-{cian,yandex,}.py — full ЕКБ sweep through residential IP
- probe-* — quick diagnostics (region leak, migration status, parser filter, db counts)
- inspect-{houses,search}-html.py — Avito DOM structure analysis
- test-playwright.py — minimal Playwright smoke
Phase C script updates (backfill-house-imv.py):
- Pipe avito_imv module logs to avito-raw.log file (env AVITO_LOG_PATH override)
- Clamp rooms=0 (studio) → 1 — Avito IMV API rejects rooms=0 with HTTP 400
This commit is contained in:
lekss361
parent
5394713bc7
commit
70e45ce5e9
17 changed files with 2546 additions and 1 deletions
371
tradein-mvp/scripts/avito-playwright-sweep.py
Normal file
371
tradein-mvp/scripts/avito-playwright-sweep.py
Normal file
|
|
@ -0,0 +1,371 @@
|
|||
"""Playwright-based Avito sweep — обходит Qrator JS challenge через real headless Chromium.
|
||||
|
||||
Use case: curl_cffi заблочен Qrator anti-DDoS (429 + captcha challenge). Real
|
||||
browser proходит JS challenge автоматически.
|
||||
|
||||
Setup:
|
||||
cd tradein-mvp/backend
|
||||
uv pip install playwright
|
||||
uv run playwright install chromium
|
||||
|
||||
Usage:
|
||||
export DATABASE_URL='postgresql+psycopg://tradein:<PASS>@localhost:15433/tradein'
|
||||
uv run --project tradein-mvp/backend python tradein-mvp/scripts/avito-playwright-sweep.py
|
||||
# options: --pages N --delay S --mode {citywide|byrooms} --headless
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
# Banner FIRST — даже до imports чтобы видно что скрипт стартанул:
|
||||
import sys as _sys
|
||||
|
||||
print("=" * 60, flush=True)
|
||||
print("AVITO PLAYWRIGHT SWEEP — starting...", flush=True)
|
||||
print("=" * 60, flush=True)
|
||||
_sys.stdout.flush()
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import sys
|
||||
import time
|
||||
import types
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlencode
|
||||
|
||||
# Force unbuffered output
|
||||
os.environ["PYTHONUNBUFFERED"] = "1"
|
||||
|
||||
_BACKEND = Path(__file__).resolve().parent.parent / "backend"
|
||||
if str(_BACKEND) not in sys.path:
|
||||
sys.path.insert(0, str(_BACKEND))
|
||||
|
||||
# WeasyPrint stub (Windows без GTK)
|
||||
for _m in ("weasyprint", "weasyprint.text", "weasyprint.text.ffi", "weasyprint.css", "weasyprint.css.targets"):
|
||||
sys.modules[_m] = types.ModuleType(_m)
|
||||
sys.modules["weasyprint"].HTML = type("HTML", (), {"__init__": lambda self, *a, **kw: None})
|
||||
sys.modules["weasyprint"].CSS = type("CSS", (), {"__init__": lambda self, *a, **kw: None})
|
||||
|
||||
from playwright.async_api import Browser, BrowserContext, async_playwright # noqa: E402
|
||||
|
||||
try:
|
||||
from playwright_stealth import Stealth # noqa: E402
|
||||
_STEALTH_AVAILABLE = True
|
||||
except ImportError:
|
||||
_STEALTH_AVAILABLE = False
|
||||
print("[warn] playwright-stealth not installed — без stealth, может triggered больше CAPTCHA", flush=True)
|
||||
|
||||
from app.core.db import SessionLocal # noqa: E402
|
||||
from app.services.scrapers.avito import AvitoScraper # noqa: E402
|
||||
from app.services.scrapers.base import save_listings # noqa: E402
|
||||
|
||||
COOKIES_FILE = Path(__file__).resolve().parent / ".avito-cookies.json"
|
||||
AVITO_BASE = "https://www.avito.ru"
|
||||
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36"
|
||||
|
||||
# Reuse AvitoScraper's parser (DOM cards extraction)
|
||||
_parser_instance = AvitoScraper()
|
||||
_parser_instance._anchor_lat = 0.0 # noqa: SLF001
|
||||
_parser_instance._anchor_lon = 0.0 # noqa: SLF001
|
||||
_parser_instance._anchor_radius_m = 1000 # noqa: SLF001
|
||||
|
||||
|
||||
def load_cookies() -> list[dict]:
|
||||
"""Load DevTools-exported cookies → Playwright format."""
|
||||
if not COOKIES_FILE.exists():
|
||||
print(f"WARN: cookies file not found at {COOKIES_FILE} — будем без cookies")
|
||||
return []
|
||||
raw = json.loads(COOKIES_FILE.read_text(encoding="utf-8"))
|
||||
out = []
|
||||
for c in raw:
|
||||
ck = {
|
||||
"name": c["name"],
|
||||
"value": c["value"],
|
||||
"domain": c["domain"],
|
||||
"path": c.get("path", "/"),
|
||||
"secure": c.get("secure", True),
|
||||
"httpOnly": c.get("httpOnly", False),
|
||||
}
|
||||
if c.get("expirationDate"):
|
||||
ck["expires"] = int(c["expirationDate"])
|
||||
ss = c.get("sameSite")
|
||||
if ss == "no_restriction":
|
||||
ck["sameSite"] = "None"
|
||||
elif ss == "lax":
|
||||
ck["sameSite"] = "Lax"
|
||||
elif ss == "strict":
|
||||
ck["sameSite"] = "Strict"
|
||||
out.append(ck)
|
||||
return out
|
||||
|
||||
|
||||
def build_citywide_url(page: int) -> str:
|
||||
return f"{AVITO_BASE}/ekaterinburg/kvartiry/prodam-ASgBAgICAUSSA8YQ?{urlencode({'s':104,'p':page})}"
|
||||
|
||||
|
||||
ROOM_SLUGS: list[tuple[str, str]] = [
|
||||
("студии", "studii-ASgBAgICAkSSA8YQygj~WA"),
|
||||
("1-комн.", "1-komnatnye-ASgBAgICAkSSA8YQygiAWQ"),
|
||||
("2-комн.", "2-komnatnye-ASgBAgICAkSSA8YQygiCWQ"),
|
||||
("3-комн.", "3-komnatnye-ASgBAgICAkSSA8YQygiEWQ"),
|
||||
("4-комн.", "4-komnatnye-ASgBAgICAkSSA8YQygiGWQ"),
|
||||
("5+комн.", "5-komnatnye-ASgBAgICAkSSA8YQygiIWQ"),
|
||||
("своб.планир.", "svobodnaya_planirovka-ASgBAgICAkSSA8YQygj8zzI"),
|
||||
]
|
||||
|
||||
|
||||
def build_rooms_url(slug: str, page: int) -> str:
|
||||
return f"{AVITO_BASE}/ekaterinburg/kvartiry/prodam/{slug}?{urlencode({'s':104,'p':page})}"
|
||||
|
||||
|
||||
async def fetch_page(context: BrowserContext, url: str) -> tuple[int, str | None]:
|
||||
"""Open page in Playwright, wait for content, return (status, html or None).
|
||||
|
||||
Strategy:
|
||||
1. goto с wait_until='domcontentloaded' (быстро, не ждём всех scripts)
|
||||
2. Если status 200 → wait_for_selector data-marker=item (Qrator challenge должен resolve пока)
|
||||
3. Иначе если 429/403 — challenge не прошёл, дать ещё 5s + retry content (maybe challenge resolves async)
|
||||
"""
|
||||
page = await context.new_page()
|
||||
try:
|
||||
try:
|
||||
response = await page.goto(url, wait_until="domcontentloaded", timeout=45000)
|
||||
except Exception as e: # noqa: BLE001
|
||||
print(f" goto FAIL: {type(e).__name__}: {str(e)[:80]}")
|
||||
return 0, None
|
||||
status = response.status if response else 0
|
||||
if status in (429, 403):
|
||||
# Qrator challenge — wait для JS auto-resolve, потом reload
|
||||
await asyncio.sleep(5)
|
||||
try:
|
||||
response = await page.reload(wait_until="domcontentloaded", timeout=30000)
|
||||
status = response.status if response else status
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
if status != 200:
|
||||
return status, None
|
||||
# Wait for item cards
|
||||
try:
|
||||
await page.wait_for_selector('[data-marker="item"]', timeout=15000)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
html = await page.content()
|
||||
return status, html
|
||||
finally:
|
||||
await page.close()
|
||||
|
||||
|
||||
async def sweep_urls(
|
||||
urls_with_label: list[tuple[str, str]],
|
||||
delay: float,
|
||||
headless: bool,
|
||||
) -> int:
|
||||
"""Generic sweep: list of (label, url) → fetch + parse + save per URL."""
|
||||
cookies = load_cookies()
|
||||
print(f"loaded {len(cookies)} cookies")
|
||||
total_fetched = 0
|
||||
total_ins = 0
|
||||
total_upd = 0
|
||||
challenge_hits = 0
|
||||
print("=== Launching Playwright browser ===", flush=True)
|
||||
async with async_playwright() as p:
|
||||
print(f" headless={headless}", flush=True)
|
||||
browser: Browser = await p.chromium.launch(
|
||||
headless=headless,
|
||||
args=[
|
||||
"--disable-blink-features=AutomationControlled",
|
||||
"--no-sandbox",
|
||||
],
|
||||
)
|
||||
print(" browser launched, creating context...", flush=True)
|
||||
context: BrowserContext = await browser.new_context(
|
||||
user_agent=UA,
|
||||
locale="ru-RU",
|
||||
timezone_id="Asia/Yekaterinburg",
|
||||
viewport={"width": 1366, "height": 900},
|
||||
)
|
||||
# Stealth — optional, skip on error
|
||||
if _STEALTH_AVAILABLE:
|
||||
try:
|
||||
stealth = Stealth()
|
||||
await stealth.apply_stealth_async(context)
|
||||
print(" stealth applied", flush=True)
|
||||
except Exception as e: # noqa: BLE001
|
||||
print(f" stealth skipped (err): {type(e).__name__}: {e}", flush=True)
|
||||
if cookies:
|
||||
await context.add_cookies(cookies)
|
||||
print(f" cookies added ({len(cookies)})", flush=True)
|
||||
|
||||
# Initial warmup: open main Avito + wait для user pass CAPTCHA если нужно
|
||||
print("\n=== WARMUP: opening Avito initial page ===", flush=True)
|
||||
print(" creating new page tab...", flush=True)
|
||||
warmup_page = await context.new_page()
|
||||
print(" navigating to Avito (timeout 30s)...", flush=True)
|
||||
try:
|
||||
await warmup_page.goto(
|
||||
"https://www.avito.ru/ekaterinburg/kvartiry/prodam-ASgBAgICAUSSA8YQ?s=104&p=1",
|
||||
wait_until="domcontentloaded",
|
||||
timeout=30000,
|
||||
)
|
||||
print(" goto done, reading content...", flush=True)
|
||||
except Exception as e: # noqa: BLE001
|
||||
print(f" goto FAIL: {type(e).__name__}: {e}", flush=True)
|
||||
# Check if challenge page — reliable detector: NO listing cards в HTML.
|
||||
# Слово "captcha" может быть в JS embed страницы normally (false positive).
|
||||
warmup_html = await warmup_page.content()
|
||||
print(f" content read: {len(warmup_html)} bytes", flush=True)
|
||||
try:
|
||||
await warmup_page.wait_for_selector('[data-marker="item"]', timeout=5000)
|
||||
has_items = True
|
||||
except Exception: # noqa: BLE001
|
||||
has_items = False
|
||||
is_qrator = "qrator" in warmup_html.lower() and len(warmup_html) < 100000
|
||||
challenge_present = (not has_items) and (is_qrator or len(warmup_html) < 50000)
|
||||
print(f" has_items={has_items} qrator={is_qrator} size={len(warmup_html)} challenge={challenge_present}", flush=True)
|
||||
if challenge_present:
|
||||
print("\n🚨 CAPTCHA detected на initial page!")
|
||||
print("👉 Окно браузера открыто — пройди капчу руками,")
|
||||
print("👉 затем нажми Enter здесь чтобы продолжить sweep...")
|
||||
try:
|
||||
input(">>> Press ENTER when ready: ")
|
||||
except (EOFError, KeyboardInterrupt):
|
||||
print("aborted")
|
||||
await browser.close()
|
||||
return 0
|
||||
# Verify challenge passed
|
||||
await warmup_page.reload(wait_until="domcontentloaded", timeout=30000)
|
||||
new_html = await warmup_page.content()
|
||||
if any(m in new_html.lower() for m in ("captcha", "qrator")):
|
||||
print("⚠️ всё ещё CAPTCHA — повторно пройди или Ctrl+C")
|
||||
input(">>> Retry ENTER: ")
|
||||
else:
|
||||
print("✅ CAPTCHA пройдена, продолжаю sweep")
|
||||
else:
|
||||
print("✅ Initial page OK (no challenge)")
|
||||
await warmup_page.close()
|
||||
|
||||
try:
|
||||
# Per-category skip flag: 0 lots на странице категории = end of pagination
|
||||
# ДЛЯ ТЕКУЩЕЙ КАТЕГОРИИ, а не для всего sweep'а (был bug: break ломал всё).
|
||||
current_cat: str | None = None
|
||||
skip_until_next_cat = False
|
||||
for i, (label, url) in enumerate(urls_with_label, 1):
|
||||
# Категория = всё до " page=" в label ("1-комн. page=5/60" → "1-комн.")
|
||||
cat = label.split(" page=")[0] if " page=" in label else label
|
||||
if cat != current_cat:
|
||||
current_cat = cat
|
||||
skip_until_next_cat = False
|
||||
challenge_hits = 0 # reset 429-counter на границе категорий
|
||||
if skip_until_next_cat:
|
||||
continue
|
||||
|
||||
print(f"[{i}/{len(urls_with_label)}] {label}", flush=True)
|
||||
status, html = await fetch_page(context, url)
|
||||
if status != 200 or not html:
|
||||
print(f" HTTP {status} → skip", flush=True)
|
||||
if status in (429, 403):
|
||||
challenge_hits += 1
|
||||
if challenge_hits >= 3:
|
||||
print(f" ⚠️ {challenge_hits}x challenge hits — sleep 60s extra", flush=True)
|
||||
print(" если CAPTCHA в окне — пройди и нажми ENTER:", flush=True)
|
||||
try:
|
||||
input(" >>> ")
|
||||
except (EOFError, KeyboardInterrupt):
|
||||
break
|
||||
challenge_hits = 0
|
||||
await asyncio.sleep(delay)
|
||||
continue
|
||||
# Parse + save
|
||||
lots = _parser_instance._parse_html(html, source_url_base=url) # noqa: SLF001
|
||||
if not lots:
|
||||
print(f" 0 lots — end of pagination for {cat!r}", flush=True)
|
||||
if " page=" in label:
|
||||
# 0 lots на пагинированной странице = end of category, не sweep
|
||||
skip_until_next_cat = True
|
||||
continue
|
||||
db = SessionLocal()
|
||||
try:
|
||||
ins, upd = save_listings(db, lots)
|
||||
total_fetched += len(lots)
|
||||
total_ins += ins
|
||||
total_upd += upd
|
||||
print(
|
||||
f" {len(lots)} lots → ins={ins} upd={upd} "
|
||||
f"[cum: tot={total_fetched} new={total_ins}]",
|
||||
flush=True,
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
print(f" save FAIL: {type(e).__name__}: {e}", flush=True)
|
||||
finally:
|
||||
db.close()
|
||||
# Jittered delay
|
||||
if i < len(urls_with_label):
|
||||
j = delay * random.uniform(0.7, 1.3)
|
||||
await asyncio.sleep(j)
|
||||
finally:
|
||||
await context.close()
|
||||
await browser.close()
|
||||
print(
|
||||
f"\n=== TOTAL: fetched={total_fetched} ins={total_ins} upd={total_upd} ===",
|
||||
flush=True,
|
||||
)
|
||||
return total_fetched
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Diagnostic — show env vars used:
|
||||
db_url = os.environ.get("DATABASE_URL", "<NOT SET>")
|
||||
# Mask password for logging
|
||||
if "://" in db_url and "@" in db_url:
|
||||
before_at, after_at = db_url.rsplit("@", 1)
|
||||
scheme_user = before_at.split("://", 1)
|
||||
if len(scheme_user) == 2 and ":" in scheme_user[1]:
|
||||
user = scheme_user[1].split(":", 1)[0]
|
||||
masked = f"{scheme_user[0]}://{user}:***@{after_at}"
|
||||
else:
|
||||
masked = db_url[:50] + "..."
|
||||
else:
|
||||
masked = db_url
|
||||
print(f"DATABASE_URL: {masked}", flush=True)
|
||||
print(f"ENVIRONMENT: {os.environ.get('ENVIRONMENT', '<default>')}", flush=True)
|
||||
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--mode", choices=["citywide", "byrooms"], default="citywide")
|
||||
p.add_argument("--pages", type=int, default=100, help="pages per query")
|
||||
p.add_argument("--delay", type=float, default=7, help="sec между requests")
|
||||
p.add_argument("--headless", action="store_true", default=True)
|
||||
p.add_argument("--no-headless", dest="headless", action="store_false")
|
||||
p.add_argument(
|
||||
"--start-cat",
|
||||
type=int,
|
||||
default=1,
|
||||
help="byrooms: с какой категории начать (1-based). 1=студии, 2=1-комн, "
|
||||
"3=2-комн, 4=3-комн, 5=4-комн, 6=5+комн, 7=своб.планир.",
|
||||
)
|
||||
args = p.parse_args()
|
||||
|
||||
urls: list[tuple[str, str]] = []
|
||||
if args.mode == "citywide":
|
||||
for page in range(1, args.pages + 1):
|
||||
urls.append((f"citywide page={page}/{args.pages}", build_citywide_url(page)))
|
||||
else: # byrooms
|
||||
cats = ROOM_SLUGS[args.start_cat - 1:]
|
||||
if not cats:
|
||||
print(f"ERROR: --start-cat={args.start_cat} > {len(ROOM_SLUGS)} cats", flush=True)
|
||||
return
|
||||
print(f"byrooms: starting from cat #{args.start_cat} ({cats[0][0]!r}), "
|
||||
f"{len(cats)} cats total", flush=True)
|
||||
for cat_name, slug in cats:
|
||||
for page in range(1, args.pages + 1):
|
||||
urls.append((f"{cat_name} page={page}/{args.pages}", build_rooms_url(slug, page)))
|
||||
|
||||
print(f"Mode: {args.mode}, pages: {args.pages}, delay: {args.delay}s, total URLs: {len(urls)}")
|
||||
print(f"Estimated time: {len(urls) * args.delay / 60:.0f} min (без challenge)")
|
||||
t0 = time.perf_counter()
|
||||
await sweep_urls(urls, args.delay, args.headless)
|
||||
print(f"\nDONE in {(time.perf_counter() - t0) / 60:.1f} min")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
|
|
@ -51,6 +51,19 @@ logging.basicConfig(
|
|||
)
|
||||
logger = logging.getLogger("phase-c")
|
||||
|
||||
# Avito raw log — pipe everything from app.services.scrapers.avito_imv to a file
|
||||
# so we keep full request/response history for diagnosing contract changes.
|
||||
_AVITO_LOG_PATH = os.environ.get("AVITO_LOG_PATH", "avito-raw.log")
|
||||
_avito_fh = logging.FileHandler(_AVITO_LOG_PATH, encoding="utf-8")
|
||||
_avito_fh.setFormatter(logging.Formatter("%(asctime)s %(levelname)s [%(name)s] %(message)s"))
|
||||
_avito_fh.setLevel(logging.DEBUG)
|
||||
_avito_logger = logging.getLogger("app.services.scrapers.avito_imv")
|
||||
_avito_logger.addHandler(_avito_fh)
|
||||
_avito_logger.setLevel(logging.DEBUG)
|
||||
# Also pipe phase-c progress lines to the file
|
||||
logger.addHandler(_avito_fh)
|
||||
logger.info("avito raw log writing to %s", os.path.abspath(_AVITO_LOG_PATH))
|
||||
|
||||
# Avito IMV API accepts only these house_type values (verified 2026-05-24).
|
||||
# Our `listings.house_type` may contain pre-2026 values like 'monolith' / 'monolith_brick'.
|
||||
_HOUSE_TYPE_TO_IMV = {
|
||||
|
|
@ -147,8 +160,10 @@ def pick_lot_params(db: Session, house_id: int) -> dict:
|
|||
{"hid": house_id},
|
||||
).mappings().first()
|
||||
|
||||
# Avito IMV: rooms=0 (студия) rejected with HTTP 400. Clamp to 1.
|
||||
rooms_raw = int(row["rooms"])
|
||||
return {
|
||||
"rooms": int(row["rooms"]),
|
||||
"rooms": max(rooms_raw, 1),
|
||||
"area_m2": float(row["area_m2"]),
|
||||
"floor": int(row["floor"] or 1),
|
||||
"floor_at_home": int(row["total_floors"] or (house and house["total_floors"]) or 9),
|
||||
|
|
|
|||
173
tradein-mvp/scripts/backfill-yandex-addresses-ekb.py
Normal file
173
tradein-mvp/scripts/backfill-yandex-addresses-ekb.py
Normal file
|
|
@ -0,0 +1,173 @@
|
|||
"""Backfill listings.address with full address (street + house number) for
|
||||
active EKB yandex listings that currently have street-only addresses.
|
||||
|
||||
Why: Yandex SERP returns only street (e.g. 'улица Горького') in card snippet.
|
||||
Detail page <title> has the full address ('Екатеринбург, улица Горького, 36').
|
||||
With house number we can call /otsenka-kvartiry-po-adresu-onlayn/ and get the
|
||||
building's history.
|
||||
|
||||
Usage:
|
||||
cd tradein-mvp/backend
|
||||
uv run python ../scripts/backfill-yandex-addresses-ekb.py --limit 5 # smoke
|
||||
uv run python ../scripts/backfill-yandex-addresses-ekb.py # full
|
||||
uv run python ../scripts/backfill-yandex-addresses-ekb.py --dry-run # no DB write
|
||||
|
||||
Requires SSH tunnel + DATABASE_URL env (see sweep-yandex-valuation-ekb.py docstring).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import random
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import types
|
||||
from pathlib import Path
|
||||
|
||||
# sys.path injection
|
||||
_BACKEND_DIR = Path(__file__).resolve().parent.parent / "backend"
|
||||
if str(_BACKEND_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(_BACKEND_DIR))
|
||||
|
||||
# Stub weasyprint (Windows GTK-less)
|
||||
for _m in (
|
||||
"weasyprint",
|
||||
"weasyprint.text",
|
||||
"weasyprint.text.ffi",
|
||||
"weasyprint.css",
|
||||
"weasyprint.css.targets",
|
||||
):
|
||||
sys.modules[_m] = types.ModuleType(_m)
|
||||
sys.modules["weasyprint"].HTML = type("HTML", (), {"__init__": lambda self, *a, **kw: None})
|
||||
sys.modules["weasyprint"].CSS = type("CSS", (), {"__init__": lambda self, *a, **kw: None})
|
||||
|
||||
from curl_cffi.requests import AsyncSession # noqa: E402
|
||||
from sqlalchemy import text # noqa: E402
|
||||
|
||||
from app.core.db import SessionLocal # noqa: E402
|
||||
|
||||
# Extract '<City>, <addr> — id <number>' from <title>
|
||||
RE_TITLE_ADDRESS = re.compile(
|
||||
r"<title>[^<]*?—\s*"
|
||||
r"(?:[^—,]*?,\s*)?" # optional ЖК/жилой комплекс prefix
|
||||
r"(Екатеринбург,\s*[^<]+?)"
|
||||
r"\s*—\s*id\s*\d+\s*</title>",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _eligible_listings(limit: int | None) -> list[dict]:
|
||||
"""Active EKB yandex listings without house number in address."""
|
||||
db = SessionLocal()
|
||||
try:
|
||||
sql = text(
|
||||
"""
|
||||
SELECT id, source_id, address, source_url
|
||||
FROM listings
|
||||
WHERE source = 'yandex'
|
||||
AND region_code = 66
|
||||
AND is_active = TRUE
|
||||
AND address IS NOT NULL
|
||||
AND NOT (address ~ ',\\s*\\d+')
|
||||
AND source_url IS NOT NULL
|
||||
ORDER BY scraped_at DESC
|
||||
"""
|
||||
)
|
||||
rows = db.execute(sql).mappings().all()
|
||||
out = [dict(r) for r in rows]
|
||||
finally:
|
||||
db.close()
|
||||
if limit is not None and limit > 0:
|
||||
out = out[:limit]
|
||||
return out
|
||||
|
||||
|
||||
def _update_listing_address(listing_id: int, new_address: str) -> None:
|
||||
db = SessionLocal()
|
||||
try:
|
||||
db.execute(
|
||||
text("UPDATE listings SET address = :addr WHERE id = :id"),
|
||||
{"addr": new_address, "id": listing_id},
|
||||
)
|
||||
db.commit()
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
async def _fetch_address(session: AsyncSession, url: str) -> str | None:
|
||||
"""Fetch detail page, extract full address from <title>."""
|
||||
try:
|
||||
r = await session.get(url, timeout=30)
|
||||
except Exception as e:
|
||||
print(f" fetch FAIL: {type(e).__name__}: {e}", flush=True)
|
||||
return None
|
||||
if r.status_code != 200:
|
||||
print(f" HTTP {r.status_code}", flush=True)
|
||||
return None
|
||||
html = r.text.replace("\xa0", " ")
|
||||
m = RE_TITLE_ADDRESS.search(html)
|
||||
if not m:
|
||||
return None
|
||||
addr = m.group(1).strip().rstrip(",").strip()
|
||||
return addr if addr else None
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
p = argparse.ArgumentParser(
|
||||
description="Backfill yandex listings.address with house number from detail page"
|
||||
)
|
||||
p.add_argument("--limit", type=int, default=None, help="Cap items (smoke testing)")
|
||||
p.add_argument("--delay", type=float, default=3.0, help="Sec between requests")
|
||||
p.add_argument("--dry-run", action="store_true", help="Don't UPDATE DB")
|
||||
args = p.parse_args()
|
||||
|
||||
items = _eligible_listings(args.limit)
|
||||
if not items:
|
||||
print("No eligible listings.", flush=True)
|
||||
return
|
||||
|
||||
print(
|
||||
f"=== Backfill {len(items)} listings (delay={args.delay}s dry_run={args.dry_run}) ===",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
t0 = time.perf_counter()
|
||||
enriched = 0
|
||||
skipped = 0
|
||||
failed = 0
|
||||
|
||||
async with AsyncSession(impersonate="chrome120") as session:
|
||||
for i, item in enumerate(items, 1):
|
||||
url = item["source_url"]
|
||||
old_addr = item["address"]
|
||||
print(
|
||||
f"[{i}/{len(items)}] id={item['id']} old={old_addr!r}",
|
||||
flush=True,
|
||||
)
|
||||
new_addr = await _fetch_address(session, url)
|
||||
if new_addr is None:
|
||||
print(" -> no address extracted", flush=True)
|
||||
failed += 1
|
||||
elif new_addr == old_addr:
|
||||
print(" -> unchanged", flush=True)
|
||||
skipped += 1
|
||||
else:
|
||||
print(f" -> {new_addr!r}", flush=True)
|
||||
if not args.dry_run:
|
||||
_update_listing_address(item["id"], new_addr)
|
||||
enriched += 1
|
||||
if i < len(items):
|
||||
await asyncio.sleep(args.delay * random.uniform(0.85, 1.3))
|
||||
|
||||
elapsed = time.perf_counter() - t0
|
||||
print(
|
||||
f"\n=== DONE in {elapsed / 60:.1f} min. "
|
||||
f"enriched={enriched} skipped={skipped} failed={failed} ===",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
90
tradein-mvp/scripts/inspect-houses-html.py
Normal file
90
tradein-mvp/scripts/inspect-houses-html.py
Normal file
|
|
@ -0,0 +1,90 @@
|
|||
"""One-off: fetch Avito houses HTML и анализ markers для понимания новой структуры.
|
||||
|
||||
После 2026-05-23 Avito поменяли rendering — старый `window.__preloadedState__`
|
||||
больше не работает (Stage 2c parser fails). Этот скрипт fetch'ит один houses
|
||||
catalog page локально через residential IP и сохраняет HTML + проверяет markers
|
||||
чтобы понять new format (__NEXT_DATA__? React SSR? иной).
|
||||
"""
|
||||
import sys
|
||||
import types
|
||||
from pathlib import Path
|
||||
|
||||
_BACKEND = Path(__file__).resolve().parent.parent / "backend"
|
||||
if str(_BACKEND) not in sys.path:
|
||||
sys.path.insert(0, str(_BACKEND))
|
||||
|
||||
for _m in ("weasyprint", "weasyprint.text", "weasyprint.text.ffi", "weasyprint.css", "weasyprint.css.targets"):
|
||||
sys.modules[_m] = types.ModuleType(_m)
|
||||
sys.modules["weasyprint"].HTML = type("HTML", (), {"__init__": lambda self, *a, **kw: None})
|
||||
sys.modules["weasyprint"].CSS = type("CSS", (), {"__init__": lambda self, *a, **kw: None})
|
||||
|
||||
import asyncio # noqa: E402
|
||||
|
||||
from curl_cffi.requests import AsyncSession # noqa: E402
|
||||
|
||||
URLS = [
|
||||
"https://www.avito.ru/catalog/houses/ekaterinburg/ul_yumasheva_6/115188",
|
||||
"https://www.avito.ru/catalog/houses/ekaterinburg/ul_akademika_postovskogo_17a/3171365",
|
||||
]
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
async with AsyncSession(
|
||||
impersonate="chrome120",
|
||||
timeout=25,
|
||||
headers={
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "ru-RU,ru;q=0.9",
|
||||
"Sec-Fetch-Dest": "document",
|
||||
"Sec-Fetch-Mode": "navigate",
|
||||
"Sec-Fetch-Site": "none",
|
||||
},
|
||||
) as s:
|
||||
for url in URLS:
|
||||
print(f"\n=== {url} ===")
|
||||
try:
|
||||
r = await s.get(url)
|
||||
except Exception as e: # noqa: BLE001
|
||||
print(f" FAIL fetch: {type(e).__name__}: {e}")
|
||||
continue
|
||||
print(f" status={r.status_code} size={len(r.content)}")
|
||||
if r.status_code != 200:
|
||||
continue
|
||||
text = r.text
|
||||
# Save first one для inspection
|
||||
if "yumasheva" in url:
|
||||
with Path("/tmp/avito-house-sample.html").open("wb") as f:
|
||||
f.write(r.content)
|
||||
print(" saved → /tmp/avito-house-sample.html")
|
||||
markers = {
|
||||
"__preloadedState__": "__preloadedState__" in text,
|
||||
"__NEXT_DATA__": "__NEXT_DATA__" in text,
|
||||
"__INITIAL_STATE__": "__INITIAL_STATE__" in text,
|
||||
"datadome": "datadome" in text.lower(),
|
||||
"captcha": "captcha" in text.lower(),
|
||||
"data-marker": "data-marker" in text,
|
||||
"data-app-state": "data-app-state" in text,
|
||||
"developmentData": "developmentData" in text,
|
||||
"avitoId": "avitoId" in text,
|
||||
"window.__": "window.__" in text,
|
||||
"<script id=": "<script id=" in text,
|
||||
"ssr-state": "ssr-state" in text,
|
||||
"__NUXT__": "__NUXT__" in text,
|
||||
"Не найдено": "Не найдено" in text,
|
||||
"недоступна": "недоступна" in text,
|
||||
}
|
||||
for k, v in markers.items():
|
||||
print(f" {'✓' if v else '✗'} {k}")
|
||||
# Look for script id="..." patterns (Next.js typical)
|
||||
import re
|
||||
script_ids = re.findall(r'<script[^>]*id="([^"]+)"', text)
|
||||
if script_ids:
|
||||
print(f" script IDs found: {script_ids[:10]}")
|
||||
data_attrs = re.findall(r'<\w+\s+(data-[\w-]+)=', text)
|
||||
if data_attrs:
|
||||
unique = sorted(set(data_attrs))[:15]
|
||||
print(f" data attrs sample: {unique}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
55
tradein-mvp/scripts/inspect-search-html.py
Normal file
55
tradein-mvp/scripts/inspect-search-html.py
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
"""Fetch Avito search page и анализ URL slugs для room-filter."""
|
||||
import re
|
||||
import sys
|
||||
import types
|
||||
from pathlib import Path
|
||||
|
||||
_BACKEND = Path(__file__).resolve().parent.parent / "backend"
|
||||
if str(_BACKEND) not in sys.path:
|
||||
sys.path.insert(0, str(_BACKEND))
|
||||
|
||||
for _m in ("weasyprint", "weasyprint.text", "weasyprint.text.ffi", "weasyprint.css", "weasyprint.css.targets"):
|
||||
sys.modules[_m] = types.ModuleType(_m)
|
||||
sys.modules["weasyprint"].HTML = type("HTML", (), {"__init__": lambda self, *a, **kw: None})
|
||||
sys.modules["weasyprint"].CSS = type("CSS", (), {"__init__": lambda self, *a, **kw: None})
|
||||
|
||||
import asyncio # noqa: E402
|
||||
|
||||
from curl_cffi.requests import AsyncSession # noqa: E402
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
url = "https://www.avito.ru/ekaterinburg/kvartiry/prodam-ASgBAgICAUSSA8YQ?s=104&p=1"
|
||||
async with AsyncSession(
|
||||
impersonate="chrome120",
|
||||
headers={
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "ru-RU,ru;q=0.9",
|
||||
"Sec-Fetch-Dest": "document",
|
||||
"Sec-Fetch-Mode": "navigate",
|
||||
"Sec-Fetch-Site": "none",
|
||||
},
|
||||
) as s:
|
||||
r = await s.get(url)
|
||||
print(f"status={r.status_code} size={len(r.content)}")
|
||||
text = r.text
|
||||
with Path("/tmp/avito-search-sample.html").open("wb") as f:
|
||||
f.write(r.content)
|
||||
print("saved → /tmp/avito-search-sample.html")
|
||||
# Find filter links — pattern /ekaterinburg/kvartiry/prodam/<slug>-ASg...
|
||||
links = re.findall(r'href="(/ekaterinburg/kvartiry/prodam/[^"]+)"', text)
|
||||
unique = sorted(set(links))
|
||||
print(f"\nUnique filter links ({len(unique)} found):")
|
||||
for link in unique[:30]:
|
||||
# Decode shortened
|
||||
if "komnatn" in link.lower() or "studii" in link.lower() or "svobodnaya" in link.lower():
|
||||
print(f" ROOMS: {link}")
|
||||
else:
|
||||
print(f" other: {link[:120]}")
|
||||
# Also check direct headings
|
||||
room_headings = re.findall(r'>(Студии?|[0-9]-[Кк]омнатн\w+|Свободная планировка)<', text)
|
||||
print(f"\nRoom headings in HTML: {sorted(set(room_headings))}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
359
tradein-mvp/scripts/local-sweep-ekb-cian.py
Normal file
359
tradein-mvp/scripts/local-sweep-ekb-cian.py
Normal file
|
|
@ -0,0 +1,359 @@
|
|||
"""Cian ЕКБ vtorichka sweep — local residential IP → prod tradein DB через SSH tunnel.
|
||||
|
||||
Why: ekb.cian.ru SERP отдаёт ВЕСЬ город (без geo-фильтра по lat/lon), поэтому
|
||||
"10 anchors" из Avito-эталона тут бесполезны — одинаковый ответ × 10.
|
||||
Вместо anchors — proper room-segmented citywide pagination
|
||||
(rooms 1/2/3/4 × N pages × ~28 listings), per-page save (visible progress +
|
||||
не теряем данные при крэше).
|
||||
|
||||
Cian реально менее агрессивен чем Avito, но **residential IP всё равно может попасть
|
||||
в blacklist при слишком частых запросах** — default delay 15s.
|
||||
|
||||
Setup (один раз):
|
||||
ssh -f -N -L 15433:172.20.0.2:5432 gendesign
|
||||
cd tradein-mvp/backend
|
||||
$env:DATABASE_URL='postgresql+psycopg://tradein:<PASS>@localhost:15433/tradein'
|
||||
$env:ENVIRONMENT='local'; $env:GLITCHTIP_DSN=''
|
||||
|
||||
Usage:
|
||||
uv run python ../scripts/local-sweep-ekb-cian.py # default: 4 rooms × 8 pages
|
||||
uv run python ../scripts/local-sweep-ekb-cian.py --pages 20 # больше глубины
|
||||
uv run python ../scripts/local-sweep-ekb-cian.py --delay 10 # быстрее (risk)
|
||||
uv run python ../scripts/local-sweep-ekb-cian.py --rooms 0 # всё (включая студии — no room param)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
# stdlib
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
import types
|
||||
from pathlib import Path
|
||||
|
||||
# Auto-inject backend/ в sys.path чтобы найти app.* package
|
||||
_BACKEND_DIR = Path(__file__).resolve().parent.parent / "backend"
|
||||
if str(_BACKEND_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(_BACKEND_DIR))
|
||||
|
||||
# Stub weasyprint (Windows без GTK ломает import chain через trade_in.py)
|
||||
for _m in (
|
||||
"weasyprint",
|
||||
"weasyprint.text",
|
||||
"weasyprint.text.ffi",
|
||||
"weasyprint.css",
|
||||
"weasyprint.css.targets",
|
||||
):
|
||||
sys.modules[_m] = types.ModuleType(_m)
|
||||
sys.modules["weasyprint"].HTML = type("HTML", (), {"__init__": lambda self, *a, **kw: None})
|
||||
sys.modules["weasyprint"].CSS = type("CSS", (), {"__init__": lambda self, *a, **kw: None})
|
||||
|
||||
# app imports после stubs
|
||||
import logging # noqa: E402
|
||||
import re # noqa: E402
|
||||
|
||||
from app.core.db import SessionLocal # noqa: E402
|
||||
from app.services.scrapers.base import ScrapedLot, save_listings # noqa: E402
|
||||
from app.services.scrapers.cian import CianScraper # noqa: E402
|
||||
|
||||
# Шумный WARNING из upstream cian_state_parser ('state extraction failed') — у нас
|
||||
# fallback на новый .concat() markup отрабатывает, warning нерелевантен.
|
||||
logging.getLogger("app.services.scrapers.cian").setLevel(logging.ERROR)
|
||||
|
||||
# ── New-markup fallback parser ───────────────────────────────────────────────
|
||||
# Cian переключил MFE bootstrap с .push({...}) на .concat([{...},{...}]):
|
||||
# window._cianConfig['frontend-serp'] = (window._cianConfig['frontend-serp'] || []).concat([…])
|
||||
# Существующий cian_state_parser.py не ловит этот шаблон → 0 lots на свежей выдаче.
|
||||
# Чтобы не трогать shared scraper module (правилами запрещено), парсим тут.
|
||||
_RE_CONCAT_START = re.compile(
|
||||
r"window\._cianConfig\['(?P<mfe>[\w-]+)'\]\s*=\s*"
|
||||
r"\(window\._cianConfig\['[\w-]+'\]\s*\|\|\s*\[\]\)\.concat\(\["
|
||||
)
|
||||
|
||||
|
||||
def _extract_balanced_array(text: str, start: int) -> str | None:
|
||||
"""Из позиции `start` (где '[') вернуть текст массива включая внешние '[' и ']'.
|
||||
|
||||
JSON-aware: учитывает strings и escapes, чтобы не сбить счётчик '[' ']' на
|
||||
скобках внутри string literals.
|
||||
"""
|
||||
if text[start] != "[":
|
||||
return None
|
||||
depth = 0
|
||||
i = start
|
||||
in_str = False
|
||||
esc = False
|
||||
while i < len(text):
|
||||
ch = text[i]
|
||||
if in_str:
|
||||
if esc:
|
||||
esc = False
|
||||
elif ch == "\\":
|
||||
esc = True
|
||||
elif ch == '"':
|
||||
in_str = False
|
||||
else:
|
||||
if ch == '"':
|
||||
in_str = True
|
||||
elif ch == "[":
|
||||
depth += 1
|
||||
elif ch == "]":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
return text[start : i + 1]
|
||||
i += 1
|
||||
return None
|
||||
|
||||
|
||||
def _parse_serp_v2(scraper: CianScraper, html: str) -> list[ScrapedLot]:
|
||||
"""Fallback парсер для нового шаблона .concat([...]) — ekb.cian.ru/cat.php c 2026-05."""
|
||||
state: dict | None = None
|
||||
for m in _RE_CONCAT_START.finditer(html):
|
||||
if m.group("mfe") != "frontend-serp":
|
||||
continue
|
||||
# Позиция '[' — это последний символ матча минус 1 (regex кончается на '[')
|
||||
arr_start = m.end() - 1
|
||||
arr_str = _extract_balanced_array(html, arr_start)
|
||||
if arr_str is None:
|
||||
continue
|
||||
try:
|
||||
entries = json.loads(arr_str)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
for entry in entries:
|
||||
if (
|
||||
isinstance(entry, dict)
|
||||
and entry.get("key") == "initialState"
|
||||
and isinstance(entry.get("value"), dict)
|
||||
):
|
||||
state = entry["value"]
|
||||
break
|
||||
if state is not None:
|
||||
break
|
||||
if state is None:
|
||||
return []
|
||||
offers = (state.get("results") or {}).get("offers") or []
|
||||
lots: list[ScrapedLot] = []
|
||||
for offer in offers:
|
||||
# Cian свежий markup: descriptionMinhash = list[int] (multi-hash signatures).
|
||||
# ScrapedLot ожидает str → сбрасываем чтобы _offer_to_lot вычислил SHA1 от description.
|
||||
if isinstance(offer.get("descriptionMinhash"), list):
|
||||
offer["descriptionMinhash"] = None
|
||||
lot = scraper._offer_to_lot(offer)
|
||||
if lot is not None:
|
||||
lots.append(lot)
|
||||
return lots
|
||||
|
||||
|
||||
def _load_cookies_file(path: Path) -> dict[str, str]:
|
||||
"""Загрузить cookies из JSON-файла.
|
||||
|
||||
Поддерживает два формата:
|
||||
- browser export (list[{name,value,...}]) — DevTools / EditThisCookie / Cookie-Editor
|
||||
- plain dict {name: value}
|
||||
"""
|
||||
raw = json.loads(path.read_text(encoding="utf-8"))
|
||||
if isinstance(raw, list):
|
||||
return {c["name"]: c["value"] for c in raw if c.get("name") and c.get("value")}
|
||||
if isinstance(raw, dict):
|
||||
return {str(k): str(v) for k, v in raw.items()}
|
||||
raise SystemExit(f"--cookies-file: unknown JSON shape (got {type(raw).__name__})")
|
||||
|
||||
|
||||
async def phase_citywide(
|
||||
pages: int,
|
||||
delay: float,
|
||||
room_segments: tuple[tuple[int, ...] | None, ...],
|
||||
cookies_file: Path | None = None,
|
||||
start_page: int = 1,
|
||||
) -> int:
|
||||
"""Room-segmented citywide pagination — per-page save, incremental.
|
||||
|
||||
room_segments — кортеж из tuple (1,) / (2,) / ... / None (= no room filter).
|
||||
None segment даёт смешанную выдачу (студии + всё подряд) — Cian
|
||||
не различает их без room param.
|
||||
"""
|
||||
seg_labels = [
|
||||
("r" + str(seg[0]) if seg else "all")
|
||||
for seg in room_segments
|
||||
]
|
||||
print(
|
||||
f"\n=== PHASE CITYWIDE: Cian ЕКБ vtorichka "
|
||||
f"({len(room_segments)} segments={seg_labels} × {pages} pages, delay {delay}s) ===\n",
|
||||
flush=True,
|
||||
)
|
||||
total = 0
|
||||
cum_ins = 0
|
||||
cum_upd = 0
|
||||
async with CianScraper() as s:
|
||||
# Override per-request delay (in addition to between-anchors delay)
|
||||
s.request_delay_sec = delay
|
||||
# Inject cookies (если есть): прокидываем в curl_cffi session перед warm-up
|
||||
if cookies_file is not None:
|
||||
cookies = _load_cookies_file(cookies_file)
|
||||
assert s._cffi is not None
|
||||
s._cffi.cookies.update(cookies)
|
||||
print(f" loaded {len(cookies)} cookies from {cookies_file.name}", flush=True)
|
||||
# Warm-up: посетить homepage чтобы получить _CIAN_GK / _yasc + не выглядеть как наглый бот
|
||||
try:
|
||||
assert s._cffi is not None
|
||||
r_warm = await s._cffi.get(f"{s.base_url}/")
|
||||
print(
|
||||
f" warmup GET / → HTTP {r_warm.status_code} "
|
||||
f"(cookies={len(list(s._cffi.cookies.jar))})",
|
||||
flush=True,
|
||||
)
|
||||
# Subsequent SERP requests should look like in-site navigation
|
||||
s._cffi.headers.update(
|
||||
{
|
||||
"Referer": f"{s.base_url}/",
|
||||
"Sec-Fetch-Site": "same-origin",
|
||||
"Sec-Fetch-Mode": "navigate",
|
||||
"Sec-Fetch-User": "?1",
|
||||
"Sec-Fetch-Dest": "document",
|
||||
}
|
||||
)
|
||||
await asyncio.sleep(min(5.0, delay))
|
||||
except Exception as e: # noqa: BLE001
|
||||
print(f" warmup FAIL: {type(e).__name__}: {e}", flush=True)
|
||||
for seg_idx, seg in enumerate(room_segments):
|
||||
seg_label = "r" + str(seg[0]) if seg else "all"
|
||||
# --start-page применяется только к первому segment'у — продолжение прерванного sweep
|
||||
page_from = start_page if seg_idx == 0 else 1
|
||||
for page in range(page_from, pages + 1):
|
||||
# Manual fetch+parse чтобы можно было дампнуть HTML при empty results
|
||||
try:
|
||||
url_ = s._build_url(rooms=seg, page=page)
|
||||
assert s._cffi is not None
|
||||
resp = await s._cffi.get(url_)
|
||||
if resp.status_code != 200:
|
||||
print(
|
||||
f" seg={seg_label} page={page} HTTP {resp.status_code} → skip",
|
||||
flush=True,
|
||||
)
|
||||
await asyncio.sleep(delay)
|
||||
continue
|
||||
lots = s._parse_serp_html(resp.text)
|
||||
if not lots:
|
||||
# Fallback: новый markup .concat([...])
|
||||
lots = _parse_serp_v2(s, resp.text)
|
||||
if lots:
|
||||
print(
|
||||
f" (parsed via v2 .concat fallback)",
|
||||
flush=True,
|
||||
)
|
||||
if not lots:
|
||||
# Дамп для разбора: captcha vs пустая страница vs изменение markup
|
||||
dump_path = Path(f"cian-empty-seg-{seg_label}-p{page}.html")
|
||||
dump_path.write_text(resp.text, encoding="utf-8")
|
||||
print(f" DUMP: {dump_path} ({len(resp.text)} bytes)", flush=True)
|
||||
await s.sleep_between_requests()
|
||||
except Exception as e: # noqa: BLE001
|
||||
print(
|
||||
f" seg={seg_label} page={page} FAIL fetch: "
|
||||
f"{type(e).__name__}: {e}",
|
||||
flush=True,
|
||||
)
|
||||
await asyncio.sleep(delay)
|
||||
continue
|
||||
if not lots:
|
||||
print(
|
||||
f" seg={seg_label} page={page}: 0 lots — end of segment "
|
||||
"(либо captcha)",
|
||||
flush=True,
|
||||
)
|
||||
break
|
||||
db = SessionLocal()
|
||||
try:
|
||||
ins, upd = save_listings(db, lots)
|
||||
cum_ins += ins
|
||||
cum_upd += upd
|
||||
total += len(lots)
|
||||
print(
|
||||
f" seg={seg_label} page={page}/{pages}: {len(lots)} lots → "
|
||||
f"ins={ins} upd={upd} "
|
||||
f"[cum: total={total} new={cum_ins} upd={cum_upd}]",
|
||||
flush=True,
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
print(
|
||||
f" seg={seg_label} page={page} FAIL save: "
|
||||
f"{type(e).__name__}: {e}",
|
||||
flush=True,
|
||||
)
|
||||
finally:
|
||||
db.close()
|
||||
# scraper.fetch_around() уже спит request_delay_sec * jitter — extra sleep не нужен
|
||||
print(
|
||||
f" PHASE CITYWIDE TOTAL: fetched={total} ins={cum_ins} upd={cum_upd}",
|
||||
flush=True,
|
||||
)
|
||||
return total
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument(
|
||||
"--pages",
|
||||
type=int,
|
||||
default=8,
|
||||
help="страниц на room-segment (default 8 → ~28×4×8 = ~900 до дедупа)",
|
||||
)
|
||||
p.add_argument(
|
||||
"--delay",
|
||||
type=float,
|
||||
default=15,
|
||||
help="sec между HTTP requests (residential-safe ≥10, default 15)",
|
||||
)
|
||||
p.add_argument(
|
||||
"--rooms",
|
||||
type=str,
|
||||
default="1,2,3,4",
|
||||
help="room segments через запятую (1,2,3,4) или 'all' для одного citywide "
|
||||
"запроса без room param (default '1,2,3,4')",
|
||||
)
|
||||
p.add_argument(
|
||||
"--cookies-file",
|
||||
type=Path,
|
||||
default=None,
|
||||
help="JSON-файл с Cian cookies (browser-export или {name:value}). "
|
||||
"Опционально — для обхода captcha. См. scripts/.cian-cookies.json",
|
||||
)
|
||||
p.add_argument(
|
||||
"--start-page",
|
||||
type=int,
|
||||
default=1,
|
||||
help="С какой страницы начать ПЕРВЫЙ сегмент (для resume прерванного sweep). "
|
||||
"Остальные segments идут с p=1. Default 1.",
|
||||
)
|
||||
args = p.parse_args()
|
||||
|
||||
# Parse rooms argument
|
||||
if args.rooms.strip().lower() == "all":
|
||||
segments: tuple[tuple[int, ...] | None, ...] = (None,)
|
||||
else:
|
||||
try:
|
||||
nums = [int(x.strip()) for x in args.rooms.split(",") if x.strip()]
|
||||
except ValueError as e:
|
||||
raise SystemExit(f"--rooms must be comma-separated ints or 'all': {e}") from e
|
||||
segments = tuple((n,) for n in nums)
|
||||
|
||||
cookies_file: Path | None = args.cookies_file
|
||||
if cookies_file is not None and not cookies_file.exists():
|
||||
raise SystemExit(f"--cookies-file not found: {cookies_file}")
|
||||
|
||||
t0 = time.perf_counter()
|
||||
await phase_citywide(
|
||||
args.pages,
|
||||
args.delay,
|
||||
segments,
|
||||
cookies_file=cookies_file,
|
||||
start_page=args.start_page,
|
||||
)
|
||||
elapsed = time.perf_counter() - t0
|
||||
print(f"\n=== ALL DONE in {elapsed / 60:.1f} min ===", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
376
tradein-mvp/scripts/local-sweep-ekb-yandex.py
Normal file
376
tradein-mvp/scripts/local-sweep-ekb-yandex.py
Normal file
|
|
@ -0,0 +1,376 @@
|
|||
"""Yandex.Недвижимость ЕКБ vtorichka sweep — runs locally через residential IP.
|
||||
|
||||
Why local: Yandex anti-bot (captcha redirect) включается быстрее на datacenter
|
||||
IP + httpx fingerprint. Запуск с residential IP + Chrome120 TLS impersonation
|
||||
(curl_cffi) + browser session cookies — проходит. Скрипт пишет в prod tradein
|
||||
DB через SSH tunnel (localhost:15433 → tradein-postgres docker container).
|
||||
|
||||
Modes:
|
||||
--mode citywide (default): один общий /vtorichniy-rynok/ sweep, ~25 страниц
|
||||
× 23 cards = ~575 уникальных. Yandex SERP лимит.
|
||||
--mode combos: итерация 5 rooms × 6 price ranges = 30 combos × до 25 pages.
|
||||
DB-level дедуп → coverage близко к total (~4000+ vtorichka
|
||||
ЕКБ). ~75 min полный прогон.
|
||||
|
||||
Setup:
|
||||
ssh -f -N -L 15433:172.20.0.2:5432 gendesign # tradein-postgres tunnel
|
||||
cd tradein-mvp/backend
|
||||
export DATABASE_URL='postgresql+psycopg://tradein:<PASS>@localhost:15433/tradein'
|
||||
export ENVIRONMENT='local'
|
||||
export GLITCHTIP_DSN=''
|
||||
|
||||
Usage:
|
||||
uv run python ../scripts/local-sweep-ekb-yandex.py # citywide
|
||||
uv run python ../scripts/local-sweep-ekb-yandex.py --mode combos # full ~4000
|
||||
uv run python ../scripts/local-sweep-ekb-yandex.py --mode combos \\
|
||||
--rooms 1,2 --price-ranges 5-7,7-10 # subset
|
||||
uv run python ../scripts/local-sweep-ekb-yandex.py --pages 1 # smoke
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
# stdlib
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import random
|
||||
import sys
|
||||
import time
|
||||
import types
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlencode
|
||||
|
||||
# Auto-inject backend/ в sys.path чтобы найти app.* package
|
||||
_BACKEND_DIR = Path(__file__).resolve().parent.parent / "backend"
|
||||
if str(_BACKEND_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(_BACKEND_DIR))
|
||||
|
||||
# Stub weasyprint (Windows без GTK ломает import chain через trade_in.py)
|
||||
for _m in (
|
||||
"weasyprint",
|
||||
"weasyprint.text",
|
||||
"weasyprint.text.ffi",
|
||||
"weasyprint.css",
|
||||
"weasyprint.css.targets",
|
||||
):
|
||||
sys.modules[_m] = types.ModuleType(_m)
|
||||
sys.modules["weasyprint"].HTML = type("HTML", (), {"__init__": lambda self, *a, **kw: None})
|
||||
sys.modules["weasyprint"].CSS = type("CSS", (), {"__init__": lambda self, *a, **kw: None})
|
||||
|
||||
# app imports после stubs
|
||||
from curl_cffi.requests import AsyncSession # noqa: E402
|
||||
|
||||
from app.core.db import SessionLocal # noqa: E402
|
||||
from app.services.scrapers.base import save_listings # noqa: E402
|
||||
from app.services.scrapers.yandex_realty import YandexRealtyScraper # noqa: E402
|
||||
|
||||
DEFAULT_CITY = "ekaterinburg"
|
||||
DEFAULT_COOKIES_FILE = Path(__file__).resolve().parent / ".yandex-cookies.json"
|
||||
|
||||
# Path-сегменты Yandex для room buckets (после канонизации). Ключ — slug для
|
||||
# --rooms CLI; значение — segment в URL.
|
||||
ROOM_PATH: dict[str, str] = {
|
||||
"studio": "studiya",
|
||||
"1": "odnokomnatnaya",
|
||||
"2": "dvuhkomnatnaya",
|
||||
"3": "trehkomnatnaya",
|
||||
"4+": "4-i-bolee",
|
||||
}
|
||||
|
||||
# Price ranges (price_min, price_max) в рублях. None — open-ended boundary.
|
||||
PRICE_RANGES: list[tuple[int | None, int | None]] = [
|
||||
(None, 5_000_000),
|
||||
(5_000_000, 7_000_000),
|
||||
(7_000_000, 10_000_000),
|
||||
(10_000_000, 15_000_000),
|
||||
(15_000_000, 25_000_000),
|
||||
(25_000_000, None),
|
||||
]
|
||||
|
||||
_CAPTCHA_MARKERS = ("/showcaptcha", "smartcaptcha", "checkboxcaptcha")
|
||||
|
||||
|
||||
def _load_cookies(path: Path) -> dict[str, str]:
|
||||
if not path.exists():
|
||||
return {}
|
||||
raw = json.loads(path.read_text(encoding="utf-8"))
|
||||
return {item["name"]: item["value"] for item in raw if "name" in item and "value" in item}
|
||||
|
||||
|
||||
def _is_captcha(html: str) -> bool:
|
||||
head = html[:5000].lower()
|
||||
return any(m in head for m in _CAPTCHA_MARKERS) or "докажите, что вы не робот" in head
|
||||
|
||||
|
||||
def _build_url(
|
||||
city: str,
|
||||
page: int = 0,
|
||||
rooms: str | None = None,
|
||||
price_min: int | None = None,
|
||||
price_max: int | None = None,
|
||||
) -> str:
|
||||
"""Yandex SERP URL. Если rooms задан — path-based бакет; иначе общий vtorichka.
|
||||
|
||||
newFlat=NO_DEAL принудительно отфильтровывает новостройки (нужно при
|
||||
combo, т.к. Yandex canonicalize иногда дропает /vtorichniy-rynok/).
|
||||
"""
|
||||
if rooms:
|
||||
path = f"/{city}/kupit/kvartira/{ROOM_PATH[rooms]}/vtorichniy-rynok/"
|
||||
else:
|
||||
path = f"/{city}/kupit/kvartira/vtorichniy-rynok/"
|
||||
base = f"https://realty.yandex.ru{path}"
|
||||
|
||||
params: dict[str, str | int] = {}
|
||||
if price_min is not None:
|
||||
params["priceMin"] = price_min
|
||||
if price_max is not None:
|
||||
params["priceMax"] = price_max
|
||||
if price_min is not None or price_max is not None:
|
||||
# Гарантия vtorichka — Yandex иногда дропает /vtorichniy-rynok/ при canonicalize
|
||||
params["newFlat"] = "NO_DEAL"
|
||||
if page > 0:
|
||||
params["page"] = page
|
||||
|
||||
if params:
|
||||
return f"{base}?{urlencode(params)}"
|
||||
return base
|
||||
|
||||
|
||||
def _combo_label(rooms: str | None, price_min: int | None, price_max: int | None) -> str:
|
||||
r = rooms or "all-rooms"
|
||||
if price_min is None and price_max is None:
|
||||
return f"{r}/any-price"
|
||||
lo = f"{price_min // 1_000_000}M" if price_min else "0"
|
||||
hi = f"{price_max // 1_000_000}M" if price_max else "inf"
|
||||
return f"{r}/{lo}-{hi}"
|
||||
|
||||
|
||||
async def sweep_one(
|
||||
session: AsyncSession,
|
||||
parser: YandexRealtyScraper,
|
||||
city: str,
|
||||
pages: int,
|
||||
delay: float,
|
||||
rooms: str | None = None,
|
||||
price_min: int | None = None,
|
||||
price_max: int | None = None,
|
||||
) -> tuple[int, int, int]:
|
||||
"""Один scan по фиксированному (rooms, price) — пагинация до pages или empty.
|
||||
|
||||
Returns: (fetched, inserted, updated)
|
||||
"""
|
||||
label = _combo_label(rooms, price_min, price_max)
|
||||
total = cum_ins = cum_upd = 0
|
||||
for page in range(pages):
|
||||
url = _build_url(city, page=page, rooms=rooms, price_min=price_min, price_max=price_max)
|
||||
try:
|
||||
resp = await session.get(url, timeout=20)
|
||||
except Exception as e: # noqa: BLE001
|
||||
print(f" [{label}] page={page} FAIL fetch: {type(e).__name__}: {e}", flush=True)
|
||||
await asyncio.sleep(delay)
|
||||
continue
|
||||
if resp.status_code != 200:
|
||||
print(f" [{label}] page={page} HTTP {resp.status_code} — stop combo", flush=True)
|
||||
break
|
||||
if _is_captcha(resp.text):
|
||||
print(
|
||||
f" [{label}] page={page} ⚠️ CAPTCHA — stop combo. "
|
||||
f"Подожди 30-60min или refresh .yandex-cookies.json",
|
||||
flush=True,
|
||||
)
|
||||
raise RuntimeError("captcha")
|
||||
# NBSP → space: yandex_helpers.parse_rub() багует на \xa0 в group(1) → price None
|
||||
html = resp.text.replace("\xa0", " ")
|
||||
lots = parser._parse_html(html, page=page)
|
||||
if not lots:
|
||||
print(f" [{label}] page={page}: 0 lots — end of pagination, stop", flush=True)
|
||||
break
|
||||
db = SessionLocal()
|
||||
try:
|
||||
ins, upd = save_listings(db, lots)
|
||||
cum_ins += ins
|
||||
cum_upd += upd
|
||||
total += len(lots)
|
||||
print(
|
||||
f" [{label}] page={page + 1}/{pages}: {len(lots)} lots → "
|
||||
f"ins={ins} upd={upd} [combo: {total}/{cum_ins}/{cum_upd}]",
|
||||
flush=True,
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
print(f" [{label}] page={page} FAIL save: {type(e).__name__}: {e}", flush=True)
|
||||
finally:
|
||||
db.close()
|
||||
if page + 1 < pages:
|
||||
await asyncio.sleep(delay * random.uniform(0.85, 1.3))
|
||||
return total, cum_ins, cum_upd
|
||||
|
||||
|
||||
async def phase_citywide(pages: int, delay: float, city: str, cookies: dict[str, str]) -> None:
|
||||
print(
|
||||
f"\n=== PHASE CITYWIDE: Yandex vtorichka (city={city}, {pages} pages, "
|
||||
f"delay {delay}s) ===\n",
|
||||
flush=True,
|
||||
)
|
||||
parser = YandexRealtyScraper(city=city)
|
||||
async with AsyncSession(impersonate="chrome120", cookies=cookies or {}) as session:
|
||||
total, ins, upd = await sweep_one(session, parser, city, pages, delay)
|
||||
print(f" CITYWIDE TOTAL: fetched={total} ins={ins} upd={upd}", flush=True)
|
||||
|
||||
|
||||
async def phase_combos(
|
||||
pages: int,
|
||||
delay: float,
|
||||
city: str,
|
||||
cookies: dict[str, str],
|
||||
rooms_list: list[str],
|
||||
price_ranges: list[tuple[int | None, int | None]],
|
||||
combo_delay: float,
|
||||
) -> None:
|
||||
combos = [(r, lo, hi) for r in rooms_list for lo, hi in price_ranges]
|
||||
print(
|
||||
f"\n=== PHASE COMBOS: {len(combos)} combos × ≤{pages} pages "
|
||||
f"(city={city}, delay {delay}s, combo-delay {combo_delay}s) ===",
|
||||
flush=True,
|
||||
)
|
||||
parser = YandexRealtyScraper(city=city)
|
||||
grand_fetched = grand_ins = grand_upd = 0
|
||||
async with AsyncSession(impersonate="chrome120", cookies=cookies or {}) as session:
|
||||
for i, (rooms, lo, hi) in enumerate(combos, 1):
|
||||
label = _combo_label(rooms, lo, hi)
|
||||
print(f"\n [{i}/{len(combos)}] combo={label}", flush=True)
|
||||
try:
|
||||
f, ins, upd = await sweep_one(
|
||||
session, parser, city, pages, delay,
|
||||
rooms=rooms, price_min=lo, price_max=hi,
|
||||
)
|
||||
except RuntimeError as e:
|
||||
if str(e) == "captcha":
|
||||
print(" ⚠️ stopping combo run — captcha", flush=True)
|
||||
break
|
||||
raise
|
||||
grand_fetched += f
|
||||
grand_ins += ins
|
||||
grand_upd += upd
|
||||
print(
|
||||
f" [{i}/{len(combos)}] {label} done: f={f} i={ins} u={upd} "
|
||||
f"[grand: f={grand_fetched} i={grand_ins} u={grand_upd}]",
|
||||
flush=True,
|
||||
)
|
||||
if i < len(combos):
|
||||
await asyncio.sleep(combo_delay * random.uniform(0.85, 1.3))
|
||||
print(
|
||||
f"\n COMBOS GRAND TOTAL: fetched={grand_fetched} "
|
||||
f"ins={grand_ins} upd={grand_upd}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
|
||||
def _parse_rooms_arg(s: str) -> list[str]:
|
||||
if s == "all":
|
||||
return list(ROOM_PATH)
|
||||
out: list[str] = []
|
||||
for tok in s.split(","):
|
||||
tok = tok.strip()
|
||||
if tok not in ROOM_PATH:
|
||||
raise argparse.ArgumentTypeError(
|
||||
f"unknown room '{tok}'. Allowed: {sorted(ROOM_PATH)} or 'all'"
|
||||
)
|
||||
out.append(tok)
|
||||
return out
|
||||
|
||||
|
||||
def _parse_price_ranges_arg(s: str) -> list[tuple[int | None, int | None]]:
|
||||
if s == "all":
|
||||
return list(PRICE_RANGES)
|
||||
out: list[tuple[int | None, int | None]] = []
|
||||
for tok in s.split(","):
|
||||
tok = tok.strip()
|
||||
if "-" not in tok:
|
||||
raise argparse.ArgumentTypeError(f"range '{tok}' missing '-'")
|
||||
lo_s, hi_s = tok.split("-", 1)
|
||||
lo = int(float(lo_s) * 1_000_000) if lo_s else None
|
||||
hi = int(float(hi_s) * 1_000_000) if hi_s else None
|
||||
out.append((lo, hi))
|
||||
return out
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
p = argparse.ArgumentParser(description="Yandex.Недвижимость ЕКБ sweep")
|
||||
p.add_argument(
|
||||
"--mode",
|
||||
choices=["citywide", "combos"],
|
||||
default="citywide",
|
||||
help="citywide = один общий vtorichka sweep (~575 уникальных). "
|
||||
"combos = rooms × price = ~4000+",
|
||||
)
|
||||
p.add_argument(
|
||||
"--pages",
|
||||
type=int,
|
||||
default=25,
|
||||
help="max pages per sweep (Yandex SERP cap ≈ 25; stops earlier on empty)",
|
||||
)
|
||||
p.add_argument(
|
||||
"--delay",
|
||||
type=float,
|
||||
default=15,
|
||||
help="sec между HTTP requests внутри combo (default 15, residential-safe)",
|
||||
)
|
||||
p.add_argument(
|
||||
"--combo-delay",
|
||||
type=float,
|
||||
default=20,
|
||||
help="sec между combos (default 20, дополнительный jitter combos mode)",
|
||||
)
|
||||
p.add_argument(
|
||||
"--rooms",
|
||||
type=_parse_rooms_arg,
|
||||
default="all",
|
||||
help=f"combos: comma-list из {sorted(ROOM_PATH)} или 'all' (default)",
|
||||
)
|
||||
p.add_argument(
|
||||
"--price-ranges",
|
||||
type=_parse_price_ranges_arg,
|
||||
default="all",
|
||||
help="combos: comma-list 'lo-hi' в млн руб ('-5,5-7,7-10,10-15,15-25,25-') "
|
||||
"или 'all' (default). Empty side = open-ended.",
|
||||
)
|
||||
p.add_argument(
|
||||
"--city",
|
||||
type=str,
|
||||
default=DEFAULT_CITY,
|
||||
help=f"Yandex city slug (default '{DEFAULT_CITY}')",
|
||||
)
|
||||
p.add_argument(
|
||||
"--cookies-file",
|
||||
type=Path,
|
||||
default=DEFAULT_COOKIES_FILE,
|
||||
help=f"Browser cookies JSON (default '{DEFAULT_COOKIES_FILE.name}'). "
|
||||
f"Empty/missing → start без cookies (риск captcha).",
|
||||
)
|
||||
args = p.parse_args()
|
||||
|
||||
cookies = _load_cookies(args.cookies_file)
|
||||
if cookies:
|
||||
print(f"loaded {len(cookies)} cookies from {args.cookies_file}", flush=True)
|
||||
else:
|
||||
print(
|
||||
f"⚠️ no cookies loaded ({args.cookies_file}, exists={args.cookies_file.exists()}). "
|
||||
f"Yandex anti-bot risk.",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
t0 = time.perf_counter()
|
||||
if args.mode == "citywide":
|
||||
await phase_citywide(args.pages, args.delay, args.city, cookies)
|
||||
else:
|
||||
await phase_combos(
|
||||
args.pages, args.delay, args.city, cookies,
|
||||
rooms_list=args.rooms,
|
||||
price_ranges=args.price_ranges,
|
||||
combo_delay=args.combo_delay,
|
||||
)
|
||||
elapsed = time.perf_counter() - t0
|
||||
print(f"\n=== ALL DONE in {elapsed / 60:.1f} min ===", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
443
tradein-mvp/scripts/local-sweep-ekb.py
Normal file
443
tradein-mvp/scripts/local-sweep-ekb.py
Normal file
|
|
@ -0,0 +1,443 @@
|
|||
"""Full ЕКБ sweep + houses enrichment — runs locally через residential IP.
|
||||
|
||||
Why: prod VPS Beget IP заблокирован Avito (datacenter fingerprint → 429).
|
||||
Local IP (домашний ISP) = residential → проходит. Скрипт пишет в prod DB
|
||||
через SSH tunnel (localhost:15433 → tradein-postgres docker container).
|
||||
|
||||
Setup:
|
||||
ssh -f -N -L 15433:172.20.0.2:5432 gendesign # tradein-postgres tunnel
|
||||
cd tradein-mvp/backend
|
||||
export DATABASE_URL='postgresql+psycopg://tradein:<PASS>@localhost:15433/tradein'
|
||||
|
||||
Usage:
|
||||
uv run python ../scripts/local-sweep-ekb.py # defaults
|
||||
uv run python ../scripts/local-sweep-ekb.py --pages 3 --detail-top 20 --delay 30
|
||||
uv run python ../scripts/local-sweep-ekb.py --skip-search # только detail + houses
|
||||
uv run python ../scripts/local-sweep-ekb.py --skip-houses # без houses
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
# stdlib
|
||||
import argparse
|
||||
import asyncio
|
||||
import sys
|
||||
import time
|
||||
import types
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
# Auto-inject backend/ в sys.path чтобы найти app.* package
|
||||
# (скрипт лежит в tradein-mvp/scripts/, app — в tradein-mvp/backend/)
|
||||
_BACKEND_DIR = Path(__file__).resolve().parent.parent / "backend"
|
||||
if str(_BACKEND_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(_BACKEND_DIR))
|
||||
|
||||
# Stub weasyprint (Windows без GTK ломает import chain через trade_in.py)
|
||||
for _m in (
|
||||
"weasyprint",
|
||||
"weasyprint.text",
|
||||
"weasyprint.text.ffi",
|
||||
"weasyprint.css",
|
||||
"weasyprint.css.targets",
|
||||
):
|
||||
sys.modules[_m] = types.ModuleType(_m)
|
||||
sys.modules["weasyprint"].HTML = type("HTML", (), {"__init__": lambda self, *a, **kw: None})
|
||||
sys.modules["weasyprint"].CSS = type("CSS", (), {"__init__": lambda self, *a, **kw: None})
|
||||
|
||||
# app imports после stubs
|
||||
from sqlalchemy import text # noqa: E402
|
||||
|
||||
from app.core.db import SessionLocal # noqa: E402
|
||||
from app.services.scrapers.avito import AvitoScraper # noqa: E402
|
||||
|
||||
|
||||
class CityWideAvitoScraper(AvitoScraper):
|
||||
"""Subclass без geo-фильтра — search всего города (ЕКБ) через pagination.
|
||||
|
||||
Override `_build_web_url` чтобы НЕ добавлять `geoCoords`/`radius` — Avito
|
||||
отдаёт ВСЕ объявления в городе sorted by date, до 100 pages (cap ~5000).
|
||||
"""
|
||||
|
||||
def _build_web_url(self, lat: float, lon: float, radius_km: int, page: int = 1) -> str: # noqa: ARG002
|
||||
from urllib.parse import urlencode
|
||||
params = {"s": 104, "p": page} # s=104 sort by date
|
||||
return f"{self.base_url}/ekaterinburg/kvartiry/prodam-ASgBAgICAUSSA8YQ?{urlencode(params)}"
|
||||
|
||||
|
||||
# Avito URL slugs для room-filter (extracted из real search page HTML 2026-05-23)
|
||||
ROOM_SLUGS: list[tuple[str, str]] = [
|
||||
("студии", "studii-ASgBAgICAkSSA8YQygj~WA"),
|
||||
("1-комн.", "1-komnatnye-ASgBAgICAkSSA8YQygiAWQ"),
|
||||
("2-комн.", "2-komnatnye-ASgBAgICAkSSA8YQygiCWQ"),
|
||||
("3-комн.", "3-komnatnye-ASgBAgICAkSSA8YQygiEWQ"),
|
||||
("4-комн.", "4-komnatnye-ASgBAgICAkSSA8YQygiGWQ"),
|
||||
("5+комн.", "5-komnatnye-ASgBAgICAkSSA8YQygiIWQ"),
|
||||
("своб.планир.", "svobodnaya_planirovka-ASgBAgICAkSSA8YQygj8zzI"),
|
||||
]
|
||||
|
||||
|
||||
class CityRoomsAvitoScraper(AvitoScraper):
|
||||
"""Subclass с URL slug filter по комнатности — для splitting когда total > 5000.
|
||||
|
||||
URL pattern: /ekaterinburg/kvartiry/prodam/<slug>?s=104&p=N
|
||||
"""
|
||||
|
||||
def __init__(self, slug: str) -> None:
|
||||
super().__init__()
|
||||
self.room_slug = slug
|
||||
|
||||
def _build_web_url(self, lat: float, lon: float, radius_km: int, page: int = 1) -> str: # noqa: ARG002
|
||||
from urllib.parse import urlencode
|
||||
params = {"s": 104, "p": page}
|
||||
return f"{self.base_url}/ekaterinburg/kvartiry/prodam/{self.room_slug}?{urlencode(params)}"
|
||||
from app.services.scrapers.avito_detail import ( # noqa: E402
|
||||
fetch_detail,
|
||||
save_detail_enrichment,
|
||||
)
|
||||
from app.services.scrapers.avito_houses import ( # noqa: E402
|
||||
fetch_house_catalog,
|
||||
save_house_catalog_enrichment,
|
||||
)
|
||||
from app.services.scrapers.base import save_listings # noqa: E402
|
||||
|
||||
# 10 anchors покрывающих центральные/северные/южные/восточные/западные районы ЕКБ
|
||||
ANCHORS: list[tuple[str, float, float]] = [
|
||||
("Центр", 56.8400, 60.6050),
|
||||
("ЮЗ", 56.7950, 60.5300),
|
||||
("Уралмаш", 56.8970, 60.6100),
|
||||
("Академический", 56.7700, 60.5500),
|
||||
("Пионерский", 56.8650, 60.6200),
|
||||
("Эльмаш", 56.8567, 60.6700),
|
||||
("Втузгородок", 56.8400, 60.6650),
|
||||
("ВИЗ", 56.8500, 60.5650),
|
||||
("Юго-Запад", 56.7770, 60.5800),
|
||||
("Сортировка", 56.8950, 60.5600),
|
||||
]
|
||||
|
||||
|
||||
async def phase_citywide(pages: int, delay: float) -> int:
|
||||
"""City-wide pagination — ВСЕ объявления ЕКБ (no geo filter), per-page save.
|
||||
|
||||
Per-page save важно: visible прогресс + не теряем данные если crash mid-way.
|
||||
"""
|
||||
print(
|
||||
f"\n=== PHASE CITYWIDE: full ЕКБ pagination ({pages} pages, delay {delay}s) ===\n",
|
||||
flush=True,
|
||||
)
|
||||
total = 0
|
||||
cum_ins = 0
|
||||
cum_upd = 0
|
||||
async with CityWideAvitoScraper() as s:
|
||||
for page in range(1, pages + 1):
|
||||
url = s._build_web_url(0.0, 0.0, 1, page=page)
|
||||
try:
|
||||
assert s._cffi is not None
|
||||
response = await s._cffi.get(url)
|
||||
if response.status_code != 200:
|
||||
print(f" ⚠️ page={page} HTTP {response.status_code} — stop", flush=True)
|
||||
break
|
||||
lots = s._parse_html(response.text, source_url_base=url)
|
||||
except Exception as e: # noqa: BLE001
|
||||
print(f" page={page} FAIL fetch: {type(e).__name__}: {e}", flush=True)
|
||||
await asyncio.sleep(delay)
|
||||
continue
|
||||
if not lots:
|
||||
print(f" page={page}: 0 lots — end of pagination", flush=True)
|
||||
break
|
||||
db = SessionLocal()
|
||||
try:
|
||||
ins, upd = save_listings(db, lots)
|
||||
cum_ins += ins
|
||||
cum_upd += upd
|
||||
total += len(lots)
|
||||
print(
|
||||
f" page={page}/{pages}: {len(lots)} lots → ins={ins} upd={upd} "
|
||||
f"[cum: total={total} new={cum_ins} upd={cum_upd}]",
|
||||
flush=True,
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
print(f" page={page} FAIL save: {type(e).__name__}: {e}", flush=True)
|
||||
finally:
|
||||
db.close()
|
||||
if page < pages:
|
||||
await asyncio.sleep(delay)
|
||||
print(f" PHASE CITYWIDE TOTAL: fetched={total} ins={cum_ins} upd={cum_upd}", flush=True)
|
||||
return total
|
||||
|
||||
|
||||
async def phase_byrooms(pages: int, delay: float) -> int:
|
||||
"""Rooms-split sweep — 7 categories × pages, per-page save.
|
||||
|
||||
Для total > 5000 (Avito search cap) split по комнатности позволяет получить
|
||||
все listings (каждый segment <5000).
|
||||
"""
|
||||
print(
|
||||
f"\n=== PHASE BYROOMS: 7 категорий × {pages} pages, delay {delay}s ===\n",
|
||||
flush=True,
|
||||
)
|
||||
grand_total = 0
|
||||
grand_ins = 0
|
||||
grand_upd = 0
|
||||
import random
|
||||
rate_limit_hits = 0
|
||||
for ci, (name, slug) in enumerate(ROOM_SLUGS, 1):
|
||||
print(f"\n--- [{ci}/{len(ROOM_SLUGS)}] Категория: {name} (slug={slug[:30]}...) ---", flush=True)
|
||||
cat_total = 0
|
||||
cat_ins = 0
|
||||
cat_upd = 0
|
||||
async with CityRoomsAvitoScraper(slug) as s:
|
||||
for page in range(1, pages + 1):
|
||||
url = s._build_web_url(0.0, 0.0, 1, page=page)
|
||||
try:
|
||||
assert s._cffi is not None
|
||||
response = await s._cffi.get(url)
|
||||
if response.status_code == 429:
|
||||
# Exponential backoff на rate-limit: 60s × 2^N (max 30 min)
|
||||
rate_limit_hits += 1
|
||||
backoff = min(60 * (2 ** (rate_limit_hits - 1)), 1800)
|
||||
print(f" {name} p={page} HTTP 429 (hit #{rate_limit_hits}) → backoff {backoff}s", flush=True)
|
||||
await asyncio.sleep(backoff)
|
||||
# retry same page after backoff
|
||||
try:
|
||||
response = await s._cffi.get(url)
|
||||
if response.status_code != 200:
|
||||
print(f" {name} p={page} retry HTTP {response.status_code} — skip page", flush=True)
|
||||
await asyncio.sleep(delay * 2)
|
||||
continue
|
||||
except Exception as e: # noqa: BLE001
|
||||
print(f" {name} p={page} retry FAIL: {type(e).__name__}: {e}", flush=True)
|
||||
continue
|
||||
elif response.status_code != 200:
|
||||
print(f" {name} p={page} HTTP {response.status_code} — skip page", flush=True)
|
||||
await asyncio.sleep(delay)
|
||||
continue
|
||||
lots = s._parse_html(response.text, source_url_base=url)
|
||||
except Exception as e: # noqa: BLE001
|
||||
print(f" {name} p={page} FAIL fetch: {type(e).__name__}: {e}", flush=True)
|
||||
await asyncio.sleep(delay)
|
||||
continue
|
||||
if not lots:
|
||||
print(f" {name} page={page}: 0 lots — end of category", flush=True)
|
||||
break
|
||||
db = SessionLocal()
|
||||
try:
|
||||
ins, upd = save_listings(db, lots)
|
||||
cat_ins += ins
|
||||
cat_upd += upd
|
||||
cat_total += len(lots)
|
||||
print(
|
||||
f" {name} p={page}/{pages}: {len(lots)} lots → ins={ins} upd={upd} "
|
||||
f"[cat: tot={cat_total} new={cat_ins}]",
|
||||
flush=True,
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
print(f" {name} page={page} FAIL save: {type(e).__name__}: {e}", flush=True)
|
||||
finally:
|
||||
db.close()
|
||||
if page < pages:
|
||||
# Jitter: ±30% от delay чтобы выглядеть менее robotic
|
||||
jittered = delay * random.uniform(0.7, 1.3)
|
||||
await asyncio.sleep(jittered)
|
||||
print(
|
||||
f" --- {name} TOTAL: fetched={cat_total} ins={cat_ins} upd={cat_upd} ---",
|
||||
flush=True,
|
||||
)
|
||||
grand_total += cat_total
|
||||
grand_ins += cat_ins
|
||||
grand_upd += cat_upd
|
||||
print(
|
||||
f"\n PHASE BYROOMS GRAND TOTAL: fetched={grand_total} new={grand_ins} updated={grand_upd}",
|
||||
flush=True,
|
||||
)
|
||||
return grand_total
|
||||
|
||||
|
||||
async def phase_a_search(pages: int, delay: float) -> int:
|
||||
"""Search всех anchors × pages, save listings."""
|
||||
print(
|
||||
f"\n=== PHASE A: search ({len(ANCHORS)} anchors × {pages} pages, "
|
||||
f"delay {delay}s) ===\n",
|
||||
flush=True,
|
||||
)
|
||||
total = 0
|
||||
for i, (name, lat, lon) in enumerate(ANCHORS, 1):
|
||||
print(f"[{i}/{len(ANCHORS)}] {name} ({lat},{lon})", flush=True)
|
||||
try:
|
||||
async with AvitoScraper() as s:
|
||||
lots = await s.fetch_around(
|
||||
lat, lon, 1500, pages=pages, delay_override_sec=delay
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
print(f" FAIL fetch: {type(e).__name__}: {e}", flush=True)
|
||||
await asyncio.sleep(delay)
|
||||
continue
|
||||
print(f" fetched {len(lots)} lots", flush=True)
|
||||
if not lots:
|
||||
print(" ⚠️ 0 lots — possible 429, stop phase A", flush=True)
|
||||
break
|
||||
db = SessionLocal()
|
||||
try:
|
||||
ins, upd = save_listings(db, lots)
|
||||
print(f" saved: ins={ins} upd={upd}", flush=True)
|
||||
total += len(lots)
|
||||
except Exception as e: # noqa: BLE001
|
||||
print(f" FAIL save: {type(e).__name__}: {e}", flush=True)
|
||||
finally:
|
||||
db.close()
|
||||
if i < len(ANCHORS):
|
||||
await asyncio.sleep(delay)
|
||||
return total
|
||||
|
||||
|
||||
async def phase_b_detail(top_n: int, delay: float) -> tuple[int, set[str]]:
|
||||
"""Detail enrichment для top-N recent listings без detail.
|
||||
|
||||
Returns: (enriched_count, set of unique house_catalog_url paths extracted)
|
||||
"""
|
||||
print(
|
||||
f"\n=== PHASE B: detail enrichment (top {top_n}, delay {delay}s) ===\n",
|
||||
flush=True,
|
||||
)
|
||||
db = SessionLocal()
|
||||
try:
|
||||
rows = db.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT source_url, source_id FROM listings
|
||||
WHERE source='avito' AND detail_enriched_at IS NULL
|
||||
AND source_url IS NOT NULL AND price_rub > 0
|
||||
ORDER BY scraped_at DESC NULLS LAST
|
||||
LIMIT :limit
|
||||
"""
|
||||
),
|
||||
{"limit": top_n},
|
||||
).mappings().all()
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
print(f" queue: {len(rows)} listings to enrich", flush=True)
|
||||
enriched = 0
|
||||
failed = 0
|
||||
house_urls: set[str] = set()
|
||||
for i, row in enumerate(rows, 1):
|
||||
url = row["source_url"]
|
||||
path = urlparse(url).path if url.startswith("http") else url
|
||||
print(f"[{i}/{len(rows)}] detail {row['source_id']}", flush=True)
|
||||
try:
|
||||
enr = await fetch_detail(path)
|
||||
db = SessionLocal()
|
||||
try:
|
||||
ok = save_detail_enrichment(db, enr)
|
||||
print(
|
||||
f" enriched: ok={ok} "
|
||||
f"rooms={enr.rooms} area={enr.area_m2} "
|
||||
f"kitchen={enr.kitchen_area_m2} owners={enr.owners_count} "
|
||||
f"house_url={enr.house_catalog_url}",
|
||||
flush=True,
|
||||
)
|
||||
if ok:
|
||||
enriched += 1
|
||||
# Извлекаем house path (strip query string + filter только /catalog/houses/)
|
||||
if enr.house_catalog_url:
|
||||
p = urlparse(enr.house_catalog_url).path
|
||||
if p.startswith("/catalog/houses/"):
|
||||
house_urls.add(p)
|
||||
finally:
|
||||
db.close()
|
||||
except Exception as e: # noqa: BLE001
|
||||
print(f" FAIL: {type(e).__name__}: {e}", flush=True)
|
||||
failed += 1
|
||||
if i < len(rows):
|
||||
await asyncio.sleep(delay)
|
||||
print(f" total: enriched={enriched} failed={failed} unique_houses_extracted={len(house_urls)}", flush=True)
|
||||
return enriched, house_urls
|
||||
|
||||
|
||||
async def phase_c_houses(delay: float, extra_urls: set[str] | None = None) -> int:
|
||||
"""Houses enrichment — unique house_url из listings (DB) + extra_urls (из Phase B in-memory).
|
||||
|
||||
save_detail_enrichment не пишет house_url в listings (баг Stage 2b),
|
||||
поэтому Phase C принимает extra_urls собранные Phase B вручную.
|
||||
"""
|
||||
print("\n=== PHASE C: houses enrichment ===\n", flush=True)
|
||||
db = SessionLocal()
|
||||
try:
|
||||
rows = db.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT DISTINCT house_url FROM listings
|
||||
WHERE source='avito' AND house_url IS NOT NULL
|
||||
AND house_id_fk IS NULL
|
||||
ORDER BY house_url
|
||||
"""
|
||||
)
|
||||
).fetchall()
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
urls_from_db = {r[0] for r in rows if r[0]}
|
||||
urls = urls_from_db | (extra_urls or set())
|
||||
print(f" queue: {len(urls)} unique houses (db={len(urls_from_db)} +extras={len(extra_urls or set())})", flush=True)
|
||||
if not urls:
|
||||
print(" no houses to enrich (skip)", flush=True)
|
||||
return 0
|
||||
|
||||
enriched = 0
|
||||
failed = 0
|
||||
for i, url in enumerate(sorted(urls), 1):
|
||||
path = urlparse(url).path if url.startswith("http") else url
|
||||
print(f"[{i}/{len(urls)}] {path}", flush=True)
|
||||
try:
|
||||
enr = await fetch_house_catalog(path)
|
||||
db = SessionLocal()
|
||||
try:
|
||||
counters = save_house_catalog_enrichment(db, enr)
|
||||
print(f" saved: {counters}", flush=True)
|
||||
enriched += 1
|
||||
finally:
|
||||
db.close()
|
||||
except Exception as e: # noqa: BLE001
|
||||
print(f" FAIL: {type(e).__name__}: {e}", flush=True)
|
||||
failed += 1
|
||||
if i < len(urls):
|
||||
await asyncio.sleep(delay)
|
||||
print(f" total: enriched={enriched} failed={failed}", flush=True)
|
||||
return enriched
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument(
|
||||
"--mode",
|
||||
choices=["anchors", "citywide", "byrooms", "both"],
|
||||
default="anchors",
|
||||
help="anchors=10 anchors; citywide=full pagination без geo; byrooms=7 категорий комнат (для total>5000); both=anchors+citywide",
|
||||
)
|
||||
p.add_argument("--pages", type=int, default=5, help="pages per anchor (anchors mode) или total pages (citywide)")
|
||||
p.add_argument("--detail-top", type=int, default=50, help="top-N для detail enrichment")
|
||||
p.add_argument("--delay", type=float, default=30, help="sec между HTTP requests")
|
||||
p.add_argument("--skip-search", action="store_true")
|
||||
p.add_argument("--skip-detail", action="store_true")
|
||||
p.add_argument("--skip-houses", action="store_true")
|
||||
args = p.parse_args()
|
||||
|
||||
t0 = time.perf_counter()
|
||||
if not args.skip_search:
|
||||
if args.mode in ("anchors", "both"):
|
||||
await phase_a_search(args.pages, args.delay)
|
||||
if args.mode in ("citywide", "both"):
|
||||
cw_pages = args.pages if args.mode == "citywide" else 100
|
||||
await phase_citywide(cw_pages, args.delay)
|
||||
if args.mode == "byrooms":
|
||||
await phase_byrooms(args.pages, args.delay)
|
||||
extracted_house_urls: set[str] = set()
|
||||
if not args.skip_detail:
|
||||
_, extracted_house_urls = await phase_b_detail(args.detail_top, args.delay)
|
||||
if not args.skip_houses:
|
||||
await phase_c_houses(args.delay, extra_urls=extracted_house_urls)
|
||||
elapsed = time.perf_counter() - t0
|
||||
print(f"\n=== ALL DONE in {elapsed / 60:.1f} min ===", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
103
tradein-mvp/scripts/probe-avito-1anchor.py
Normal file
103
tradein-mvp/scripts/probe-avito-1anchor.py
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
"""Avito probe: 1 anchor × 1 page с настраиваемой задержкой.
|
||||
|
||||
Минимальный dry-run чтобы проверить bans / detection после anti-bot
|
||||
hardening (PR #487) + global delay setting (PR #485).
|
||||
|
||||
Setup:
|
||||
ssh -f -N -L 15433:172.20.0.2:5432 gendesign
|
||||
cd tradein-mvp/backend
|
||||
$env:DATABASE_URL = 'postgresql+psycopg://tradein:<PASS>@localhost:15433/tradein'
|
||||
|
||||
Usage:
|
||||
uv run python ../scripts/probe-avito-1anchor.py
|
||||
uv run python ../scripts/probe-avito-1anchor.py --delay 5 --radius 1000
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import sys
|
||||
import time
|
||||
import types
|
||||
from pathlib import Path
|
||||
|
||||
_BACKEND_DIR = Path(__file__).resolve().parent.parent / "backend"
|
||||
if str(_BACKEND_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(_BACKEND_DIR))
|
||||
|
||||
for _m in (
|
||||
"weasyprint",
|
||||
"weasyprint.text",
|
||||
"weasyprint.text.ffi",
|
||||
"weasyprint.css",
|
||||
"weasyprint.css.targets",
|
||||
):
|
||||
sys.modules[_m] = types.ModuleType(_m)
|
||||
sys.modules["weasyprint"].HTML = type("HTML", (), {"__init__": lambda self, *a, **kw: None})
|
||||
sys.modules["weasyprint"].CSS = type("CSS", (), {"__init__": lambda self, *a, **kw: None})
|
||||
|
||||
from app.core.db import SessionLocal # noqa: E402
|
||||
from app.services.scrapers.avito import AvitoScraper # noqa: E402
|
||||
from app.services.scrapers.base import save_listings # noqa: E402
|
||||
|
||||
ANCHOR_NAME = "Центр ЕКБ"
|
||||
ANCHOR_LAT = 56.8400
|
||||
ANCHOR_LON = 60.6050
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--delay", type=float, default=3.0, help="sec между HTTP requests")
|
||||
p.add_argument("--radius", type=int, default=1500, help="метры")
|
||||
p.add_argument("--pages", type=int, default=1)
|
||||
args = p.parse_args()
|
||||
|
||||
print(
|
||||
f"=== Avito probe ===\n"
|
||||
f" anchor: {ANCHOR_NAME} ({ANCHOR_LAT}, {ANCHOR_LON})\n"
|
||||
f" radius: {args.radius}m\n"
|
||||
f" pages: {args.pages}\n"
|
||||
f" delay: {args.delay}s\n",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
t0 = time.perf_counter()
|
||||
try:
|
||||
async with AvitoScraper() as s:
|
||||
lots = await s.fetch_around(
|
||||
ANCHOR_LAT, ANCHOR_LON, args.radius,
|
||||
pages=args.pages, delay_override_sec=args.delay,
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
print(f"FETCH FAIL: {type(e).__name__}: {e}", flush=True)
|
||||
return
|
||||
|
||||
fetched = len(lots)
|
||||
print(f"fetched: {fetched} lots in {time.perf_counter() - t0:.1f}s", flush=True)
|
||||
if fetched == 0:
|
||||
print("⚠️ 0 lots — вероятно 429/ban. Смотри backend logs.", flush=True)
|
||||
return
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
ins, upd = save_listings(db, lots)
|
||||
print(f"saved: ins={ins} upd={upd}", flush=True)
|
||||
except Exception as e: # noqa: BLE001
|
||||
print(f"SAVE FAIL: {type(e).__name__}: {e}", flush=True)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
sample = lots[:3]
|
||||
print("\nsample listings:", flush=True)
|
||||
for lot in sample:
|
||||
addr = (lot.address or "")[:60]
|
||||
print(
|
||||
f" source_id={lot.source_id} price={lot.price_rub}₽ "
|
||||
f"rooms={lot.rooms} area={lot.area_m2}м² floor={lot.floor}/{lot.total_floors} "
|
||||
f"addr={addr!r}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
58
tradein-mvp/scripts/probe-db-counts.py
Normal file
58
tradein-mvp/scripts/probe-db-counts.py
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
"""Quick DB diagnostics: Avito listings count + recent insert rate."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import psycopg
|
||||
|
||||
dsn = os.environ["DATABASE_URL"].replace("postgresql+psycopg://", "postgresql://")
|
||||
conn = psycopg.connect(dsn)
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute("SELECT count(*) FROM listings WHERE source = 'avito'")
|
||||
print("avito total:", cur.fetchone()[0])
|
||||
|
||||
cur.execute(
|
||||
"SELECT count(*) FROM listings "
|
||||
"WHERE source='avito' AND scraped_at > NOW() - interval '20 minutes'"
|
||||
)
|
||||
print("avito scraped last 20min:", cur.fetchone()[0])
|
||||
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT column_name FROM information_schema.columns
|
||||
WHERE table_name='listings' AND column_name IN
|
||||
('created_at','first_seen_at','first_seen','inserted_at')
|
||||
"""
|
||||
)
|
||||
cols = [r[0] for r in cur.fetchall()]
|
||||
print(f"timestamp cols available: {cols}")
|
||||
if "first_seen_at" in cols:
|
||||
cur.execute(
|
||||
"SELECT count(*) FROM listings "
|
||||
"WHERE source='avito' AND first_seen_at > NOW() - interval '20 minutes'"
|
||||
)
|
||||
print("avito first_seen last 20min (new):", cur.fetchone()[0])
|
||||
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT date_trunc('minute', scraped_at) AS m, count(*)
|
||||
FROM listings
|
||||
WHERE source='avito' AND scraped_at > NOW() - interval '30 minutes'
|
||||
GROUP BY 1 ORDER BY 1 DESC
|
||||
"""
|
||||
)
|
||||
print("\nper-minute scraped (last 30min):")
|
||||
for row in cur.fetchall():
|
||||
print(f" {row[0]} {row[1]}")
|
||||
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT rooms, count(*)
|
||||
FROM listings
|
||||
WHERE source='avito' AND scraped_at > NOW() - interval '30 minutes'
|
||||
GROUP BY 1 ORDER BY 1
|
||||
"""
|
||||
)
|
||||
print("\nby rooms (last 30min):")
|
||||
for row in cur.fetchall():
|
||||
print(f" rooms={row[0]} count={row[1]}")
|
||||
129
tradein-mvp/scripts/probe-db-leak-full.py
Normal file
129
tradein-mvp/scripts/probe-db-leak-full.py
Normal file
|
|
@ -0,0 +1,129 @@
|
|||
"""Полная проверка региональной утечки в БД Avito listings.
|
||||
|
||||
Угол 1: source_url slug (canonical, надёжный)
|
||||
Угол 2: address с маркером области (квартиры в ЕКБ не содержат "обл.")
|
||||
Угол 3: address contains имя пригорода/соседнего города
|
||||
Угол 4: top самых частых первых слов address (sanity check)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from collections import Counter
|
||||
|
||||
import psycopg
|
||||
|
||||
dsn = os.environ["DATABASE_URL"].replace("postgresql+psycopg://", "postgresql://")
|
||||
conn = psycopg.connect(dsn)
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute("SELECT count(*) FROM listings WHERE source='avito'")
|
||||
total = cur.fetchone()[0]
|
||||
print(f"=== Avito total: {total} ===\n")
|
||||
|
||||
# Угол 1: URL slug (canonical)
|
||||
print("[1] URL slug check:")
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT
|
||||
count(*) FILTER (WHERE source_url ~ 'avito\\.ru/ekaterinburg/') AS ekb,
|
||||
count(*) FILTER (WHERE source_url IS NOT NULL AND source_url !~ 'avito\\.ru/ekaterinburg/') AS not_ekb,
|
||||
count(*) FILTER (WHERE source_url IS NULL) AS no_url
|
||||
FROM listings WHERE source='avito'
|
||||
"""
|
||||
)
|
||||
ekb, not_ekb, no_url = cur.fetchone()
|
||||
print(f" ekaterinburg URL: {ekb}")
|
||||
print(f" NOT ekaterinburg: {not_ekb} {'← LEAK' if not_ekb else '✓ clean'}")
|
||||
print(f" NULL url: {no_url}")
|
||||
|
||||
# Угол 2: address с "область" / "обл."
|
||||
print("\n[2] Address contains region marker (область/обл.):")
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT count(*) FROM listings
|
||||
WHERE source='avito'
|
||||
AND address IS NOT NULL
|
||||
AND (address ILIKE '%область%' OR address ~* '\\bобл\\b' OR address ~* '\\bобл\\.')
|
||||
"""
|
||||
)
|
||||
n = cur.fetchone()[0]
|
||||
print(f" count: {n} {'← suspicious' if n > 5 else '✓ ok'}")
|
||||
if n > 0:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT source_url, address FROM listings
|
||||
WHERE source='avito' AND address IS NOT NULL
|
||||
AND (address ILIKE '%область%' OR address ~* '\\bобл\\b' OR address ~* '\\bобл\\.')
|
||||
LIMIT 10
|
||||
"""
|
||||
)
|
||||
for url, addr in cur.fetchall():
|
||||
slug = url.split('avito.ru/')[1].split('/')[0] if url and 'avito.ru/' in url else '?'
|
||||
print(f" [{slug}] {addr[:90]!r}")
|
||||
|
||||
# Угол 3: address contains имя другого города/пригорода
|
||||
print("\n[3] Address contains satellite-city marker:")
|
||||
satellites = [
|
||||
"Верхняя Пышма", "Арамиль", "Сысерть", "Дегтярск", "Первоуральск",
|
||||
"Заречный", "Невьянск", "Нижний Тагил", "Кыштым", "Челябинск",
|
||||
"Алапаевск", "Краснофимск", "Озёрск", "Озерск", "Снежинск", "Касли",
|
||||
"Рефтинский", "Свободный", "Большой Исток", "Пышма", "Покровское",
|
||||
"Реж", "Лесной", "Полевской", "Берёзовск", "Березовск", "Михайловск",
|
||||
"Туринск", "Североуральск", "Тавда", "Богданович", "Артёмовский",
|
||||
"Артемовский", "Качканар", "Ивдель", "Серов", "Ирбит", "Новоуральск",
|
||||
"Камышлов", "Сухой Лог",
|
||||
]
|
||||
for city in satellites:
|
||||
cur.execute(
|
||||
"SELECT count(*) FROM listings WHERE source='avito' AND address ILIKE %s",
|
||||
(f"%{city}%",),
|
||||
)
|
||||
n = cur.fetchone()[0]
|
||||
if n > 0:
|
||||
print(f" {city}: {n}")
|
||||
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT count(*) FROM listings WHERE source='avito' AND (
|
||||
address ILIKE '%Верхняя Пышма%' OR address ILIKE '%Арамиль%' OR
|
||||
address ILIKE '%Сысерть%' OR address ILIKE '%Дегтярск%' OR
|
||||
address ILIKE '%Первоуральск%' OR address ILIKE '%Заречный%' OR
|
||||
address ILIKE '%Невьянск%' OR address ILIKE '%Нижний Тагил%' OR
|
||||
address ILIKE '%Кыштым%' OR address ILIKE '%Челябинск%' OR
|
||||
address ILIKE '%Алапаевск%' OR address ILIKE '%Краснофимск%' OR
|
||||
address ILIKE '%Озёрск%' OR address ILIKE '%Озерск%' OR
|
||||
address ILIKE '%Снежинск%' OR address ILIKE '%Касли%' OR
|
||||
address ILIKE '%Рефтинский%' OR address ILIKE '%Большой Исток%' OR
|
||||
address ILIKE '%Реж%' OR address ILIKE '%Лесной%' OR address ILIKE '%Полевской%' OR
|
||||
address ILIKE '%Берёзовск%' OR address ILIKE '%Березовск%'
|
||||
)
|
||||
"""
|
||||
)
|
||||
sat_total = cur.fetchone()[0]
|
||||
print(f" TOTAL satellite-city leaks: {sat_total}")
|
||||
|
||||
# Угол 4: top first-words of address
|
||||
print("\n[4] Top 20 first-words in address (sanity check):")
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT split_part(trim(address), ' ', 1) AS first_word, count(*) AS n
|
||||
FROM listings WHERE source='avito' AND COALESCE(address, '') != ''
|
||||
GROUP BY 1 ORDER BY 2 DESC LIMIT 20
|
||||
"""
|
||||
)
|
||||
for word, n in cur.fetchall():
|
||||
print(f" {n:>5} {word!r}")
|
||||
|
||||
# Угол 5: address NULL — сколько неopределённых
|
||||
print("\n[5] Coverage:")
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT
|
||||
count(*) FILTER (WHERE COALESCE(address, '') = '') AS no_addr,
|
||||
count(*) FILTER (WHERE COALESCE(address, '') != '') AS has_addr
|
||||
FROM listings WHERE source='avito'
|
||||
"""
|
||||
)
|
||||
no_a, has_a = cur.fetchone()
|
||||
print(f" has address: {has_a}")
|
||||
print(f" no address: {no_a}")
|
||||
75
tradein-mvp/scripts/probe-db-region-leak.py
Normal file
75
tradein-mvp/scripts/probe-db-region-leak.py
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
"""Проверка: есть ли в DB Avito листинги из других городов (region leak)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import psycopg
|
||||
|
||||
dsn = os.environ["DATABASE_URL"].replace("postgresql+psycopg://", "postgresql://")
|
||||
conn = psycopg.connect(dsn)
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute("SELECT count(*) FROM listings WHERE source='avito'")
|
||||
total = cur.fetchone()[0]
|
||||
print(f"avito total: {total}")
|
||||
|
||||
# Все с lat/lon — проверим bounding box ЕКБ (56.7..57.0, 60.4..60.8)
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT
|
||||
count(*) FILTER (WHERE lat BETWEEN 56.5 AND 57.0 AND lon BETWEEN 60.4 AND 61.0) as in_ekb,
|
||||
count(*) FILTER (WHERE lat IS NOT NULL AND (lat < 56.5 OR lat > 57.0 OR lon < 60.4 OR lon > 61.0)) as outside_ekb,
|
||||
count(*) FILTER (WHERE lat IS NULL) as no_coords
|
||||
FROM listings WHERE source='avito'
|
||||
"""
|
||||
)
|
||||
in_ekb, outside, no_coords = cur.fetchone()
|
||||
print(f" in EKB bbox: {in_ekb}")
|
||||
print(f" outside EKB bbox: {outside} ← out-of-region leak!")
|
||||
print(f" no coords yet: {no_coords}")
|
||||
|
||||
# Sample outside
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT source_id, lat, lon, address, price_rub, rooms
|
||||
FROM listings
|
||||
WHERE source='avito' AND lat IS NOT NULL
|
||||
AND (lat < 56.5 OR lat > 57.0 OR lon < 60.4 OR lon > 61.0)
|
||||
ORDER BY scraped_at DESC NULLS LAST
|
||||
LIMIT 10
|
||||
"""
|
||||
)
|
||||
print("\nsample non-EKB listings:")
|
||||
for r in cur.fetchall():
|
||||
sid, lat, lon, addr, price, rooms = r
|
||||
print(f" {sid} ({lat:.3f},{lon:.3f}) rooms={rooms} {price:,}₽ — {(addr or '')[:80]!r}")
|
||||
|
||||
# Address-based leak (без lat/lon) — поиск явных маркеров других городов
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT count(*) FROM listings
|
||||
WHERE source='avito'
|
||||
AND lat IS NULL
|
||||
AND address IS NOT NULL
|
||||
AND (address ILIKE '%москв%' OR address ILIKE '%омск%' OR address ILIKE '%чебок%'
|
||||
OR address ILIKE '%петерб%' OR address ILIKE '%казан%' OR address ILIKE '%новосиб%')
|
||||
"""
|
||||
)
|
||||
print(f"\naddress-based leak (no coords, other-city keyword): {cur.fetchone()[0]}")
|
||||
|
||||
# По комнатам — где leak вероятнее всего
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT rooms,
|
||||
count(*) AS total,
|
||||
count(*) FILTER (WHERE lat IS NOT NULL AND (lat < 56.5 OR lat > 57.0 OR lon < 60.4 OR lon > 61.0)) AS leak
|
||||
FROM listings WHERE source='avito'
|
||||
GROUP BY rooms
|
||||
ORDER BY rooms NULLS LAST
|
||||
"""
|
||||
)
|
||||
print("\nleak by rooms:")
|
||||
print(f" {'rooms':<8} {'total':>8} {'leak':>8}")
|
||||
for r in cur.fetchall():
|
||||
rooms, total, leak = r
|
||||
pct = (leak * 100.0 / total) if total else 0
|
||||
print(f" {str(rooms):<8} {total:>8} {leak:>8} ({pct:.0f}%)")
|
||||
51
tradein-mvp/scripts/probe-db-url-leak.py
Normal file
51
tradein-mvp/scripts/probe-db-url-leak.py
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
"""URL-based region leak check для Avito listings."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
from collections import Counter
|
||||
|
||||
import psycopg
|
||||
|
||||
dsn = os.environ["DATABASE_URL"].replace("postgresql+psycopg://", "postgresql://")
|
||||
conn = psycopg.connect(dsn)
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute("SELECT source_url FROM listings WHERE source='avito'")
|
||||
slug_re = re.compile(r"avito\.ru/([a-z_]+)/")
|
||||
|
||||
cities = Counter()
|
||||
no_match = 0
|
||||
for (url,) in cur.fetchall():
|
||||
if not url:
|
||||
no_match += 1
|
||||
continue
|
||||
m = slug_re.search(url)
|
||||
if m:
|
||||
cities[m.group(1)] += 1
|
||||
else:
|
||||
no_match += 1
|
||||
|
||||
print("city slug distribution in source_url:")
|
||||
for city, n in cities.most_common():
|
||||
flag = " ← EKB" if city == "ekaterinburg" else (" ← OTHER CITY" if city not in ("ekaterinburg",) else "")
|
||||
print(f" {city:<30} {n}{flag}")
|
||||
print(f" (no slug parse): {no_match}")
|
||||
|
||||
# Дальше — sample non-EKB
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT source_id, source_url, address, price_rub
|
||||
FROM listings
|
||||
WHERE source='avito'
|
||||
AND source_url IS NOT NULL
|
||||
AND source_url !~ 'avito\\.ru/ekaterinburg/'
|
||||
ORDER BY scraped_at DESC NULLS LAST
|
||||
LIMIT 10
|
||||
"""
|
||||
)
|
||||
print("\nsample non-EKB listings (by URL):")
|
||||
for sid, url, addr, price in cur.fetchall():
|
||||
print(f" {sid} {price:,}₽")
|
||||
print(f" URL: {url}")
|
||||
print(f" addr: {(addr or '')[:80]!r}")
|
||||
74
tradein-mvp/scripts/probe-migration-028.py
Normal file
74
tradein-mvp/scripts/probe-migration-028.py
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
"""Проверка состояния миграции 028_matching_tables.sql на прод tradein-postgres."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import psycopg
|
||||
|
||||
dsn = os.environ["DATABASE_URL"].replace("postgresql+psycopg://", "postgresql://")
|
||||
conn = psycopg.connect(dsn)
|
||||
cur = conn.cursor()
|
||||
|
||||
EXPECTED_COLUMNS = [
|
||||
("houses", "address_fingerprint"),
|
||||
("listings", "canonical"),
|
||||
("listings", "merged_into"),
|
||||
]
|
||||
EXPECTED_INDEXES = [
|
||||
"houses_addr_fp_idx",
|
||||
"listings_merged_into_idx",
|
||||
"house_sources_house_idx",
|
||||
"house_sources_low_conf_idx",
|
||||
"listing_sources_listing_idx",
|
||||
"listing_sources_low_conf_idx",
|
||||
"listing_sources_scraped_idx",
|
||||
"haa_house_idx",
|
||||
"haa_fingerprint_idx",
|
||||
]
|
||||
EXPECTED_TABLES = ["house_sources", "listing_sources", "house_address_aliases"]
|
||||
|
||||
print("=== Columns ===")
|
||||
for tbl, col in EXPECTED_COLUMNS:
|
||||
cur.execute(
|
||||
"SELECT 1 FROM information_schema.columns WHERE table_name=%s AND column_name=%s",
|
||||
(tbl, col),
|
||||
)
|
||||
status = "✓" if cur.fetchone() else "✗ MISSING"
|
||||
print(f" {tbl}.{col:<25} {status}")
|
||||
|
||||
print("\n=== Tables ===")
|
||||
for tbl in EXPECTED_TABLES:
|
||||
cur.execute(
|
||||
"SELECT 1 FROM information_schema.tables WHERE table_name=%s AND table_schema='public'",
|
||||
(tbl,),
|
||||
)
|
||||
status = "✓" if cur.fetchone() else "✗ MISSING"
|
||||
print(f" {tbl:<30} {status}")
|
||||
|
||||
print("\n=== Indexes ===")
|
||||
for idx in EXPECTED_INDEXES:
|
||||
cur.execute(
|
||||
"SELECT 1 FROM pg_indexes WHERE indexname=%s",
|
||||
(idx,),
|
||||
)
|
||||
status = "✓" if cur.fetchone() else "✗ MISSING"
|
||||
print(f" {idx:<35} {status}")
|
||||
|
||||
# Проверим есть ли активные транзакции/locks от deploy которые ещё висят
|
||||
print("\n=== Active transactions (locks on relevant tables) ===")
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT pid, usename, application_name, state, wait_event_type, wait_event,
|
||||
query_start, substring(query, 1, 80) AS query_head
|
||||
FROM pg_stat_activity
|
||||
WHERE state IS DISTINCT FROM 'idle'
|
||||
AND pid != pg_backend_pid()
|
||||
ORDER BY query_start NULLS LAST
|
||||
"""
|
||||
)
|
||||
rows = cur.fetchall()
|
||||
if not rows:
|
||||
print(" (none — clean)")
|
||||
else:
|
||||
for r in rows:
|
||||
print(f" pid={r[0]} user={r[1]} app={r[2]} state={r[3]} wait={r[4]}/{r[5]} started={r[6]}")
|
||||
print(f" query: {r[7]!r}")
|
||||
90
tradein-mvp/scripts/probe-migration-status.py
Normal file
90
tradein-mvp/scripts/probe-migration-status.py
Normal file
|
|
@ -0,0 +1,90 @@
|
|||
"""Проверка какие миграции 028..054 НЕ применены."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import psycopg
|
||||
|
||||
dsn = os.environ["DATABASE_URL"].replace("postgresql+psycopg://", "postgresql://")
|
||||
conn = psycopg.connect(dsn)
|
||||
cur = conn.cursor()
|
||||
|
||||
# Маркеры по миграциям (table/column/index/etc)
|
||||
CHECKS = [
|
||||
("028 matching_tables", "table", "house_sources"),
|
||||
("029 extend matching/valuation/dynamics", "column", ("listing_sources", "price_divergence_pct")),
|
||||
("030 avito_imv cache_key unique", "index_unique", ("avito_imv_evaluations", "cache_key")),
|
||||
("030 listings_alter_yandex", "column", ("listings", "yandex_id")),
|
||||
("031 houses_alter_yandex", "column", ("houses", "yandex_id")),
|
||||
("032 yandex_history", "table", "yandex_house_history"),
|
||||
("040 houses_extend", "column", ("houses", "year_renovated")),
|
||||
("041 house_sources noop", "noop", None),
|
||||
("042 listing_sources price_divergence_idx", "index", "listing_sources_price_divergence_idx"),
|
||||
("043 house_reviews_extend", "column", ("house_reviews", "review_text")),
|
||||
("044 external_valuations_link", "column", ("external_valuations", "link_url")),
|
||||
("045 house_placement_history_extend", "column", ("house_placement_history", "first_seen_at")),
|
||||
("046 views", "view", "v_house_canonical"),
|
||||
("050 search_optimization", "matview", "listings_search_mv"),
|
||||
("051 scrape_runs_extend", "column", ("scrape_runs", "heartbeat_at")),
|
||||
("052 scrape_schedules", "table", "scrape_schedules"),
|
||||
("053 scraper_settings", "table", "scraper_settings"),
|
||||
("054 scraper_settings_global", "row", ("scraper_settings", "source='global'")),
|
||||
]
|
||||
|
||||
|
||||
def check(kind: str, target) -> bool:
|
||||
if kind == "noop":
|
||||
return True
|
||||
if kind == "table":
|
||||
cur.execute(
|
||||
"SELECT 1 FROM information_schema.tables WHERE table_schema='public' AND table_name=%s",
|
||||
(target,),
|
||||
)
|
||||
elif kind == "view":
|
||||
cur.execute(
|
||||
"SELECT 1 FROM information_schema.views WHERE table_schema='public' AND table_name=%s",
|
||||
(target,),
|
||||
)
|
||||
elif kind == "matview":
|
||||
cur.execute("SELECT 1 FROM pg_matviews WHERE matviewname=%s", (target,))
|
||||
elif kind == "column":
|
||||
tbl, col = target
|
||||
cur.execute(
|
||||
"SELECT 1 FROM information_schema.columns WHERE table_name=%s AND column_name=%s",
|
||||
(tbl, col),
|
||||
)
|
||||
elif kind == "index":
|
||||
cur.execute("SELECT 1 FROM pg_indexes WHERE indexname=%s", (target,))
|
||||
elif kind == "index_unique":
|
||||
tbl, col = target
|
||||
cur.execute(
|
||||
"""SELECT 1 FROM pg_indexes WHERE tablename=%s AND indexdef ~ %s""",
|
||||
(tbl, f"UNIQUE.*{col}"),
|
||||
)
|
||||
elif kind == "row":
|
||||
tbl, where = target
|
||||
cur.execute(f"SELECT 1 FROM {tbl} WHERE {where}")
|
||||
else:
|
||||
return False
|
||||
return cur.fetchone() is not None
|
||||
|
||||
|
||||
print(f"{'Migration':<48} Status")
|
||||
print("-" * 60)
|
||||
missing = []
|
||||
for name, kind, target in CHECKS:
|
||||
try:
|
||||
ok = check(kind, target)
|
||||
except Exception as e: # noqa: BLE001
|
||||
ok = False
|
||||
print(f"{name:<48} ✗ ERROR: {type(e).__name__}: {str(e)[:50]}")
|
||||
conn.rollback()
|
||||
missing.append(name)
|
||||
continue
|
||||
status = "✓" if ok else "✗ MISSING"
|
||||
print(f"{name:<48} {status}")
|
||||
if not ok:
|
||||
missing.append(name)
|
||||
|
||||
print(f"\nMissing: {len(missing)}/{len(CHECKS)}")
|
||||
for m in missing:
|
||||
print(f" - {m}")
|
||||
49
tradein-mvp/scripts/probe-parser-region-filter.py
Normal file
49
tradein-mvp/scripts/probe-parser-region-filter.py
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
"""Прогнать current parser на HTML из Playwright debug-страницы (своб.планир p=10).
|
||||
|
||||
Проверяет что URL-filter работает: до фикса parser возвращал ~30 карточек других
|
||||
городов, после — должен 0 или только ЕКБ.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
import types
|
||||
from pathlib import Path
|
||||
|
||||
_BACKEND = Path(__file__).resolve().parent.parent / "backend"
|
||||
if str(_BACKEND) not in sys.path:
|
||||
sys.path.insert(0, str(_BACKEND))
|
||||
for _m in ("weasyprint", "weasyprint.text", "weasyprint.text.ffi", "weasyprint.css", "weasyprint.css.targets"):
|
||||
sys.modules[_m] = types.ModuleType(_m)
|
||||
sys.modules["weasyprint"].HTML = type("HTML", (), {"__init__": lambda self, *a, **kw: None})
|
||||
sys.modules["weasyprint"].CSS = type("CSS", (), {"__init__": lambda self, *a, **kw: None})
|
||||
|
||||
from app.services.scrapers.avito import AvitoScraper # noqa: E402
|
||||
|
||||
URL = "https://www.avito.ru/ekaterinburg/kvartiry/prodam/svobodnaya_planirovka-ASgBAgICAkSSA8YQygj8zzI?p=10&s=104"
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
async with AvitoScraper() as s:
|
||||
assert s._cffi is not None
|
||||
# Use scraper's own session — может попасть в 403 (curl_cffi), это OK,
|
||||
# тогда переключимся на reading from Playwright MCP page.
|
||||
print(f"Fetching {URL[:80]}...")
|
||||
try:
|
||||
r = await s._cffi.get(URL)
|
||||
print(f" HTTP {r.status_code}, html size: {len(r.text)}")
|
||||
if r.status_code != 200:
|
||||
print(" curl_cffi 403 — Avito не отдал. Пробни через MCP / browser save.")
|
||||
return
|
||||
except Exception as e: # noqa: BLE001
|
||||
print(f" fetch fail: {type(e).__name__}: {e}")
|
||||
return
|
||||
lots = s._parse_html(r.text, source_url_base=URL)
|
||||
print(f"\nParsed {len(lots)} EKB lots (после фильтра).")
|
||||
for lot in lots[:5]:
|
||||
print(f" {lot.source_id} {lot.source_url[:80]}")
|
||||
print(f" {lot.address or '-'!r}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
34
tradein-mvp/scripts/test-playwright.py
Normal file
34
tradein-mvp/scripts/test-playwright.py
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
"""Minimal Playwright test — открывает Chrome окно с Avito на 30 sec."""
|
||||
import asyncio
|
||||
from playwright.async_api import async_playwright
|
||||
|
||||
|
||||
async def go():
|
||||
async with async_playwright() as p:
|
||||
print("launching browser...", flush=True)
|
||||
browser = await p.chromium.launch(headless=False)
|
||||
print("browser launched, opening page...", flush=True)
|
||||
page = await browser.new_page()
|
||||
try:
|
||||
await page.goto(
|
||||
"https://www.avito.ru/ekaterinburg/kvartiry/prodam-ASgBAgICAUSSA8YQ?s=104&p=1",
|
||||
timeout=30000,
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"goto FAIL: {type(e).__name__}: {e}", flush=True)
|
||||
await asyncio.sleep(15)
|
||||
await browser.close()
|
||||
return
|
||||
print("page loaded", flush=True)
|
||||
try:
|
||||
title = await page.title()
|
||||
print(f"title: {title}", flush=True)
|
||||
except Exception as e:
|
||||
print(f"title FAIL: {e}", flush=True)
|
||||
print("окно должно быть видно ~30 сек", flush=True)
|
||||
await asyncio.sleep(30)
|
||||
await browser.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(go())
|
||||
Loading…
Add table
Reference in a new issue