371 lines
16 KiB
Python
371 lines
16 KiB
Python
"""Playwright-based Avito sweep — обходит Qrator JS challenge через real headless Chromium.
|
||
|
||
Use case: curl_cffi заблочен Qrator anti-DDoS (429 + captcha challenge). Real
|
||
browser proходит JS challenge автоматически.
|
||
|
||
Setup:
|
||
cd tradein-mvp/backend
|
||
uv pip install playwright
|
||
uv run playwright install chromium
|
||
|
||
Usage:
|
||
export DATABASE_URL='postgresql+psycopg://tradein:<PASS>@localhost:15433/tradein'
|
||
uv run --project tradein-mvp/backend python tradein-mvp/scripts/avito-playwright-sweep.py
|
||
# options: --pages N --delay S --mode {citywide|byrooms} --headless
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
# Banner FIRST — даже до imports чтобы видно что скрипт стартанул:
|
||
import sys as _sys
|
||
|
||
print("=" * 60, flush=True)
|
||
print("AVITO PLAYWRIGHT SWEEP — starting...", flush=True)
|
||
print("=" * 60, flush=True)
|
||
_sys.stdout.flush()
|
||
|
||
import argparse
|
||
import asyncio
|
||
import json
|
||
import os
|
||
import random
|
||
import sys
|
||
import time
|
||
import types
|
||
from pathlib import Path
|
||
from urllib.parse import urlencode
|
||
|
||
# Force unbuffered output
|
||
os.environ["PYTHONUNBUFFERED"] = "1"
|
||
|
||
_BACKEND = Path(__file__).resolve().parent.parent / "backend"
|
||
if str(_BACKEND) not in sys.path:
|
||
sys.path.insert(0, str(_BACKEND))
|
||
|
||
# WeasyPrint stub (Windows без GTK)
|
||
for _m in ("weasyprint", "weasyprint.text", "weasyprint.text.ffi", "weasyprint.css", "weasyprint.css.targets"):
|
||
sys.modules[_m] = types.ModuleType(_m)
|
||
sys.modules["weasyprint"].HTML = type("HTML", (), {"__init__": lambda self, *a, **kw: None})
|
||
sys.modules["weasyprint"].CSS = type("CSS", (), {"__init__": lambda self, *a, **kw: None})
|
||
|
||
from playwright.async_api import Browser, BrowserContext, async_playwright # noqa: E402
|
||
|
||
try:
|
||
from playwright_stealth import Stealth # noqa: E402
|
||
_STEALTH_AVAILABLE = True
|
||
except ImportError:
|
||
_STEALTH_AVAILABLE = False
|
||
print("[warn] playwright-stealth not installed — без stealth, может triggered больше CAPTCHA", flush=True)
|
||
|
||
from app.core.db import SessionLocal # noqa: E402
|
||
from app.services.scrapers.avito import AvitoScraper # noqa: E402
|
||
from app.services.scrapers.base import save_listings # noqa: E402
|
||
|
||
COOKIES_FILE = Path(__file__).resolve().parent / ".avito-cookies.json"
|
||
AVITO_BASE = "https://www.avito.ru"
|
||
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36"
|
||
|
||
# Reuse AvitoScraper's parser (DOM cards extraction)
|
||
_parser_instance = AvitoScraper()
|
||
_parser_instance._anchor_lat = 0.0 # noqa: SLF001
|
||
_parser_instance._anchor_lon = 0.0 # noqa: SLF001
|
||
_parser_instance._anchor_radius_m = 1000 # noqa: SLF001
|
||
|
||
|
||
def load_cookies() -> list[dict]:
|
||
"""Load DevTools-exported cookies → Playwright format."""
|
||
if not COOKIES_FILE.exists():
|
||
print(f"WARN: cookies file not found at {COOKIES_FILE} — будем без cookies")
|
||
return []
|
||
raw = json.loads(COOKIES_FILE.read_text(encoding="utf-8"))
|
||
out = []
|
||
for c in raw:
|
||
ck = {
|
||
"name": c["name"],
|
||
"value": c["value"],
|
||
"domain": c["domain"],
|
||
"path": c.get("path", "/"),
|
||
"secure": c.get("secure", True),
|
||
"httpOnly": c.get("httpOnly", False),
|
||
}
|
||
if c.get("expirationDate"):
|
||
ck["expires"] = int(c["expirationDate"])
|
||
ss = c.get("sameSite")
|
||
if ss == "no_restriction":
|
||
ck["sameSite"] = "None"
|
||
elif ss == "lax":
|
||
ck["sameSite"] = "Lax"
|
||
elif ss == "strict":
|
||
ck["sameSite"] = "Strict"
|
||
out.append(ck)
|
||
return out
|
||
|
||
|
||
def build_citywide_url(page: int) -> str:
|
||
return f"{AVITO_BASE}/ekaterinburg/kvartiry/prodam-ASgBAgICAUSSA8YQ?{urlencode({'s':104,'p':page})}"
|
||
|
||
|
||
ROOM_SLUGS: list[tuple[str, str]] = [
|
||
("студии", "studii-ASgBAgICAkSSA8YQygj~WA"),
|
||
("1-комн.", "1-komnatnye-ASgBAgICAkSSA8YQygiAWQ"),
|
||
("2-комн.", "2-komnatnye-ASgBAgICAkSSA8YQygiCWQ"),
|
||
("3-комн.", "3-komnatnye-ASgBAgICAkSSA8YQygiEWQ"),
|
||
("4-комн.", "4-komnatnye-ASgBAgICAkSSA8YQygiGWQ"),
|
||
("5+комн.", "5-komnatnye-ASgBAgICAkSSA8YQygiIWQ"),
|
||
("своб.планир.", "svobodnaya_planirovka-ASgBAgICAkSSA8YQygj8zzI"),
|
||
]
|
||
|
||
|
||
def build_rooms_url(slug: str, page: int) -> str:
|
||
return f"{AVITO_BASE}/ekaterinburg/kvartiry/prodam/{slug}?{urlencode({'s':104,'p':page})}"
|
||
|
||
|
||
async def fetch_page(context: BrowserContext, url: str) -> tuple[int, str | None]:
|
||
"""Open page in Playwright, wait for content, return (status, html or None).
|
||
|
||
Strategy:
|
||
1. goto с wait_until='domcontentloaded' (быстро, не ждём всех scripts)
|
||
2. Если status 200 → wait_for_selector data-marker=item (Qrator challenge должен resolve пока)
|
||
3. Иначе если 429/403 — challenge не прошёл, дать ещё 5s + retry content (maybe challenge resolves async)
|
||
"""
|
||
page = await context.new_page()
|
||
try:
|
||
try:
|
||
response = await page.goto(url, wait_until="domcontentloaded", timeout=45000)
|
||
except Exception as e: # noqa: BLE001
|
||
print(f" goto FAIL: {type(e).__name__}: {str(e)[:80]}")
|
||
return 0, None
|
||
status = response.status if response else 0
|
||
if status in (429, 403):
|
||
# Qrator challenge — wait для JS auto-resolve, потом reload
|
||
await asyncio.sleep(5)
|
||
try:
|
||
response = await page.reload(wait_until="domcontentloaded", timeout=30000)
|
||
status = response.status if response else status
|
||
except Exception: # noqa: BLE001
|
||
pass
|
||
if status != 200:
|
||
return status, None
|
||
# Wait for item cards
|
||
try:
|
||
await page.wait_for_selector('[data-marker="item"]', timeout=15000)
|
||
except Exception: # noqa: BLE001
|
||
pass
|
||
html = await page.content()
|
||
return status, html
|
||
finally:
|
||
await page.close()
|
||
|
||
|
||
async def sweep_urls(
|
||
urls_with_label: list[tuple[str, str]],
|
||
delay: float,
|
||
headless: bool,
|
||
) -> int:
|
||
"""Generic sweep: list of (label, url) → fetch + parse + save per URL."""
|
||
cookies = load_cookies()
|
||
print(f"loaded {len(cookies)} cookies")
|
||
total_fetched = 0
|
||
total_ins = 0
|
||
total_upd = 0
|
||
challenge_hits = 0
|
||
print("=== Launching Playwright browser ===", flush=True)
|
||
async with async_playwright() as p:
|
||
print(f" headless={headless}", flush=True)
|
||
browser: Browser = await p.chromium.launch(
|
||
headless=headless,
|
||
args=[
|
||
"--disable-blink-features=AutomationControlled",
|
||
"--no-sandbox",
|
||
],
|
||
)
|
||
print(" browser launched, creating context...", flush=True)
|
||
context: BrowserContext = await browser.new_context(
|
||
user_agent=UA,
|
||
locale="ru-RU",
|
||
timezone_id="Asia/Yekaterinburg",
|
||
viewport={"width": 1366, "height": 900},
|
||
)
|
||
# Stealth — optional, skip on error
|
||
if _STEALTH_AVAILABLE:
|
||
try:
|
||
stealth = Stealth()
|
||
await stealth.apply_stealth_async(context)
|
||
print(" stealth applied", flush=True)
|
||
except Exception as e: # noqa: BLE001
|
||
print(f" stealth skipped (err): {type(e).__name__}: {e}", flush=True)
|
||
if cookies:
|
||
await context.add_cookies(cookies)
|
||
print(f" cookies added ({len(cookies)})", flush=True)
|
||
|
||
# Initial warmup: open main Avito + wait для user pass CAPTCHA если нужно
|
||
print("\n=== WARMUP: opening Avito initial page ===", flush=True)
|
||
print(" creating new page tab...", flush=True)
|
||
warmup_page = await context.new_page()
|
||
print(" navigating to Avito (timeout 30s)...", flush=True)
|
||
try:
|
||
await warmup_page.goto(
|
||
"https://www.avito.ru/ekaterinburg/kvartiry/prodam-ASgBAgICAUSSA8YQ?s=104&p=1",
|
||
wait_until="domcontentloaded",
|
||
timeout=30000,
|
||
)
|
||
print(" goto done, reading content...", flush=True)
|
||
except Exception as e: # noqa: BLE001
|
||
print(f" goto FAIL: {type(e).__name__}: {e}", flush=True)
|
||
# Check if challenge page — reliable detector: NO listing cards в HTML.
|
||
# Слово "captcha" может быть в JS embed страницы normally (false positive).
|
||
warmup_html = await warmup_page.content()
|
||
print(f" content read: {len(warmup_html)} bytes", flush=True)
|
||
try:
|
||
await warmup_page.wait_for_selector('[data-marker="item"]', timeout=5000)
|
||
has_items = True
|
||
except Exception: # noqa: BLE001
|
||
has_items = False
|
||
is_qrator = "qrator" in warmup_html.lower() and len(warmup_html) < 100000
|
||
challenge_present = (not has_items) and (is_qrator or len(warmup_html) < 50000)
|
||
print(f" has_items={has_items} qrator={is_qrator} size={len(warmup_html)} challenge={challenge_present}", flush=True)
|
||
if challenge_present:
|
||
print("\n🚨 CAPTCHA detected на initial page!")
|
||
print("👉 Окно браузера открыто — пройди капчу руками,")
|
||
print("👉 затем нажми Enter здесь чтобы продолжить sweep...")
|
||
try:
|
||
input(">>> Press ENTER when ready: ")
|
||
except (EOFError, KeyboardInterrupt):
|
||
print("aborted")
|
||
await browser.close()
|
||
return 0
|
||
# Verify challenge passed
|
||
await warmup_page.reload(wait_until="domcontentloaded", timeout=30000)
|
||
new_html = await warmup_page.content()
|
||
if any(m in new_html.lower() for m in ("captcha", "qrator")):
|
||
print("⚠️ всё ещё CAPTCHA — повторно пройди или Ctrl+C")
|
||
input(">>> Retry ENTER: ")
|
||
else:
|
||
print("✅ CAPTCHA пройдена, продолжаю sweep")
|
||
else:
|
||
print("✅ Initial page OK (no challenge)")
|
||
await warmup_page.close()
|
||
|
||
try:
|
||
# Per-category skip flag: 0 lots на странице категории = end of pagination
|
||
# ДЛЯ ТЕКУЩЕЙ КАТЕГОРИИ, а не для всего sweep'а (был bug: break ломал всё).
|
||
current_cat: str | None = None
|
||
skip_until_next_cat = False
|
||
for i, (label, url) in enumerate(urls_with_label, 1):
|
||
# Категория = всё до " page=" в label ("1-комн. page=5/60" → "1-комн.")
|
||
cat = label.split(" page=")[0] if " page=" in label else label
|
||
if cat != current_cat:
|
||
current_cat = cat
|
||
skip_until_next_cat = False
|
||
challenge_hits = 0 # reset 429-counter на границе категорий
|
||
if skip_until_next_cat:
|
||
continue
|
||
|
||
print(f"[{i}/{len(urls_with_label)}] {label}", flush=True)
|
||
status, html = await fetch_page(context, url)
|
||
if status != 200 or not html:
|
||
print(f" HTTP {status} → skip", flush=True)
|
||
if status in (429, 403):
|
||
challenge_hits += 1
|
||
if challenge_hits >= 3:
|
||
print(f" ⚠️ {challenge_hits}x challenge hits — sleep 60s extra", flush=True)
|
||
print(" если CAPTCHA в окне — пройди и нажми ENTER:", flush=True)
|
||
try:
|
||
input(" >>> ")
|
||
except (EOFError, KeyboardInterrupt):
|
||
break
|
||
challenge_hits = 0
|
||
await asyncio.sleep(delay)
|
||
continue
|
||
# Parse + save
|
||
lots = _parser_instance._parse_html(html, source_url_base=url) # noqa: SLF001
|
||
if not lots:
|
||
print(f" 0 lots — end of pagination for {cat!r}", flush=True)
|
||
if " page=" in label:
|
||
# 0 lots на пагинированной странице = end of category, не sweep
|
||
skip_until_next_cat = True
|
||
continue
|
||
db = SessionLocal()
|
||
try:
|
||
ins, upd = save_listings(db, lots)
|
||
total_fetched += len(lots)
|
||
total_ins += ins
|
||
total_upd += upd
|
||
print(
|
||
f" {len(lots)} lots → ins={ins} upd={upd} "
|
||
f"[cum: tot={total_fetched} new={total_ins}]",
|
||
flush=True,
|
||
)
|
||
except Exception as e: # noqa: BLE001
|
||
print(f" save FAIL: {type(e).__name__}: {e}", flush=True)
|
||
finally:
|
||
db.close()
|
||
# Jittered delay
|
||
if i < len(urls_with_label):
|
||
j = delay * random.uniform(0.7, 1.3)
|
||
await asyncio.sleep(j)
|
||
finally:
|
||
await context.close()
|
||
await browser.close()
|
||
print(
|
||
f"\n=== TOTAL: fetched={total_fetched} ins={total_ins} upd={total_upd} ===",
|
||
flush=True,
|
||
)
|
||
return total_fetched
|
||
|
||
|
||
async def main() -> None:
|
||
# Diagnostic — show env vars used:
|
||
db_url = os.environ.get("DATABASE_URL", "<NOT SET>")
|
||
# Mask password for logging
|
||
if "://" in db_url and "@" in db_url:
|
||
before_at, after_at = db_url.rsplit("@", 1)
|
||
scheme_user = before_at.split("://", 1)
|
||
if len(scheme_user) == 2 and ":" in scheme_user[1]:
|
||
user = scheme_user[1].split(":", 1)[0]
|
||
masked = f"{scheme_user[0]}://{user}:***@{after_at}"
|
||
else:
|
||
masked = db_url[:50] + "..."
|
||
else:
|
||
masked = db_url
|
||
print(f"DATABASE_URL: {masked}", flush=True)
|
||
print(f"ENVIRONMENT: {os.environ.get('ENVIRONMENT', '<default>')}", flush=True)
|
||
|
||
p = argparse.ArgumentParser()
|
||
p.add_argument("--mode", choices=["citywide", "byrooms"], default="citywide")
|
||
p.add_argument("--pages", type=int, default=100, help="pages per query")
|
||
p.add_argument("--delay", type=float, default=7, help="sec между requests")
|
||
p.add_argument("--headless", action="store_true", default=True)
|
||
p.add_argument("--no-headless", dest="headless", action="store_false")
|
||
p.add_argument(
|
||
"--start-cat",
|
||
type=int,
|
||
default=1,
|
||
help="byrooms: с какой категории начать (1-based). 1=студии, 2=1-комн, "
|
||
"3=2-комн, 4=3-комн, 5=4-комн, 6=5+комн, 7=своб.планир.",
|
||
)
|
||
args = p.parse_args()
|
||
|
||
urls: list[tuple[str, str]] = []
|
||
if args.mode == "citywide":
|
||
for page in range(1, args.pages + 1):
|
||
urls.append((f"citywide page={page}/{args.pages}", build_citywide_url(page)))
|
||
else: # byrooms
|
||
cats = ROOM_SLUGS[args.start_cat - 1:]
|
||
if not cats:
|
||
print(f"ERROR: --start-cat={args.start_cat} > {len(ROOM_SLUGS)} cats", flush=True)
|
||
return
|
||
print(f"byrooms: starting from cat #{args.start_cat} ({cats[0][0]!r}), "
|
||
f"{len(cats)} cats total", flush=True)
|
||
for cat_name, slug in cats:
|
||
for page in range(1, args.pages + 1):
|
||
urls.append((f"{cat_name} page={page}/{args.pages}", build_rooms_url(slug, page)))
|
||
|
||
print(f"Mode: {args.mode}, pages: {args.pages}, delay: {args.delay}s, total URLs: {len(urls)}")
|
||
print(f"Estimated time: {len(urls) * args.delay / 60:.0f} min (без challenge)")
|
||
t0 = time.perf_counter()
|
||
await sweep_urls(urls, args.delay, args.headless)
|
||
print(f"\nDONE in {(time.perf_counter() - t0) / 60:.1f} min")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
asyncio.run(main())
|