70e45ce5e9
New scripts:
- avito-playwright-sweep.py — Playwright-based Avito sweep (bypasses Qrator)
- backfill-yandex-addresses-ekb.py — fill street-only listings.address с house number
- local-sweep-ekb-{cian,yandex,}.py — full ЕКБ sweep through residential IP
- probe-* — quick diagnostics (region leak, migration status, parser filter, db counts)
- inspect-{houses,search}-html.py — Avito DOM structure analysis
- test-playwright.py — minimal Playwright smoke
Phase C script updates (backfill-house-imv.py):
- Pipe avito_imv module logs to avito-raw.log file (env AVITO_LOG_PATH override)
- Clamp rooms=0 (studio) → 1 — Avito IMV API rejects rooms=0 with HTTP 400
103 lines
3.2 KiB
Python
103 lines
3.2 KiB
Python
"""Avito probe: 1 anchor × 1 page с настраиваемой задержкой.
|
||
|
||
Минимальный dry-run чтобы проверить bans / detection после anti-bot
|
||
hardening (PR #487) + global delay setting (PR #485).
|
||
|
||
Setup:
|
||
ssh -f -N -L 15433:172.20.0.2:5432 gendesign
|
||
cd tradein-mvp/backend
|
||
$env:DATABASE_URL = 'postgresql+psycopg://tradein:<PASS>@localhost:15433/tradein'
|
||
|
||
Usage:
|
||
uv run python ../scripts/probe-avito-1anchor.py
|
||
uv run python ../scripts/probe-avito-1anchor.py --delay 5 --radius 1000
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import asyncio
|
||
import sys
|
||
import time
|
||
import types
|
||
from pathlib import Path
|
||
|
||
_BACKEND_DIR = Path(__file__).resolve().parent.parent / "backend"
|
||
if str(_BACKEND_DIR) not in sys.path:
|
||
sys.path.insert(0, str(_BACKEND_DIR))
|
||
|
||
for _m in (
|
||
"weasyprint",
|
||
"weasyprint.text",
|
||
"weasyprint.text.ffi",
|
||
"weasyprint.css",
|
||
"weasyprint.css.targets",
|
||
):
|
||
sys.modules[_m] = types.ModuleType(_m)
|
||
sys.modules["weasyprint"].HTML = type("HTML", (), {"__init__": lambda self, *a, **kw: None})
|
||
sys.modules["weasyprint"].CSS = type("CSS", (), {"__init__": lambda self, *a, **kw: None})
|
||
|
||
from app.core.db import SessionLocal # noqa: E402
|
||
from app.services.scrapers.avito import AvitoScraper # noqa: E402
|
||
from app.services.scrapers.base import save_listings # noqa: E402
|
||
|
||
ANCHOR_NAME = "Центр ЕКБ"
|
||
ANCHOR_LAT = 56.8400
|
||
ANCHOR_LON = 60.6050
|
||
|
||
|
||
async def main() -> None:
|
||
p = argparse.ArgumentParser()
|
||
p.add_argument("--delay", type=float, default=3.0, help="sec между HTTP requests")
|
||
p.add_argument("--radius", type=int, default=1500, help="метры")
|
||
p.add_argument("--pages", type=int, default=1)
|
||
args = p.parse_args()
|
||
|
||
print(
|
||
f"=== Avito probe ===\n"
|
||
f" anchor: {ANCHOR_NAME} ({ANCHOR_LAT}, {ANCHOR_LON})\n"
|
||
f" radius: {args.radius}m\n"
|
||
f" pages: {args.pages}\n"
|
||
f" delay: {args.delay}s\n",
|
||
flush=True,
|
||
)
|
||
|
||
t0 = time.perf_counter()
|
||
try:
|
||
async with AvitoScraper() as s:
|
||
lots = await s.fetch_around(
|
||
ANCHOR_LAT, ANCHOR_LON, args.radius,
|
||
pages=args.pages, delay_override_sec=args.delay,
|
||
)
|
||
except Exception as e: # noqa: BLE001
|
||
print(f"FETCH FAIL: {type(e).__name__}: {e}", flush=True)
|
||
return
|
||
|
||
fetched = len(lots)
|
||
print(f"fetched: {fetched} lots in {time.perf_counter() - t0:.1f}s", flush=True)
|
||
if fetched == 0:
|
||
print("⚠️ 0 lots — вероятно 429/ban. Смотри backend logs.", flush=True)
|
||
return
|
||
|
||
db = SessionLocal()
|
||
try:
|
||
ins, upd = save_listings(db, lots)
|
||
print(f"saved: ins={ins} upd={upd}", flush=True)
|
||
except Exception as e: # noqa: BLE001
|
||
print(f"SAVE FAIL: {type(e).__name__}: {e}", flush=True)
|
||
finally:
|
||
db.close()
|
||
|
||
sample = lots[:3]
|
||
print("\nsample listings:", flush=True)
|
||
for lot in sample:
|
||
addr = (lot.address or "")[:60]
|
||
print(
|
||
f" source_id={lot.source_id} price={lot.price_rub}₽ "
|
||
f"rooms={lot.rooms} area={lot.area_m2}м² floor={lot.floor}/{lot.total_floors} "
|
||
f"addr={addr!r}",
|
||
flush=True,
|
||
)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
asyncio.run(main())
|