70e45ce5e9
New scripts:
- avito-playwright-sweep.py — Playwright-based Avito sweep (bypasses Qrator)
- backfill-yandex-addresses-ekb.py — fill street-only listings.address с house number
- local-sweep-ekb-{cian,yandex,}.py — full ЕКБ sweep through residential IP
- probe-* — quick diagnostics (region leak, migration status, parser filter, db counts)
- inspect-{houses,search}-html.py — Avito DOM structure analysis
- test-playwright.py — minimal Playwright smoke
Phase C script updates (backfill-house-imv.py):
- Pipe avito_imv module logs to avito-raw.log file (env AVITO_LOG_PATH override)
- Clamp rooms=0 (studio) → 1 — Avito IMV API rejects rooms=0 with HTTP 400
90 lines
3.8 KiB
Python
90 lines
3.8 KiB
Python
"""One-off: fetch Avito houses HTML и анализ markers для понимания новой структуры.
|
||
|
||
После 2026-05-23 Avito поменяли rendering — старый `window.__preloadedState__`
|
||
больше не работает (Stage 2c parser fails). Этот скрипт fetch'ит один houses
|
||
catalog page локально через residential IP и сохраняет HTML + проверяет markers
|
||
чтобы понять new format (__NEXT_DATA__? React SSR? иной).
|
||
"""
|
||
import sys
|
||
import types
|
||
from pathlib import Path
|
||
|
||
_BACKEND = Path(__file__).resolve().parent.parent / "backend"
|
||
if str(_BACKEND) not in sys.path:
|
||
sys.path.insert(0, str(_BACKEND))
|
||
|
||
for _m in ("weasyprint", "weasyprint.text", "weasyprint.text.ffi", "weasyprint.css", "weasyprint.css.targets"):
|
||
sys.modules[_m] = types.ModuleType(_m)
|
||
sys.modules["weasyprint"].HTML = type("HTML", (), {"__init__": lambda self, *a, **kw: None})
|
||
sys.modules["weasyprint"].CSS = type("CSS", (), {"__init__": lambda self, *a, **kw: None})
|
||
|
||
import asyncio # noqa: E402
|
||
|
||
from curl_cffi.requests import AsyncSession # noqa: E402
|
||
|
||
URLS = [
|
||
"https://www.avito.ru/catalog/houses/ekaterinburg/ul_yumasheva_6/115188",
|
||
"https://www.avito.ru/catalog/houses/ekaterinburg/ul_akademika_postovskogo_17a/3171365",
|
||
]
|
||
|
||
|
||
async def main() -> None:
|
||
async with AsyncSession(
|
||
impersonate="chrome120",
|
||
timeout=25,
|
||
headers={
|
||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||
"Accept-Language": "ru-RU,ru;q=0.9",
|
||
"Sec-Fetch-Dest": "document",
|
||
"Sec-Fetch-Mode": "navigate",
|
||
"Sec-Fetch-Site": "none",
|
||
},
|
||
) as s:
|
||
for url in URLS:
|
||
print(f"\n=== {url} ===")
|
||
try:
|
||
r = await s.get(url)
|
||
except Exception as e: # noqa: BLE001
|
||
print(f" FAIL fetch: {type(e).__name__}: {e}")
|
||
continue
|
||
print(f" status={r.status_code} size={len(r.content)}")
|
||
if r.status_code != 200:
|
||
continue
|
||
text = r.text
|
||
# Save first one для inspection
|
||
if "yumasheva" in url:
|
||
with Path("/tmp/avito-house-sample.html").open("wb") as f:
|
||
f.write(r.content)
|
||
print(" saved → /tmp/avito-house-sample.html")
|
||
markers = {
|
||
"__preloadedState__": "__preloadedState__" in text,
|
||
"__NEXT_DATA__": "__NEXT_DATA__" in text,
|
||
"__INITIAL_STATE__": "__INITIAL_STATE__" in text,
|
||
"datadome": "datadome" in text.lower(),
|
||
"captcha": "captcha" in text.lower(),
|
||
"data-marker": "data-marker" in text,
|
||
"data-app-state": "data-app-state" in text,
|
||
"developmentData": "developmentData" in text,
|
||
"avitoId": "avitoId" in text,
|
||
"window.__": "window.__" in text,
|
||
"<script id=": "<script id=" in text,
|
||
"ssr-state": "ssr-state" in text,
|
||
"__NUXT__": "__NUXT__" in text,
|
||
"Не найдено": "Не найдено" in text,
|
||
"недоступна": "недоступна" in text,
|
||
}
|
||
for k, v in markers.items():
|
||
print(f" {'✓' if v else '✗'} {k}")
|
||
# Look for script id="..." patterns (Next.js typical)
|
||
import re
|
||
script_ids = re.findall(r'<script[^>]*id="([^"]+)"', text)
|
||
if script_ids:
|
||
print(f" script IDs found: {script_ids[:10]}")
|
||
data_attrs = re.findall(r'<\w+\s+(data-[\w-]+)=', text)
|
||
if data_attrs:
|
||
unique = sorted(set(data_attrs))[:15]
|
||
print(f" data attrs sample: {unique}")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
asyncio.run(main())
|