70e45ce5e9
New scripts:
- avito-playwright-sweep.py — Playwright-based Avito sweep (bypasses Qrator)
- backfill-yandex-addresses-ekb.py — fill street-only listings.address с house number
- local-sweep-ekb-{cian,yandex,}.py — full ЕКБ sweep through residential IP
- probe-* — quick diagnostics (region leak, migration status, parser filter, db counts)
- inspect-{houses,search}-html.py — Avito DOM structure analysis
- test-playwright.py — minimal Playwright smoke
Phase C script updates (backfill-house-imv.py):
- Pipe avito_imv module logs to avito-raw.log file (env AVITO_LOG_PATH override)
- Clamp rooms=0 (studio) → 1 — Avito IMV API rejects rooms=0 with HTTP 400
51 lines
1.4 KiB
Python
51 lines
1.4 KiB
Python
"""URL-based region leak check для Avito listings."""
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import re
|
|
from collections import Counter
|
|
|
|
import psycopg
|
|
|
|
dsn = os.environ["DATABASE_URL"].replace("postgresql+psycopg://", "postgresql://")
|
|
conn = psycopg.connect(dsn)
|
|
cur = conn.cursor()
|
|
|
|
cur.execute("SELECT source_url FROM listings WHERE source='avito'")
|
|
slug_re = re.compile(r"avito\.ru/([a-z_]+)/")
|
|
|
|
cities = Counter()
|
|
no_match = 0
|
|
for (url,) in cur.fetchall():
|
|
if not url:
|
|
no_match += 1
|
|
continue
|
|
m = slug_re.search(url)
|
|
if m:
|
|
cities[m.group(1)] += 1
|
|
else:
|
|
no_match += 1
|
|
|
|
print("city slug distribution in source_url:")
|
|
for city, n in cities.most_common():
|
|
flag = " ← EKB" if city == "ekaterinburg" else (" ← OTHER CITY" if city not in ("ekaterinburg",) else "")
|
|
print(f" {city:<30} {n}{flag}")
|
|
print(f" (no slug parse): {no_match}")
|
|
|
|
# Дальше — sample non-EKB
|
|
cur.execute(
|
|
"""
|
|
SELECT source_id, source_url, address, price_rub
|
|
FROM listings
|
|
WHERE source='avito'
|
|
AND source_url IS NOT NULL
|
|
AND source_url !~ 'avito\\.ru/ekaterinburg/'
|
|
ORDER BY scraped_at DESC NULLS LAST
|
|
LIMIT 10
|
|
"""
|
|
)
|
|
print("\nsample non-EKB listings (by URL):")
|
|
for sid, url, addr, price in cur.fetchall():
|
|
print(f" {sid} {price:,}₽")
|
|
print(f" URL: {url}")
|
|
print(f" addr: {(addr or '')[:80]!r}")
|