70e45ce5e9
New scripts:
- avito-playwright-sweep.py — Playwright-based Avito sweep (bypasses Qrator)
- backfill-yandex-addresses-ekb.py — fill street-only listings.address с house number
- local-sweep-ekb-{cian,yandex,}.py — full ЕКБ sweep through residential IP
- probe-* — quick diagnostics (region leak, migration status, parser filter, db counts)
- inspect-{houses,search}-html.py — Avito DOM structure analysis
- test-playwright.py — minimal Playwright smoke
Phase C script updates (backfill-house-imv.py):
- Pipe avito_imv module logs to avito-raw.log file (env AVITO_LOG_PATH override)
- Clamp rooms=0 (studio) → 1 — Avito IMV API rejects rooms=0 with HTTP 400
129 lines
5.4 KiB
Python
129 lines
5.4 KiB
Python
"""Полная проверка региональной утечки в БД Avito listings.
|
||
|
||
Угол 1: source_url slug (canonical, надёжный)
|
||
Угол 2: address с маркером области (квартиры в ЕКБ не содержат "обл.")
|
||
Угол 3: address contains имя пригорода/соседнего города
|
||
Угол 4: top самых частых первых слов address (sanity check)
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
from collections import Counter
|
||
|
||
import psycopg
|
||
|
||
dsn = os.environ["DATABASE_URL"].replace("postgresql+psycopg://", "postgresql://")
|
||
conn = psycopg.connect(dsn)
|
||
cur = conn.cursor()
|
||
|
||
cur.execute("SELECT count(*) FROM listings WHERE source='avito'")
|
||
total = cur.fetchone()[0]
|
||
print(f"=== Avito total: {total} ===\n")
|
||
|
||
# Угол 1: URL slug (canonical)
|
||
print("[1] URL slug check:")
|
||
cur.execute(
|
||
"""
|
||
SELECT
|
||
count(*) FILTER (WHERE source_url ~ 'avito\\.ru/ekaterinburg/') AS ekb,
|
||
count(*) FILTER (WHERE source_url IS NOT NULL AND source_url !~ 'avito\\.ru/ekaterinburg/') AS not_ekb,
|
||
count(*) FILTER (WHERE source_url IS NULL) AS no_url
|
||
FROM listings WHERE source='avito'
|
||
"""
|
||
)
|
||
ekb, not_ekb, no_url = cur.fetchone()
|
||
print(f" ekaterinburg URL: {ekb}")
|
||
print(f" NOT ekaterinburg: {not_ekb} {'← LEAK' if not_ekb else '✓ clean'}")
|
||
print(f" NULL url: {no_url}")
|
||
|
||
# Угол 2: address с "область" / "обл."
|
||
print("\n[2] Address contains region marker (область/обл.):")
|
||
cur.execute(
|
||
"""
|
||
SELECT count(*) FROM listings
|
||
WHERE source='avito'
|
||
AND address IS NOT NULL
|
||
AND (address ILIKE '%область%' OR address ~* '\\bобл\\b' OR address ~* '\\bобл\\.')
|
||
"""
|
||
)
|
||
n = cur.fetchone()[0]
|
||
print(f" count: {n} {'← suspicious' if n > 5 else '✓ ok'}")
|
||
if n > 0:
|
||
cur.execute(
|
||
"""
|
||
SELECT source_url, address FROM listings
|
||
WHERE source='avito' AND address IS NOT NULL
|
||
AND (address ILIKE '%область%' OR address ~* '\\bобл\\b' OR address ~* '\\bобл\\.')
|
||
LIMIT 10
|
||
"""
|
||
)
|
||
for url, addr in cur.fetchall():
|
||
slug = url.split('avito.ru/')[1].split('/')[0] if url and 'avito.ru/' in url else '?'
|
||
print(f" [{slug}] {addr[:90]!r}")
|
||
|
||
# Угол 3: address contains имя другого города/пригорода
|
||
print("\n[3] Address contains satellite-city marker:")
|
||
satellites = [
|
||
"Верхняя Пышма", "Арамиль", "Сысерть", "Дегтярск", "Первоуральск",
|
||
"Заречный", "Невьянск", "Нижний Тагил", "Кыштым", "Челябинск",
|
||
"Алапаевск", "Краснофимск", "Озёрск", "Озерск", "Снежинск", "Касли",
|
||
"Рефтинский", "Свободный", "Большой Исток", "Пышма", "Покровское",
|
||
"Реж", "Лесной", "Полевской", "Берёзовск", "Березовск", "Михайловск",
|
||
"Туринск", "Североуральск", "Тавда", "Богданович", "Артёмовский",
|
||
"Артемовский", "Качканар", "Ивдель", "Серов", "Ирбит", "Новоуральск",
|
||
"Камышлов", "Сухой Лог",
|
||
]
|
||
for city in satellites:
|
||
cur.execute(
|
||
"SELECT count(*) FROM listings WHERE source='avito' AND address ILIKE %s",
|
||
(f"%{city}%",),
|
||
)
|
||
n = cur.fetchone()[0]
|
||
if n > 0:
|
||
print(f" {city}: {n}")
|
||
|
||
cur.execute(
|
||
"""
|
||
SELECT count(*) FROM listings WHERE source='avito' AND (
|
||
address ILIKE '%Верхняя Пышма%' OR address ILIKE '%Арамиль%' OR
|
||
address ILIKE '%Сысерть%' OR address ILIKE '%Дегтярск%' OR
|
||
address ILIKE '%Первоуральск%' OR address ILIKE '%Заречный%' OR
|
||
address ILIKE '%Невьянск%' OR address ILIKE '%Нижний Тагил%' OR
|
||
address ILIKE '%Кыштым%' OR address ILIKE '%Челябинск%' OR
|
||
address ILIKE '%Алапаевск%' OR address ILIKE '%Краснофимск%' OR
|
||
address ILIKE '%Озёрск%' OR address ILIKE '%Озерск%' OR
|
||
address ILIKE '%Снежинск%' OR address ILIKE '%Касли%' OR
|
||
address ILIKE '%Рефтинский%' OR address ILIKE '%Большой Исток%' OR
|
||
address ILIKE '%Реж%' OR address ILIKE '%Лесной%' OR address ILIKE '%Полевской%' OR
|
||
address ILIKE '%Берёзовск%' OR address ILIKE '%Березовск%'
|
||
)
|
||
"""
|
||
)
|
||
sat_total = cur.fetchone()[0]
|
||
print(f" TOTAL satellite-city leaks: {sat_total}")
|
||
|
||
# Угол 4: top first-words of address
|
||
print("\n[4] Top 20 first-words in address (sanity check):")
|
||
cur.execute(
|
||
"""
|
||
SELECT split_part(trim(address), ' ', 1) AS first_word, count(*) AS n
|
||
FROM listings WHERE source='avito' AND COALESCE(address, '') != ''
|
||
GROUP BY 1 ORDER BY 2 DESC LIMIT 20
|
||
"""
|
||
)
|
||
for word, n in cur.fetchall():
|
||
print(f" {n:>5} {word!r}")
|
||
|
||
# Угол 5: address NULL — сколько неopределённых
|
||
print("\n[5] Coverage:")
|
||
cur.execute(
|
||
"""
|
||
SELECT
|
||
count(*) FILTER (WHERE COALESCE(address, '') = '') AS no_addr,
|
||
count(*) FILTER (WHERE COALESCE(address, '') != '') AS has_addr
|
||
FROM listings WHERE source='avito'
|
||
"""
|
||
)
|
||
no_a, has_a = cur.fetchone()
|
||
print(f" has address: {has_a}")
|
||
print(f" no address: {no_a}")
|