75 lines
2.6 KiB
Python
75 lines
2.6 KiB
Python
"""Проверка: есть ли в DB Avito листинги из других городов (region leak)."""
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
import psycopg
|
||
|
||
dsn = os.environ["DATABASE_URL"].replace("postgresql+psycopg://", "postgresql://")
|
||
conn = psycopg.connect(dsn)
|
||
cur = conn.cursor()
|
||
|
||
cur.execute("SELECT count(*) FROM listings WHERE source='avito'")
|
||
total = cur.fetchone()[0]
|
||
print(f"avito total: {total}")
|
||
|
||
# Все с lat/lon — проверим bounding box ЕКБ (56.7..57.0, 60.4..60.8)
|
||
cur.execute(
|
||
"""
|
||
SELECT
|
||
count(*) FILTER (WHERE lat BETWEEN 56.5 AND 57.0 AND lon BETWEEN 60.4 AND 61.0) as in_ekb,
|
||
count(*) FILTER (WHERE lat IS NOT NULL AND (lat < 56.5 OR lat > 57.0 OR lon < 60.4 OR lon > 61.0)) as outside_ekb,
|
||
count(*) FILTER (WHERE lat IS NULL) as no_coords
|
||
FROM listings WHERE source='avito'
|
||
"""
|
||
)
|
||
in_ekb, outside, no_coords = cur.fetchone()
|
||
print(f" in EKB bbox: {in_ekb}")
|
||
print(f" outside EKB bbox: {outside} ← out-of-region leak!")
|
||
print(f" no coords yet: {no_coords}")
|
||
|
||
# Sample outside
|
||
cur.execute(
|
||
"""
|
||
SELECT source_id, lat, lon, address, price_rub, rooms
|
||
FROM listings
|
||
WHERE source='avito' AND lat IS NOT NULL
|
||
AND (lat < 56.5 OR lat > 57.0 OR lon < 60.4 OR lon > 61.0)
|
||
ORDER BY scraped_at DESC NULLS LAST
|
||
LIMIT 10
|
||
"""
|
||
)
|
||
print("\nsample non-EKB listings:")
|
||
for r in cur.fetchall():
|
||
sid, lat, lon, addr, price, rooms = r
|
||
print(f" {sid} ({lat:.3f},{lon:.3f}) rooms={rooms} {price:,}₽ — {(addr or '')[:80]!r}")
|
||
|
||
# Address-based leak (без lat/lon) — поиск явных маркеров других городов
|
||
cur.execute(
|
||
"""
|
||
SELECT count(*) FROM listings
|
||
WHERE source='avito'
|
||
AND lat IS NULL
|
||
AND address IS NOT NULL
|
||
AND (address ILIKE '%москв%' OR address ILIKE '%омск%' OR address ILIKE '%чебок%'
|
||
OR address ILIKE '%петерб%' OR address ILIKE '%казан%' OR address ILIKE '%новосиб%')
|
||
"""
|
||
)
|
||
print(f"\naddress-based leak (no coords, other-city keyword): {cur.fetchone()[0]}")
|
||
|
||
# По комнатам — где leak вероятнее всего
|
||
cur.execute(
|
||
"""
|
||
SELECT rooms,
|
||
count(*) AS total,
|
||
count(*) FILTER (WHERE lat IS NOT NULL AND (lat < 56.5 OR lat > 57.0 OR lon < 60.4 OR lon > 61.0)) AS leak
|
||
FROM listings WHERE source='avito'
|
||
GROUP BY rooms
|
||
ORDER BY rooms NULLS LAST
|
||
"""
|
||
)
|
||
print("\nleak by rooms:")
|
||
print(f" {'rooms':<8} {'total':>8} {'leak':>8}")
|
||
for r in cur.fetchall():
|
||
rooms, total, leak = r
|
||
pct = (leak * 100.0 / total) if total else 0
|
||
print(f" {str(rooms):<8} {total:>8} {leak:>8} ({pct:.0f}%)")
|