70e45ce5e9
New scripts:
- avito-playwright-sweep.py — Playwright-based Avito sweep (bypasses Qrator)
- backfill-yandex-addresses-ekb.py — fill street-only listings.address с house number
- local-sweep-ekb-{cian,yandex,}.py — full ЕКБ sweep through residential IP
- probe-* — quick diagnostics (region leak, migration status, parser filter, db counts)
- inspect-{houses,search}-html.py — Avito DOM structure analysis
- test-playwright.py — minimal Playwright smoke
Phase C script updates (backfill-house-imv.py):
- Pipe avito_imv module logs to avito-raw.log file (env AVITO_LOG_PATH override)
- Clamp rooms=0 (studio) → 1 — Avito IMV API rejects rooms=0 with HTTP 400
90 lines
3.4 KiB
Python
90 lines
3.4 KiB
Python
"""Проверка какие миграции 028..054 НЕ применены."""
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
import psycopg
|
||
|
||
dsn = os.environ["DATABASE_URL"].replace("postgresql+psycopg://", "postgresql://")
|
||
conn = psycopg.connect(dsn)
|
||
cur = conn.cursor()
|
||
|
||
# Маркеры по миграциям (table/column/index/etc)
|
||
CHECKS = [
|
||
("028 matching_tables", "table", "house_sources"),
|
||
("029 extend matching/valuation/dynamics", "column", ("listing_sources", "price_divergence_pct")),
|
||
("030 avito_imv cache_key unique", "index_unique", ("avito_imv_evaluations", "cache_key")),
|
||
("030 listings_alter_yandex", "column", ("listings", "yandex_id")),
|
||
("031 houses_alter_yandex", "column", ("houses", "yandex_id")),
|
||
("032 yandex_history", "table", "yandex_house_history"),
|
||
("040 houses_extend", "column", ("houses", "year_renovated")),
|
||
("041 house_sources noop", "noop", None),
|
||
("042 listing_sources price_divergence_idx", "index", "listing_sources_price_divergence_idx"),
|
||
("043 house_reviews_extend", "column", ("house_reviews", "review_text")),
|
||
("044 external_valuations_link", "column", ("external_valuations", "link_url")),
|
||
("045 house_placement_history_extend", "column", ("house_placement_history", "first_seen_at")),
|
||
("046 views", "view", "v_house_canonical"),
|
||
("050 search_optimization", "matview", "listings_search_mv"),
|
||
("051 scrape_runs_extend", "column", ("scrape_runs", "heartbeat_at")),
|
||
("052 scrape_schedules", "table", "scrape_schedules"),
|
||
("053 scraper_settings", "table", "scraper_settings"),
|
||
("054 scraper_settings_global", "row", ("scraper_settings", "source='global'")),
|
||
]
|
||
|
||
|
||
def check(kind: str, target) -> bool:
|
||
if kind == "noop":
|
||
return True
|
||
if kind == "table":
|
||
cur.execute(
|
||
"SELECT 1 FROM information_schema.tables WHERE table_schema='public' AND table_name=%s",
|
||
(target,),
|
||
)
|
||
elif kind == "view":
|
||
cur.execute(
|
||
"SELECT 1 FROM information_schema.views WHERE table_schema='public' AND table_name=%s",
|
||
(target,),
|
||
)
|
||
elif kind == "matview":
|
||
cur.execute("SELECT 1 FROM pg_matviews WHERE matviewname=%s", (target,))
|
||
elif kind == "column":
|
||
tbl, col = target
|
||
cur.execute(
|
||
"SELECT 1 FROM information_schema.columns WHERE table_name=%s AND column_name=%s",
|
||
(tbl, col),
|
||
)
|
||
elif kind == "index":
|
||
cur.execute("SELECT 1 FROM pg_indexes WHERE indexname=%s", (target,))
|
||
elif kind == "index_unique":
|
||
tbl, col = target
|
||
cur.execute(
|
||
"""SELECT 1 FROM pg_indexes WHERE tablename=%s AND indexdef ~ %s""",
|
||
(tbl, f"UNIQUE.*{col}"),
|
||
)
|
||
elif kind == "row":
|
||
tbl, where = target
|
||
cur.execute(f"SELECT 1 FROM {tbl} WHERE {where}")
|
||
else:
|
||
return False
|
||
return cur.fetchone() is not None
|
||
|
||
|
||
print(f"{'Migration':<48} Status")
|
||
print("-" * 60)
|
||
missing = []
|
||
for name, kind, target in CHECKS:
|
||
try:
|
||
ok = check(kind, target)
|
||
except Exception as e: # noqa: BLE001
|
||
ok = False
|
||
print(f"{name:<48} ✗ ERROR: {type(e).__name__}: {str(e)[:50]}")
|
||
conn.rollback()
|
||
missing.append(name)
|
||
continue
|
||
status = "✓" if ok else "✗ MISSING"
|
||
print(f"{name:<48} {status}")
|
||
if not ok:
|
||
missing.append(name)
|
||
|
||
print(f"\nMissing: {len(missing)}/{len(CHECKS)}")
|
||
for m in missing:
|
||
print(f" - {m}")
|