122 lines
3.6 KiB
Python
122 lines
3.6 KiB
Python
"""Backfill houses.cian_zhk_url for rows that have cian_internal_house_id but no URL.
|
|
|
|
Requires migration 071_houses_cian_zhk_url.sql to be applied first.
|
|
|
|
What it does:
|
|
SELECT houses WHERE cian_internal_house_id IS NOT NULL AND cian_zhk_url IS NULL
|
|
For each row: resolve https://cian.ru/zhk/<id>/ → follow redirect → store final URL.
|
|
|
|
Cian redirects the canonical /zhk/<id>/ path to the slug URL
|
|
(e.g. https://zhk-park-ekb-i.cian.ru/). We capture the final redirect destination.
|
|
|
|
Usage:
|
|
cd tradein-mvp/backend
|
|
DATABASE_URL=postgresql://... uv run python ../scripts/backfill_cian_zhk_url.py
|
|
|
|
Env vars:
|
|
DATABASE_URL required — psycopg v3 DSN
|
|
DELAY_SEC default 5.0 — seconds between requests (Cian rate limit)
|
|
BATCH_LIMIT default 500 — max houses to process per run (safety cap)
|
|
|
|
Idempotent — skips houses where cian_zhk_url IS NOT NULL. Safe to re-run.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
import os
|
|
import sys
|
|
import time
|
|
|
|
import psycopg
|
|
|
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "backend"))
|
|
|
|
from app.services.scrapers.cian_newbuilding import resolve_cian_zhk_url
|
|
|
|
logging.basicConfig(
|
|
format="%(asctime)s [%(levelname)s] %(message)s",
|
|
level=logging.INFO,
|
|
)
|
|
logger = logging.getLogger("backfill_cian_zhk_url")
|
|
|
|
DATABASE_URL = os.environ.get("DATABASE_URL", "")
|
|
DELAY_SEC = float(os.environ.get("DELAY_SEC", "5.0"))
|
|
BATCH_LIMIT = int(os.environ.get("BATCH_LIMIT", "500"))
|
|
|
|
|
|
async def main() -> None:
|
|
if not DATABASE_URL:
|
|
logger.error("DATABASE_URL env var not set")
|
|
sys.exit(1)
|
|
|
|
conn = await psycopg.AsyncConnection.connect(DATABASE_URL)
|
|
|
|
async with conn:
|
|
rows = await (
|
|
await conn.execute(
|
|
"""
|
|
SELECT id, cian_internal_house_id
|
|
FROM houses
|
|
WHERE cian_internal_house_id IS NOT NULL
|
|
AND cian_zhk_url IS NULL
|
|
ORDER BY id
|
|
LIMIT %(lim)s
|
|
""",
|
|
{"lim": BATCH_LIMIT},
|
|
)
|
|
).fetchall()
|
|
|
|
total = len(rows)
|
|
logger.info(
|
|
"Found %d houses with cian_internal_house_id and no cian_zhk_url (limit=%d)",
|
|
total,
|
|
BATCH_LIMIT,
|
|
)
|
|
if total == 0:
|
|
logger.info("Nothing to backfill.")
|
|
return
|
|
|
|
succeeded = 0
|
|
failed = 0
|
|
t0 = time.time()
|
|
|
|
conn = await psycopg.AsyncConnection.connect(DATABASE_URL)
|
|
async with conn:
|
|
for i, (house_id, cian_id) in enumerate(rows, start=1):
|
|
url = await resolve_cian_zhk_url(cian_id)
|
|
if url:
|
|
await conn.execute(
|
|
"""
|
|
UPDATE houses
|
|
SET cian_zhk_url = %(url)s
|
|
WHERE id = %(hid)s
|
|
AND cian_zhk_url IS NULL
|
|
""",
|
|
{"url": url, "hid": house_id},
|
|
)
|
|
await conn.commit()
|
|
succeeded += 1
|
|
logger.info("[%d/%d] house_id=%s id=%s → %s", i, total, house_id, cian_id, url)
|
|
else:
|
|
failed += 1
|
|
logger.warning(
|
|
"[%d/%d] house_id=%s id=%s — could not resolve URL",
|
|
i, total, house_id, cian_id,
|
|
)
|
|
|
|
if i < total:
|
|
await asyncio.sleep(DELAY_SEC)
|
|
|
|
elapsed = time.time() - t0
|
|
logger.info(
|
|
"Backfill complete: %d succeeded, %d failed, %.1fs total (%.1fs/req avg)",
|
|
succeeded,
|
|
failed,
|
|
elapsed,
|
|
elapsed / max(succeeded + failed, 1),
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main())
|