"""Phase C: backfill Avito IMV per house. Iterates `houses` with imv_status='pending' AND lat/lon NOT NULL. For each: pick median lot-params from linked listings -> call evaluate_via_imv() -> save house_imv_evaluations + house_placement_history + house_suggestions. Resumable: marks houses.imv_status = 'ok' / 'not_found' / 'transient_error' / 'error'. Usage: # Locally with SSH tunnel (LocalForward 35432 -> tradein-postgres): DATABASE_URL=postgresql+psycopg://tradein:@localhost:35432/tradein \\ python scripts/backfill-house-imv.py --batch 50 --delay 5 # Inside container: docker exec tradein-backend python /app/scripts/backfill-house-imv.py --batch 100 --delay 3 Args: --batch N How many houses to process this run (default 50) --delay SEC Sleep between houses (default 5 to avoid Avito captcha) --limit-total Stop after N successful evaluations (debug; default no limit) --only-status Process only houses with this imv_status (default 'pending'; also accepts 'transient_error') """ from __future__ import annotations import argparse import asyncio import json import logging import os import sys from pathlib import Path # Ensure repo backend in path sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "backend")) from sqlalchemy import create_engine, text from sqlalchemy.orm import Session from app.services.scrapers.avito_imv import ( IMVAddressNotFoundError, IMVAuthError, IMVEvaluation, IMVTransientError, evaluate_via_imv, ) logging.basicConfig( level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s", ) logger = logging.getLogger("phase-c") # Avito raw log — pipe everything from app.services.scrapers.avito_imv to a file # so we keep full request/response history for diagnosing contract changes. _AVITO_LOG_PATH = os.environ.get("AVITO_LOG_PATH", "avito-raw.log") _avito_fh = logging.FileHandler(_AVITO_LOG_PATH, encoding="utf-8") _avito_fh.setFormatter(logging.Formatter("%(asctime)s %(levelname)s [%(name)s] %(message)s")) _avito_fh.setLevel(logging.DEBUG) _avito_logger = logging.getLogger("app.services.scrapers.avito_imv") _avito_logger.addHandler(_avito_fh) _avito_logger.setLevel(logging.DEBUG) # Also pipe phase-c progress lines to the file logger.addHandler(_avito_fh) logger.info("avito raw log writing to %s", os.path.abspath(_AVITO_LOG_PATH)) # Avito IMV API accepts only these house_type values (verified 2026-05-24). # Our `listings.house_type` may contain pre-2026 values like 'monolith' / 'monolith_brick'. _HOUSE_TYPE_TO_IMV = { "panel": "panel", "brick": "brick", "monolith": "monolithic", # API renamed; raw value rejected with 400 "monolithic": "monolithic", "monolith_brick": "monolithic", # API doesn't accept hybrid → fall back to monolithic "block": "block", "wood": "wood", } _HOUSE_TYPE_DEFAULT = "panel" # most common in EKB; safe default for unknown def _map_house_type(raw: str | None) -> str: if not raw: return _HOUSE_TYPE_DEFAULT return _HOUSE_TYPE_TO_IMV.get(raw.lower().strip(), _HOUSE_TYPE_DEFAULT) # Bounding boxes for known regions in our DB. # When house.lat/lon falls inside one, we prepend the region name to address # before sending to Avito (resolves ambiguity — e.g. "Орджоникидзевский район" exists # in EKB AND in Khakasia republic). _REGION_BBOX = [ # (region_name, lat_min, lat_max, lon_min, lon_max) ("Свердловская область, Екатеринбург", 56.5, 57.1, 59.9, 61.0), ("Свердловская область", 56.0, 61.0, 58.0, 65.0), # rest of region ("Кировская область, Киров", 58.4, 58.8, 49.4, 49.9), ("Ставропольский край, Ставрополь", 44.9, 45.2, 41.7, 42.2), ("Тюменская область, Тюмень", 57.0, 57.3, 65.3, 65.8), ("Челябинская область, Челябинск", 55.0, 55.4, 61.2, 61.7), ] _REGION_DEFAULT = "Свердловская область, Екатеринбург" # bulk of our data is EKB def _detect_region_prefix(lat: float | None, lon: float | None) -> str: if lat is None or lon is None: return _REGION_DEFAULT for name, lat_min, lat_max, lon_min, lon_max in _REGION_BBOX: if lat_min <= lat <= lat_max and lon_min <= lon <= lon_max: return name return _REGION_DEFAULT def _enrich_address_for_imv(raw: str, lat: float | None, lon: float | None) -> str: """Prepend region prefix if address lacks one (ambiguity fix for Avito geocoder).""" addr = (raw or "").strip() # Already has region marker? skip addr_lower = addr.lower() if any( m in addr_lower for m in ("область,", " обл.,", " обл,", "край,", "респ.,", "республика,") ): return addr prefix = _detect_region_prefix(lat, lon) return f"{prefix}, {addr}" DATABASE_URL = os.environ.get("DATABASE_URL") if not DATABASE_URL: raise SystemExit( "Set DATABASE_URL env" " (e.g. postgresql+psycopg://tradein:...@localhost:35432/tradein)" ) def pick_lot_params(db: Session, house_id: int) -> dict: """Pick representative lot-params from linked listings — median values.""" row = db.execute( text(""" SELECT CAST(percentile_cont(0.5) WITHIN GROUP (ORDER BY rooms) AS integer) AS rooms, CAST(percentile_cont(0.5) WITHIN GROUP (ORDER BY area_m2) AS numeric(8,2)) AS area_m2, CAST(percentile_cont(0.5) WITHIN GROUP (ORDER BY floor) AS integer) AS floor, CAST(percentile_cont(0.5) WITHIN GROUP (ORDER BY total_floors) AS integer) AS total_floors, mode() WITHIN GROUP (ORDER BY house_type) AS house_type FROM listings WHERE house_id_fk = :hid AND rooms IS NOT NULL AND area_m2 IS NOT NULL """), {"hid": house_id}, ).mappings().first() if row is None or row["rooms"] is None: return {} # no listings with required fields, can't pick params # Fall back to house's own type/floors if listings don't have them house = db.execute( text("SELECT house_type, total_floors FROM houses WHERE id = :hid"), {"hid": house_id}, ).mappings().first() # Avito IMV: rooms=0 (студия) rejected with HTTP 400. Clamp to 1. rooms_raw = int(row["rooms"]) return { "rooms": max(rooms_raw, 1), "area_m2": float(row["area_m2"]), "floor": int(row["floor"] or 1), "floor_at_home": int(row["total_floors"] or (house and house["total_floors"]) or 9), "house_type": _map_house_type(row["house_type"] or (house and house["house_type"])), "renovation_type": "cosmetic", "has_balcony": True, "has_loggia": False, } def save_imv_result(db: Session, house_id: int, params: dict, result: IMVEvaluation) -> None: """Save 3 things: house_imv_evaluations + history items + suggestions. Uses already-parsed dataclass fields from IMVEvaluation (placement_history, suggestions) instead of re-parsing raw_response to avoid duplication and ensure consistency with _parse_placement_item / _parse_suggestion logic. """ # 1. House-level IMV evaluation (UPSERT on house_id) db.execute( text(""" INSERT INTO house_imv_evaluations ( house_id, cache_key, recommended_price, lower_price, higher_price, market_count, raw_response, fetched_at, rooms, area_m2, floor, floor_at_home, house_type, renovation_type, has_balcony, has_loggia ) VALUES ( :hid, :ck, :rec, :lo, :hi, :mc, CAST(:raw AS jsonb), NOW(), :rooms, :area, :floor, :fah, :htype, :rtype, :balcony, :loggia ) ON CONFLICT (house_id) DO UPDATE SET cache_key = EXCLUDED.cache_key, recommended_price = EXCLUDED.recommended_price, lower_price = EXCLUDED.lower_price, higher_price = EXCLUDED.higher_price, market_count = EXCLUDED.market_count, raw_response = EXCLUDED.raw_response, fetched_at = NOW(), rooms = EXCLUDED.rooms, area_m2 = EXCLUDED.area_m2, floor = EXCLUDED.floor, floor_at_home = EXCLUDED.floor_at_home, house_type = EXCLUDED.house_type, renovation_type = EXCLUDED.renovation_type, has_balcony = EXCLUDED.has_balcony, has_loggia = EXCLUDED.has_loggia """), { "hid": house_id, "ck": result.cache_key, "rec": result.recommended_price, "lo": result.lower_price, "hi": result.higher_price, "mc": result.market_count, "raw": ( json.dumps(result.raw_response, ensure_ascii=False) if result.raw_response else None ), "rooms": params["rooms"], "area": params["area_m2"], "floor": params["floor"], "fah": params["floor_at_home"], "htype": params["house_type"], "rtype": params["renovation_type"], "balcony": params["has_balcony"], "loggia": params["has_loggia"], }, ) # 2. Placement history items (already parsed into IMVPlacementHistoryItem dataclasses) for item in result.placement_history: db.execute( text(""" INSERT INTO house_placement_history ( source, house_id, ext_item_id, title, rooms, area_m2, floor, total_floors, start_price, start_price_date, last_price, last_price_date, removed_date, exposure_days, raw_payload, scraped_at ) VALUES ( 'avito_imv', :hid, :ext, :title, :rooms, :area, :floor, :total_floors, :sp, :spd, :lp, :lpd, :rmd, :exp, CAST(:raw AS jsonb), NOW() ) ON CONFLICT (source, ext_item_id) DO UPDATE SET house_id = EXCLUDED.house_id, title = EXCLUDED.title, rooms = EXCLUDED.rooms, area_m2 = EXCLUDED.area_m2, floor = EXCLUDED.floor, total_floors = EXCLUDED.total_floors, start_price = EXCLUDED.start_price, start_price_date = EXCLUDED.start_price_date, last_price = EXCLUDED.last_price, last_price_date = EXCLUDED.last_price_date, removed_date = EXCLUDED.removed_date, exposure_days = EXCLUDED.exposure_days, raw_payload = EXCLUDED.raw_payload, scraped_at = NOW() """), { "hid": house_id, "ext": item.ext_item_id, "title": item.title, "rooms": item.rooms, "area": item.area_m2, "floor": item.floor, "total_floors": item.total_floors, "sp": item.start_price, "spd": item.start_price_date, "lp": item.last_price, "lpd": item.last_price_date, "rmd": item.removed_date, "exp": item.exposure_days, "raw": ( json.dumps(item.raw_payload, ensure_ascii=False) if item.raw_payload else None ), }, ) # 3. Suggestions (already parsed into IMVSuggestion dataclasses) for sug in result.suggestions: db.execute( text(""" INSERT INTO house_suggestions ( house_id, ext_item_id, title, address, price_rub, exposure_days, publish_date, item_link, metro_name, metro_distance, has_good_price_badge, raw_payload, fetched_at ) VALUES ( :hid, :ext, :title, :addr, :price, :exp, :pdate, :link, :mname, :mdist, :gpb, CAST(:raw AS jsonb), NOW() ) ON CONFLICT (house_id, ext_item_id) DO UPDATE SET title = EXCLUDED.title, price_rub = EXCLUDED.price_rub, exposure_days = EXCLUDED.exposure_days, publish_date = EXCLUDED.publish_date, item_link = EXCLUDED.item_link, metro_name = EXCLUDED.metro_name, metro_distance = EXCLUDED.metro_distance, has_good_price_badge = EXCLUDED.has_good_price_badge, raw_payload = EXCLUDED.raw_payload, fetched_at = NOW() """), { "hid": house_id, "ext": sug.ext_item_id, "title": sug.title, "addr": sug.address, "price": sug.price_rub, "exp": sug.exposure_days, "pdate": sug.publish_date, "link": sug.item_url, "mname": sug.metro_name, "mdist": sug.metro_distance, "gpb": sug.has_good_price_badge, "raw": ( json.dumps(sug.raw_payload, ensure_ascii=False) if sug.raw_payload else None ), }, ) # 4. Mark house success db.execute( text(""" UPDATE houses SET imv_status = 'ok', last_imv_attempt_at = NOW(), imv_error_reason = NULL WHERE id = :hid """), {"hid": house_id}, ) def mark_status(db: Session, house_id: int, status: str, reason: str | None = None) -> None: db.execute( text(""" UPDATE houses SET imv_status = :s, last_imv_attempt_at = NOW(), imv_error_reason = :r WHERE id = :hid """), {"hid": house_id, "s": status, "r": reason}, ) async def process_one(db: Session, house: dict) -> str: """Process a single house: geocode + IMV call + save. Returns final status string: 'ok' / 'no_params' / 'no_address' / 'not_found' / 'auth_error' / 'transient' / 'error' """ params = pick_lot_params(db, house["id"]) if not params: mark_status(db, house["id"], "no_params", "no listings with rooms+area") return "no_params" address = house["address"] or house["full_address"] if not address: mark_status(db, house["id"], "no_address", "house.address is NULL") return "no_address" # Enrich with region prefix to prevent Avito geocoder mis-resolving ambiguous # street names to wrong regions (e.g. "улица Победы" exists in EKB and Khakasia). enriched = _enrich_address_for_imv(address, house.get("lat"), house.get("lon")) if enriched != address: logger.debug("enriched address house=%d %r -> %r", house["id"], address, enriched) try: result = await evaluate_via_imv(address=enriched, **params) except IMVAddressNotFoundError as e: mark_status(db, house["id"], "not_found", str(e)[:200]) return "not_found" except IMVAuthError as e: mark_status(db, house["id"], "transient_error", f"auth: {e!s}"[:200]) logger.warning("AUTH ERROR house=%d — pausing 60s", house["id"]) await asyncio.sleep(60) # long back-off on auth block return "auth_error" except IMVTransientError as e: mark_status(db, house["id"], "transient_error", str(e)[:200]) return "transient" except Exception as e: mark_status(db, house["id"], "error", repr(e)[:200]) logger.error("Unexpected error for house=%d: %r", house["id"], e) return "error" save_imv_result(db, house["id"], params, result) return "ok" async def main() -> None: parser = argparse.ArgumentParser(description="Phase C: backfill Avito IMV per house") parser.add_argument("--batch", type=int, default=50, help="Houses to process this run") parser.add_argument("--delay", type=float, default=5.0, help="Sleep between houses (s)") parser.add_argument( "--limit-total", type=int, default=0, help="Stop after N successful evaluations (0 = no limit)" ) parser.add_argument( "--only-status", default="pending", help="Process only houses with this imv_status (default: pending)" ) args = parser.parse_args() engine = create_engine(DATABASE_URL, future=True) with Session(engine) as db: rows = db.execute( text(""" SELECT id, address, full_address, lat, lon FROM houses WHERE imv_status = :status AND lat IS NOT NULL AND lon IS NOT NULL AND address IS NOT NULL ORDER BY last_imv_attempt_at NULLS FIRST, id LIMIT :batch """), {"status": args.only_status, "batch": args.batch}, ).mappings().all() if not rows: logger.info("nothing to process — all done or no houses match filter") return logger.info( "processing %d houses (status=%r delay=%.1fs)", len(rows), args.only_status, args.delay ) counters: dict[str, int] = { "ok": 0, "not_found": 0, "no_params": 0, "no_address": 0, "transient": 0, "auth_error": 0, "error": 0, } for i, house in enumerate(rows, 1): status = await process_one(db, dict(house)) counters[status] = counters.get(status, 0) + 1 db.commit() logger.info( "[%d/%d] house=%d status=%s addr=%r", i, len(rows), house["id"], status, (house["address"] or "")[:50], ) if args.limit_total and counters["ok"] >= args.limit_total: logger.info("hit --limit-total=%d; stopping", args.limit_total) break if i < len(rows): await asyncio.sleep(args.delay) logger.info("done: %s", counters) if __name__ == "__main__": asyncio.run(main())