gendesign/tradein-mvp/backend/app/services/yandex_address_backfill.py

"""T10: Yandex listing address backfill from detail-page <title>.

Yandex SERP cards contain only street-level addresses (e.g. «улица Горького»).
The offer detail page <title> has the full address including house number:
«Екатеринбург, улица Горького, 36 — id 12345».

This service finds active yandex listings without a house number in `address`,
fetches each detail page via curl_cffi(chrome120), extracts the full address
from the <title> regex, and UPDATE listings.address.

Deliberately NOT wired into scheduler — manual trigger only.
Triggered via POST /api/v1/admin/scrape/yandex-address-backfill.
"""

from __future__ import annotations

import asyncio
import logging
import re
import time
from dataclasses import dataclass, field

from curl_cffi.requests import AsyncSession
from sqlalchemy import text
from sqlalchemy.orm import Session

logger = logging.getLogger(__name__)

# Extract '<City>, <full address>' from <title> — matches any city, not just Ekaterinburg.
# Handles EKB metro-area satellite towns: Верхняя Пышма, Берёзовский, Среднеуральск, Арамиль, etc.
# Examples:
#   «Продажа квартиры — Екатеринбург, улица Горького, 36 — id 7654321 на Яндекс.Недвижимости»
#   «Купить квартиру … — ЖК «Дуэт», Верхняя Пышма, улица Мальцева, 14 — id 75332…»
#   «Купить квартиру … — ЖК «Уют-Сити», Берёзовский, Александровский проспект, 3 — id 3988…»
# Group 1 = «<City>, <street/house>» (stripped of trailing punctuation/spaces)
# The optional ЖК-prefix (?:[^—,]*?,\s*) consumes «ЖК «Дуэт», » before the city name.
# City capture starts with [А-ЯЁ] to ensure the prefix never swallows the city itself.
_RE_TITLE_ADDRESS = re.compile(
    r"<title>[^<]*?—\s*"
    r"(?:(?![А-ЯЁ][а-яё])[^—,]*?,\s*)?"  # optional ЖК-prefix — fires only when NOT a city name
    r"([А-ЯЁ][^—<]+?,\s*[^<—]+?)"  # <City>, <street/house> — any city (uppercase Cyrillic start)
    r"\s*—\s*id\s*\d+[^<]*</title>",  # allow trailing text after id NNN (e.g. 'на Яндекс...')
)

# Listings without a house number: address contains no digit after comma-space
# (e.g. «улица Горького» but not «улица Горького, 36»).
# This regex matches a number right after «, » (house number pattern).
_RE_HAS_HOUSE_NUMBER = re.compile(r",\s*\d+")


@dataclass
class YandexAddressBackfillResult:
    checked: int = 0
    saved: int = 0
    skipped: int = 0
    errors: int = 0
    duration_sec: float = field(default=0.0)


def _extract_address_from_title(html: str) -> str | None:
    """Extract full address from Yandex offer detail page <title>.

    City-agnostic: works for EKB and satellite towns (Верхняя Пышма, Берёзовский, etc.).
    Returns «<City>, <street>, <house>» or None if the title format is not recognised.
    """
    html_clean = html.replace("\xa0", " ")
    m = _RE_TITLE_ADDRESS.search(html_clean)
    if not m:
        return None
    addr = m.group(1).strip().rstrip(",").strip()
    return addr if addr else None


def _eligible_listings(db: Session, limit: int) -> list[dict]:
    """Active EKB yandex listings without house number in address."""
    rows = (
        db.execute(
            text("""
                SELECT id, source_id, address, source_url
                FROM listings
                WHERE source = 'yandex'
                  AND region_code = 66
                  AND is_active = TRUE
                  AND address IS NOT NULL
                  AND NOT (address ~ ',\\s*\\d+')
                  AND source_url IS NOT NULL
                ORDER BY scraped_at DESC
                LIMIT :lim
            """),
            {"lim": limit},
        )
        .mappings()
        .all()
    )
    return [dict(r) for r in rows]


async def backfill_yandex_addresses(
    db: Session,
    *,
    limit: int = 200,
    request_delay_sec: float = 3.0,
) -> YandexAddressBackfillResult:
    """Fetch Yandex detail pages and update address with full street+house number.

    Args:
        db: SQLAlchemy session.
        limit: max listings to process.
        request_delay_sec: sleep between requests (anti-bot; default 3.0s).

    Returns:
        YandexAddressBackfillResult with checked/saved/skipped/errors counters.
    """
    from app.core.config import settings

    result = YandexAddressBackfillResult()
    t0 = time.time()

    items = _eligible_listings(db, limit)
    result.checked = len(items)

    if not items:
        logger.info("yandex_address_backfill: no eligible listings")
        result.duration_sec = time.time() - t0
        return result

    logger.info(
        "yandex_address_backfill: %d listings, delay=%.1fs",
        result.checked,
        request_delay_sec,
    )

    _proxy_url = settings.scraper_proxy_url
    _proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None

    async with AsyncSession(
        impersonate="chrome120",
        timeout=30.0,
        proxies=_proxies,
        headers={
            "Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",
        },
    ) as session:
        for i, item in enumerate(items):
            lid: int = item["id"]
            old_addr: str = item["address"]
            url: str = item["source_url"]

            try:
                resp = await session.get(url, allow_redirects=True)
            except Exception as exc:
                logger.warning("yandex_address_backfill: fetch failed listing_id=%s: %s", lid, exc)
                result.errors += 1
                await asyncio.sleep(request_delay_sec)
                continue

            if resp.status_code != 200:
                logger.warning(
                    "yandex_address_backfill: HTTP %d listing_id=%s", resp.status_code, lid
                )
                result.errors += 1
                await asyncio.sleep(request_delay_sec)
                continue

            new_addr = _extract_address_from_title(resp.text)

            if new_addr is None:
                logger.debug("yandex_address_backfill: no address extracted listing_id=%s", lid)
                result.skipped += 1
            elif new_addr == old_addr:
                logger.debug(
                    "yandex_address_backfill: unchanged listing_id=%s addr=%r", lid, old_addr
                )
                result.skipped += 1
            elif not _RE_HAS_HOUSE_NUMBER.search(new_addr):
                # Extracted address still has no house number — skip to avoid overwrite
                # with another street-only value.
                logger.debug(
                    "yandex_address_backfill: still no house number listing_id=%s addr=%r",
                    lid,
                    new_addr,
                )
                result.skipped += 1
            else:
                logger.info(
                    "yandex_address_backfill: updating listing_id=%s %r -> %r",
                    lid,
                    old_addr,
                    new_addr,
                )
                try:
                    db.execute(
                        text("""
                            UPDATE listings
                            SET address = :addr,
                                geocode_tried_at = NULL
                            WHERE id = CAST(:id AS bigint)
                        """),
                        {"addr": new_addr, "id": lid},
                    )
                    db.commit()
                    result.saved += 1
                except Exception as exc:
                    logger.warning(
                        "yandex_address_backfill: DB update failed listing_id=%s: %s",
                        lid,
                        exc,
                    )
                    result.errors += 1
                    try:
                        db.rollback()
                    except Exception:
                        pass

            if i < len(items) - 1:
                await asyncio.sleep(request_delay_sec)

    result.duration_sec = time.time() - t0
    logger.info(
        "yandex_address_backfill done: checked=%d saved=%d skipped=%d errors=%d %.1fs",
        result.checked,
        result.saved,
        result.skipped,
        result.errors,
        result.duration_sec,
    )
    return result