288 lines
9.3 KiB
Python
288 lines
9.3 KiB
Python
"""Продуктовые адаптеры над `app.services.*`, удовлетворяющие scraper_kit.contracts.
|
||
|
||
Тонкие врапперы, инжектируемые в скрапперы (после переноса в scraper_kit, шаги C-G
|
||
#2131) вместо прямых импортов `app.*`. Каждый класс структурно удовлетворяет
|
||
соответствующему Protocol'у из `scraper_kit.contracts`:
|
||
|
||
RealMatcherAdapter → HouseMatcher
|
||
RealScraperConfig → ScraperConfig
|
||
RealSessionFactory → SessionFactory
|
||
RealEnrichmentJobs → EnrichmentJobs
|
||
|
||
Пока НИКТО их не использует — скрапперы всё ещё на прямых импортах. Это подготовка:
|
||
переключение вызовов на инжекцию — отдельный шаг (C). Держим сигнатуры в точном
|
||
соответствии с исходными функциями, чтобы шаг C состыковался без правок контрактов.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from collections.abc import Callable
|
||
from typing import TYPE_CHECKING, Any
|
||
|
||
from app.core.config import settings as _settings
|
||
from app.core.db import SessionLocal as _SessionLocal
|
||
from app.services.house_dedup_merge import run_house_dedup_merge as _run_house_dedup_merge
|
||
from app.services.house_imv_backfill import backfill_house_imv as _backfill_house_imv
|
||
from app.services.house_imv_backfill import (
|
||
process_houses_imv_batch as _process_houses_imv_batch,
|
||
)
|
||
from app.services.matching import match_or_create_house as _match_or_create_house
|
||
from app.services.matching import upsert_listing_source as _upsert_listing_source
|
||
from app.services.yandex_address_backfill import (
|
||
_RE_HAS_HOUSE_NUMBER,
|
||
_extract_address_from_title,
|
||
)
|
||
from app.services.yandex_address_backfill import (
|
||
backfill_yandex_addresses as _backfill_yandex_addresses,
|
||
)
|
||
from app.services.yandex_price_history import (
|
||
record_yandex_price_history as _record_yandex_price_history,
|
||
)
|
||
|
||
if TYPE_CHECKING:
|
||
from sqlalchemy.orm import Session
|
||
|
||
from app.services.house_imv_backfill import HouseIMVBackfillResult
|
||
from app.services.yandex_address_backfill import YandexAddressBackfillResult
|
||
|
||
|
||
class RealMatcherAdapter:
|
||
"""HouseMatcher-адаптер над `app.services.matching`."""
|
||
|
||
def match_or_create_house(
|
||
self,
|
||
db: Session,
|
||
ext_source: str,
|
||
ext_id: str,
|
||
address: str | None = None,
|
||
lat: float | None = None,
|
||
lon: float | None = None,
|
||
*,
|
||
year_built: int | None = None,
|
||
building_cadastral_number: str | None = None,
|
||
cadastral_number: str | None = None,
|
||
source_url: str | None = None,
|
||
) -> tuple[int | None, float, str]:
|
||
# house_id is None when the matcher refuses a numberless address without a
|
||
# cadastral number (method 'no_house_number', P1). Callers must tolerate None.
|
||
return _match_or_create_house(
|
||
db,
|
||
ext_source,
|
||
ext_id,
|
||
address,
|
||
lat,
|
||
lon,
|
||
year_built=year_built,
|
||
building_cadastral_number=building_cadastral_number,
|
||
cadastral_number=cadastral_number,
|
||
source_url=source_url,
|
||
)
|
||
|
||
def upsert_listing_source(
|
||
self,
|
||
db: Session,
|
||
*,
|
||
listing_id: int,
|
||
ext_source: str,
|
||
ext_id: str,
|
||
method: str,
|
||
confidence: float,
|
||
price_rub: float | None = None,
|
||
area_m2: float | None = None,
|
||
floor: int | None = None,
|
||
rooms_count: int | None = None,
|
||
source_url: str | None = None,
|
||
source_data: dict[str, Any] | None = None,
|
||
) -> None:
|
||
_upsert_listing_source(
|
||
db,
|
||
listing_id=listing_id,
|
||
ext_source=ext_source,
|
||
ext_id=ext_id,
|
||
method=method,
|
||
confidence=confidence,
|
||
price_rub=price_rub,
|
||
area_m2=area_m2,
|
||
floor=floor,
|
||
rooms_count=rooms_count,
|
||
source_url=source_url,
|
||
source_data=source_data,
|
||
)
|
||
|
||
|
||
class RealScraperConfig:
|
||
"""ScraperConfig-адаптер: read-only проекция singleton'а `settings`.
|
||
|
||
Свойства проксируют в `app.core.config.settings`, включая computed-properties
|
||
(`scraper_proxy_url`, `cian_proxy_url`), чтобы скрапперы читали ровно те же
|
||
значения, что и сейчас через прямой `settings.<name>`.
|
||
"""
|
||
|
||
@property
|
||
def scraper_fetch_mode(self) -> str:
|
||
return _settings.scraper_fetch_mode
|
||
|
||
@property
|
||
def browser_http_endpoint(self) -> str:
|
||
return _settings.browser_http_endpoint
|
||
|
||
@property
|
||
def scraper_proxy_url(self) -> str | None:
|
||
return _settings.scraper_proxy_url
|
||
|
||
@property
|
||
def avito_proxy_rotate_url(self) -> str | None:
|
||
return _settings.avito_proxy_rotate_url
|
||
|
||
@property
|
||
def avito_proxy_max_rotations(self) -> int:
|
||
return _settings.avito_proxy_max_rotations
|
||
|
||
@property
|
||
def avito_serp_ekb_only(self) -> bool:
|
||
return _settings.avito_serp_ekb_only
|
||
|
||
@property
|
||
def yandex_proxy_rotate_url(self) -> str | None:
|
||
return _settings.yandex_proxy_rotate_url
|
||
|
||
@property
|
||
def cian_proxy_url(self) -> str | None:
|
||
return _settings.cian_proxy_url
|
||
|
||
@property
|
||
def cian_valuation_min_rub(self) -> float:
|
||
return _settings.cian_valuation_min_rub
|
||
|
||
@property
|
||
def cian_valuation_max_rub(self) -> float:
|
||
return _settings.cian_valuation_max_rub
|
||
|
||
@property
|
||
def cookie_encryption_key(self) -> str:
|
||
return _settings.cookie_encryption_key
|
||
|
||
@property
|
||
def glitchtip_dsn(self) -> str | None:
|
||
return _settings.glitchtip_dsn
|
||
|
||
# ── Оркестрация sweep-pipeline (#2135) ───────────────────────────────
|
||
@property
|
||
def avito_serp_ok_not_banned(self) -> bool:
|
||
return _settings.avito_serp_ok_not_banned
|
||
|
||
@property
|
||
def avito_proxy_rotate_settle_s(self) -> float:
|
||
return _settings.avito_proxy_rotate_settle_s
|
||
|
||
@property
|
||
def proxy_rotate_attempts(self) -> int:
|
||
return _settings.proxy_rotate_attempts
|
||
|
||
@property
|
||
def proxy_rotate_attempt_timeout_s(self) -> float:
|
||
return _settings.proxy_rotate_attempt_timeout_s
|
||
|
||
@property
|
||
def cian_proxy_rotate_url(self) -> str | None:
|
||
return _settings.cian_proxy_rotate_url
|
||
|
||
@property
|
||
def cian_proxy_max_rotations(self) -> int:
|
||
return _settings.cian_proxy_max_rotations
|
||
|
||
@property
|
||
def yandex_proxy_max_rotations(self) -> int:
|
||
return _settings.yandex_proxy_max_rotations
|
||
|
||
@property
|
||
def scraper_skip_seen_today(self) -> bool:
|
||
return _settings.scraper_skip_seen_today
|
||
|
||
@property
|
||
def cian_full_load_per_fetch_timeout_s(self) -> float:
|
||
return _settings.cian_full_load_per_fetch_timeout_s
|
||
|
||
|
||
class RealSessionFactory:
|
||
"""SessionFactory-адаптер над `app.core.db.SessionLocal`."""
|
||
|
||
def __call__(self) -> Session:
|
||
return _SessionLocal()
|
||
|
||
|
||
class RealEnrichmentJobs:
|
||
"""EnrichmentJobs-адаптер над enrichment-сервисами `app.services.*`."""
|
||
|
||
async def house_imv_backfill(
|
||
self,
|
||
db: Session,
|
||
*,
|
||
batch_size: int = 50,
|
||
request_delay_sec: float = 5.0,
|
||
only_status: str = "pending",
|
||
house_id: int | None = None,
|
||
heartbeat: Callable[[], None] | None = None,
|
||
) -> HouseIMVBackfillResult:
|
||
return await _backfill_house_imv(
|
||
db,
|
||
batch_size=batch_size,
|
||
request_delay_sec=request_delay_sec,
|
||
only_status=only_status,
|
||
house_id=house_id,
|
||
heartbeat=heartbeat,
|
||
)
|
||
|
||
def house_dedup_merge(
|
||
self,
|
||
db: Session,
|
||
*,
|
||
run_id: int,
|
||
params: dict[str, Any],
|
||
) -> dict[str, int]:
|
||
return _run_house_dedup_merge(db, run_id=run_id, params=params)
|
||
|
||
async def yandex_address_backfill(
|
||
self,
|
||
db: Session,
|
||
*,
|
||
limit: int = 200,
|
||
request_delay_sec: float = 3.0,
|
||
) -> YandexAddressBackfillResult:
|
||
return await _backfill_yandex_addresses(
|
||
db,
|
||
limit=limit,
|
||
request_delay_sec=request_delay_sec,
|
||
)
|
||
|
||
async def process_houses_imv_batch(
|
||
self,
|
||
db: Session,
|
||
house_ids: set[int],
|
||
*,
|
||
request_delay_sec: float = 5.0,
|
||
heartbeat: Callable[[], None] | None = None,
|
||
) -> HouseIMVBackfillResult:
|
||
return await _process_houses_imv_batch(
|
||
db,
|
||
house_ids,
|
||
request_delay_sec=request_delay_sec,
|
||
heartbeat=heartbeat,
|
||
)
|
||
|
||
def record_yandex_price_history(self, db: Session, lots: list[Any]) -> int:
|
||
return _record_yandex_price_history(db, lots)
|
||
|
||
def extract_address_from_title(self, html: str) -> str | None:
|
||
return _extract_address_from_title(html)
|
||
|
||
def address_has_house_number(self, address: str) -> bool:
|
||
return bool(_RE_HAS_HOUSE_NUMBER.search(address))
|
||
|
||
|
||
__all__ = [
|
||
"RealEnrichmentJobs",
|
||
"RealMatcherAdapter",
|
||
"RealScraperConfig",
|
||
"RealSessionFactory",
|
||
]
|