lekss361
f93ef5374d
All checks were successful
Deploy Trade-In / changes (push) Successful in 10s
Deploy Trade-In / build-frontend (push) Has been skipped
Deploy Trade-In / build-browser (push) Has been skipped
Deploy Trade-In / test (push) Successful in 47s
Deploy Trade-In / build-backend (push) Successful in 1m30s
Deploy Trade-In / deploy (push) Successful in 1m32s
320 lines
14 KiB
Python
320 lines
14 KiB
Python
"""Scheduled backfill: detail-enrichment (Layer B) for domklik listings (issue #2000).
|
|
|
|
Nightly window 15:00-18:00 UTC (migration 175, source=domclick_detail_backfill).
|
|
Offset from avito_detail_backfill (09-12 UTC) and yandex_detail_backfill (12-15 UTC)
|
|
to avoid parallel egress across scraper sources sharing the tradein-browser pool.
|
|
|
|
Problem: DomClick SERP layer (Layer A, scraper_kit.providers.domclick.serp) discovers
|
|
listings via the BFF JSON API, but detail-enrichment (Layer B — repair_state,
|
|
living/kitchen area, year_built, owners_count, price history) requires a per-card
|
|
browser-fetch that is QRATOR-guarded (issue #2000). Two prior PRs solved the block at
|
|
the single-fetch level:
|
|
- PR #2430 -- organic same-site SERP-origin navigation before the card fetch.
|
|
- PR #2433 -- cookie-injection MVP (authenticated test-account session bypasses the
|
|
QRATOR reputation-block even on an already-suspicious proxy-IP).
|
|
Both are wired together in the debug endpoint `POST /scrape/domclick/debug/detail-fetch`
|
|
(app/api/v1/admin.py). This module ports that same session -> cookies -> fetch_detail
|
|
wiring into the production scheduled orchestrator (previously only reachable manually).
|
|
|
|
Solution: single snapshot SELECT at start (guarantees termination) + one BrowserFetcher
|
|
per run (async context manager, source="domclick" -- dedicated residential proxy pool,
|
|
see 173_scrape_proxies_add_domclick_affinity.sql) + cookies loaded ONCE via
|
|
domclick_session.load_session(db) and threaded into every fetch_detail() call.
|
|
|
|
NAMING TRAP (verified live against prod DB 2026-07-04, do NOT "fix" this anywhere):
|
|
listings.source value for DomClick rows is 'domklik' (with a "k") -- that is the
|
|
DATA/business identifier used across listings.source, DomClickScraper.name/.source,
|
|
schemas, PDF exporter labels. 'domclick' (with a "c") is the newer INFRA/transport
|
|
identifier -- BrowserFetcher(source="domclick"), scrape_proxies.provider_affinity,
|
|
the SSRF host-guard, cookie-session tables. Both strings are correct in their own
|
|
context; this split is intentional and pre-existing, not a bug to unify here.
|
|
|
|
No curl fallback: unlike Avito's dual-mode (curl_cffi backconnect OR browser),
|
|
DomClick only has a BrowserFetcher path -- scraper_kit.providers.domclick.detail.
|
|
fetch_detail() raises DomClickBlockedError if the browser fetch itself fails. Exactly
|
|
one BrowserFetcher is constructed per run.
|
|
|
|
Exception triad differs from Avito:
|
|
- DomClickBlockedError (QRATOR challenge page OR any browser-fetch failure) --
|
|
increments consecutive_blocks, abort via mark_done (NOT mark_failed) once
|
|
max_consecutive_blocks is hit -- a block-abort is an expected operational
|
|
outcome (QRATOR reputation burn), not a task failure. Mirrors Avito's
|
|
AvitoBlockedError handling. No IP-rotation/cooldown recovery step exists here
|
|
(DomClick uses one dedicated residential proxy, not a rotating pool) -- an
|
|
aborted run simply retries the remaining backlog next window.
|
|
- DomClickParseError (__SSR_STATE__ missing/malformed -- schema drift, NOT a
|
|
block) -- counted as failed++, logged, does NOT touch consecutive_blocks and
|
|
does NOT abort the run (neutral to the block-breaker, mirrors how Avito's
|
|
AvitoListingGoneError is neutral to its breaker for a different reason).
|
|
- There is no "listing gone / 404" exception type for DomClick yet -- not
|
|
invented here.
|
|
|
|
Cookie injection is mandatory wiring, not optional: cookies are loaded ONCE per run.
|
|
If None (no valid session uploaded / expired) -- the run still proceeds (cookie-
|
|
injection is a QRATOR-defeat mechanism, not a hard requirement; organic SERP-origin
|
|
navigation from PR #2430 still applies) but a warning is logged once at run start so
|
|
operators notice the test-account session needs refreshing via
|
|
`POST /scrape/domclick/upload-cookies` (no auto-login -- documented MVP limitation,
|
|
see app/services/domclick_session.py module docstring).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
import random
|
|
import time
|
|
from dataclasses import dataclass, field
|
|
|
|
from scraper_kit.browser_fetcher import BrowserFetcher
|
|
from scraper_kit.domclick_exceptions import DomClickBlockedError, DomClickParseError
|
|
from scraper_kit.providers.domclick.detail import fetch_detail, save_detail_enrichment
|
|
from sqlalchemy import text
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app.core.config import settings
|
|
from app.core.shutdown import shutdown_requested
|
|
from app.services import domclick_session as domclick_session_svc
|
|
from app.services import scrape_runs as runs_mod
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
__all__ = [
|
|
"DomClickDetailBackfillResult",
|
|
"run_domclick_detail_backfill",
|
|
]
|
|
|
|
|
|
@dataclass
|
|
class DomClickDetailBackfillResult:
|
|
"""Counters for one backfill run."""
|
|
|
|
attempted: int = 0
|
|
enriched: int = 0
|
|
blocked: int = 0
|
|
failed: int = 0
|
|
duration_sec: float = field(default=0.0)
|
|
|
|
def to_dict(self) -> dict[str, int]:
|
|
return {
|
|
"attempted": self.attempted,
|
|
"enriched": self.enriched,
|
|
"blocked": self.blocked,
|
|
"failed": self.failed,
|
|
"duration_sec": int(self.duration_sec),
|
|
}
|
|
|
|
|
|
async def run_domclick_detail_backfill(
|
|
db: Session,
|
|
*,
|
|
run_id: int,
|
|
params: dict,
|
|
) -> DomClickDetailBackfillResult:
|
|
"""Backfill detail_enriched_at for domklik listings via cookie-injected browser fetch.
|
|
|
|
Params (from default_params jsonb in scrape_schedules):
|
|
batch_size: int -- snapshot size (SELECT LIMIT), default 200.
|
|
budget_sec: float -- wall-clock budget per run, default 3600s.
|
|
request_delay_sec: float -- delay between listings, default 12.0s.
|
|
max_consecutive_blocks: int -- abort threshold, default 3.
|
|
|
|
Lifecycle: update_heartbeat -> snapshot -> loop with budget guard ->
|
|
mark_done (incl. partial/block-abort) / mark_failed (exception only).
|
|
"""
|
|
batch_size = int(params.get("batch_size", 200))
|
|
budget_sec = float(params.get("budget_sec", 3600))
|
|
request_delay_sec = float(params.get("request_delay_sec", 12.0))
|
|
max_consecutive_blocks = int(params.get("max_consecutive_blocks", 3))
|
|
|
|
counters = DomClickDetailBackfillResult()
|
|
current_counters: dict[str, int] = counters.to_dict()
|
|
start = time.monotonic()
|
|
|
|
logger.info("domclick_detail_backfill: run_id=%d starting", run_id)
|
|
|
|
try:
|
|
# Cookie injection (#2000 PR #2433) -- loaded ONCE per run, threaded into every
|
|
# fetch_detail() call below. None is a valid (degraded) state, not an error.
|
|
cookies = domclick_session_svc.load_session(db)
|
|
if cookies is None:
|
|
logger.warning(
|
|
"domclick_detail_backfill: run_id=%d -- no valid DomClick session cookies "
|
|
"in DB; proceeding WITHOUT cookie-injection (QRATOR-defeat degraded to "
|
|
"organic SERP-origin navigation only, PR #2430). Refresh test-account "
|
|
"session via POST /scrape/domclick/upload-cookies.",
|
|
run_id,
|
|
)
|
|
|
|
runs_mod.update_heartbeat(db, run_id, current_counters)
|
|
|
|
# SNAPSHOT: single SELECT at start -- NOT re-selected in loop.
|
|
# NAMING TRAP (see module docstring): source='domklik' here (data identifier) is
|
|
# NOT the same string as BrowserFetcher(source="domclick") below (infra
|
|
# identifier) -- both are correct, do not "fix" one to match the other.
|
|
snapshot = (
|
|
db.execute(
|
|
text(
|
|
"""
|
|
SELECT id, source_url
|
|
FROM listings
|
|
WHERE source = 'domklik'
|
|
AND detail_enriched_at IS NULL
|
|
AND source_url IS NOT NULL
|
|
AND is_active = TRUE
|
|
ORDER BY (lat IS NULL) DESC, scraped_at DESC NULLS LAST
|
|
LIMIT CAST(:batch_size AS int)
|
|
"""
|
|
),
|
|
{"batch_size": batch_size},
|
|
)
|
|
.mappings()
|
|
.all()
|
|
)
|
|
|
|
if not snapshot:
|
|
logger.info(
|
|
"domclick_detail_backfill: run_id=%d -- no pending listings "
|
|
"(detail_enriched_at IS NULL = 0), done",
|
|
run_id,
|
|
)
|
|
runs_mod.mark_done(db, run_id, current_counters)
|
|
return counters
|
|
|
|
logger.info(
|
|
"domclick_detail_backfill: run_id=%d snapshot=%d (budget=%.0fs "
|
|
"delay=%.1fs max_blocks=%d cookies=%s)",
|
|
run_id,
|
|
len(snapshot),
|
|
budget_sec,
|
|
request_delay_sec,
|
|
max_consecutive_blocks,
|
|
"yes" if cookies is not None else "no",
|
|
)
|
|
|
|
consecutive_blocks = 0
|
|
do_sleep = False
|
|
|
|
# Exactly ONE BrowserFetcher per run (no curl fallback for DomClick, see
|
|
# module docstring). source="domclick" -- infra identifier, dedicated
|
|
# residential proxy (scrape_proxies.provider_affinity='domclick',
|
|
# 173_scrape_proxies_add_domclick_affinity.sql).
|
|
async with BrowserFetcher(source="domclick", endpoint=settings.browser_http_endpoint) as bf:
|
|
for idx, row in enumerate(snapshot):
|
|
# Budget guard
|
|
elapsed = time.monotonic() - start
|
|
if elapsed > budget_sec:
|
|
logger.info(
|
|
"domclick_detail_backfill: run_id=%d -- budget %.0fs exhausted "
|
|
"(elapsed=%.1fs), stopping at #%d/%d",
|
|
run_id,
|
|
budget_sec,
|
|
elapsed,
|
|
idx,
|
|
len(snapshot),
|
|
)
|
|
break
|
|
|
|
# #1182 Phase 2: кооперативный SIGTERM-drain (деплой recreate scraper).
|
|
if shutdown_requested():
|
|
logger.info(
|
|
"domclick_detail_backfill: run_id=%d SIGTERM-drain — stopping at " "#%d/%d",
|
|
run_id,
|
|
idx,
|
|
len(snapshot),
|
|
)
|
|
break
|
|
|
|
# Jittered delay (±30%) -- organic pacing, matches the request_delay_sec
|
|
# spirit of PR #2430's same-site navigation (less robotic cadence).
|
|
if do_sleep:
|
|
await asyncio.sleep(request_delay_sec * random.uniform(0.7, 1.4))
|
|
do_sleep = True
|
|
|
|
source_url: str = row["source_url"]
|
|
listing_id: int = row["id"]
|
|
counters.attempted += 1
|
|
|
|
try:
|
|
enrichment = await fetch_detail(source_url, browser_fetcher=bf, cookies=cookies)
|
|
if save_detail_enrichment(db, listing_id, enrichment):
|
|
counters.enriched += 1
|
|
consecutive_blocks = 0
|
|
|
|
except DomClickParseError as e:
|
|
# Schema drift, not a block -- neutral to the block-breaker (does
|
|
# NOT touch consecutive_blocks, does NOT abort the run).
|
|
counters.failed += 1
|
|
logger.warning(
|
|
"domclick_detail_backfill: run_id=%d listing %s PARSE-ERROR "
|
|
"(schema drift, not a block): %s",
|
|
run_id,
|
|
source_url,
|
|
e,
|
|
)
|
|
|
|
except DomClickBlockedError as e:
|
|
consecutive_blocks += 1
|
|
counters.blocked += 1
|
|
logger.warning(
|
|
"domclick_detail_backfill: run_id=%d BLOCKED #%d/%d "
|
|
"(consecutive=%d): %s",
|
|
run_id,
|
|
idx + 1,
|
|
len(snapshot),
|
|
consecutive_blocks,
|
|
e,
|
|
)
|
|
if consecutive_blocks >= max_consecutive_blocks:
|
|
logger.error(
|
|
"domclick_detail_backfill: run_id=%d ABORT -- %d consecutive "
|
|
"blocks, QRATOR reputation likely burned for the session/proxy. "
|
|
"enriched=%d attempted=%d",
|
|
run_id,
|
|
consecutive_blocks,
|
|
counters.enriched,
|
|
counters.attempted,
|
|
)
|
|
break
|
|
|
|
except Exception as e:
|
|
counters.failed += 1
|
|
logger.warning(
|
|
"domclick_detail_backfill: run_id=%d listing %s failed: %s",
|
|
run_id,
|
|
source_url,
|
|
e,
|
|
)
|
|
try:
|
|
db.rollback()
|
|
except Exception:
|
|
pass
|
|
|
|
if counters.attempted % 25 == 0:
|
|
current_counters = counters.to_dict()
|
|
runs_mod.update_heartbeat(db, run_id, current_counters)
|
|
|
|
counters.duration_sec = time.monotonic() - start
|
|
current_counters = counters.to_dict()
|
|
runs_mod.mark_done(db, run_id, current_counters)
|
|
logger.info(
|
|
"domclick_detail_backfill: run_id=%d DONE -- attempted=%d enriched=%d "
|
|
"blocked=%d failed=%d duration=%.1fs",
|
|
run_id,
|
|
counters.attempted,
|
|
counters.enriched,
|
|
counters.blocked,
|
|
counters.failed,
|
|
counters.duration_sec,
|
|
)
|
|
return counters
|
|
|
|
except Exception as exc:
|
|
counters.duration_sec = time.monotonic() - start
|
|
logger.exception(
|
|
"domclick_detail_backfill: run_id=%d FAILED after %.1fs",
|
|
run_id,
|
|
counters.duration_sec,
|
|
)
|
|
runs_mod.mark_failed(db, run_id, str(exc)[:1000], counters.to_dict())
|
|
raise
|