diff --git a/tradein-mvp/backend/app/services/scrapers/cian_newbuilding.py b/tradein-mvp/backend/app/services/scrapers/cian_newbuilding.py index 242e15b4..cc702d3f 100644 --- a/tradein-mvp/backend/app/services/scrapers/cian_newbuilding.py +++ b/tradein-mvp/backend/app/services/scrapers/cian_newbuilding.py @@ -391,7 +391,7 @@ def _extract_nested_offers(offers_state: dict[str, Any]) -> list[dict[str, Any]] # ---- save helpers ---- -async def save_newbuilding_enrichment( +def save_newbuilding_enrichment( db: Any, house_id: int, enrichment: NewbuildingEnrichment, diff --git a/tradein-mvp/backend/app/tasks/cian_history_backfill.py b/tradein-mvp/backend/app/tasks/cian_history_backfill.py index b8f29a7e..abbc03c0 100644 --- a/tradein-mvp/backend/app/tasks/cian_history_backfill.py +++ b/tradein-mvp/backend/app/tasks/cian_history_backfill.py @@ -207,13 +207,24 @@ async def backfill_cian_history( continue try: - await save_newbuilding_enrichment(db, house_id, enrichment) + save_newbuilding_enrichment(db, house_id, enrichment) result.houses_succeeded += 1 except Exception as exc: logger.warning( "cian_newbuilding save failed for house_id=%s: %s", house_id, exc ) result.houses_failed_save += 1 + # Roll back to clean session state so next house can proceed. + # save_newbuilding_enrichment commits on success; on failure the + # transaction is left open/dirty — rollback to avoid session poison. + try: + db.rollback() + except Exception as rb_exc: + logger.warning( + "cian_newbuilding rollback failed for house_id=%s: %s", + house_id, + rb_exc, + ) await asyncio.sleep(delay) diff --git a/tradein-mvp/scripts/backfill_cian_zhk_url.py b/tradein-mvp/scripts/backfill_cian_zhk_url.py index 0ab63fbf..5da7bb99 100644 --- a/tradein-mvp/scripts/backfill_cian_zhk_url.py +++ b/tradein-mvp/scripts/backfill_cian_zhk_url.py @@ -32,7 +32,7 @@ import psycopg sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "backend")) -from app.services.scrapers.cian_newbuilding import resolve_cian_zhk_url # noqa: E402 +from app.services.scrapers.cian_newbuilding import resolve_cian_zhk_url logging.basicConfig( format="%(asctime)s [%(levelname)s] %(message)s", @@ -100,7 +100,10 @@ async def main() -> None: logger.info("[%d/%d] house_id=%s id=%s → %s", i, total, house_id, cian_id, url) else: failed += 1 - logger.warning("[%d/%d] house_id=%s id=%s — could not resolve URL", i, total, house_id, cian_id) + logger.warning( + "[%d/%d] house_id=%s id=%s — could not resolve URL", + i, total, house_id, cian_id, + ) if i < total: await asyncio.sleep(DELAY_SEC) diff --git a/tradein-mvp/scripts/cian_backfill_all.sh b/tradein-mvp/scripts/cian_backfill_all.sh index 7458d59f..e2f2880d 100644 --- a/tradein-mvp/scripts/cian_backfill_all.sh +++ b/tradein-mvp/scripts/cian_backfill_all.sh @@ -46,13 +46,15 @@ while (( i < MAX_ITERS )); do resp=$(curl -fsS -X POST "$url") lt=$(echo "$resp" | jq -r '.listings_total // 0') vt=$(echo "$resp" | jq -r '.valuations_total // 0') + ht=$(echo "$resp" | jq -r '.houses_total // 0') ls=$(echo "$resp" | jq -r '.listings_succeeded // 0') vs=$(echo "$resp" | jq -r '.valuations_succeeded // 0') + hs=$(echo "$resp" | jq -r '.houses_succeeded // 0') dur=$(echo "$resp" | jq -r '.duration_sec // 0') cum_listings=$((cum_listings + ls)) cum_valuations=$((cum_valuations + vs)) - echo "[$i] listings=$lt (ok=$ls) valuations=$vt (ok=$vs) ${dur}s cum: hist=$cum_listings val=$cum_valuations" - if (( lt == 0 && vt == 0 )); then + echo "[$i] listings=$lt (ok=$ls) valuations=$vt (ok=$vs) houses=$ht (ok=$hs) ${dur}s cum: hist=$cum_listings val=$cum_valuations" + if (( lt == 0 && vt == 0 && ht == 0 )); then echo "Drained — stopping." break fi