fix(tradein): SAVEPOINT-isolate houses backfill + ruff/shell fixes (#568)

tradein-mvp/backend/app/services/scrapers/cian_newbuilding.py

@@ -391,7 +391,7 @@ def _extract_nested_offers(offers_state: dict[str, Any]) -> list[dict[str, Any]]
 # ---- save helpers ----
 
 
-async def save_newbuilding_enrichment(
+def save_newbuilding_enrichment(
     db: Any,
     house_id: int,
     enrichment: NewbuildingEnrichment,

tradein-mvp/backend/app/tasks/cian_history_backfill.py

@@ -207,13 +207,24 @@ async def backfill_cian_history(
                     continue
 
                 try:
-                    await save_newbuilding_enrichment(db, house_id, enrichment)
+                    save_newbuilding_enrichment(db, house_id, enrichment)
                     result.houses_succeeded += 1
                 except Exception as exc:
                     logger.warning(
                         "cian_newbuilding save failed for house_id=%s: %s", house_id, exc
                     )
                     result.houses_failed_save += 1
+                    # Roll back to clean session state so next house can proceed.
+                    # save_newbuilding_enrichment commits on success; on failure the
+                    # transaction is left open/dirty — rollback to avoid session poison.
+                    try:
+                        db.rollback()
+                    except Exception as rb_exc:
+                        logger.warning(
+                            "cian_newbuilding rollback failed for house_id=%s: %s",
+                            house_id,
+                            rb_exc,
+                        )
 
                 await asyncio.sleep(delay)
 

tradein-mvp/scripts/backfill_cian_zhk_url.py

@@ -32,7 +32,7 @@ import psycopg
 
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "backend"))
 
-from app.services.scrapers.cian_newbuilding import resolve_cian_zhk_url  # noqa: E402
+from app.services.scrapers.cian_newbuilding import resolve_cian_zhk_url
 
 logging.basicConfig(
     format="%(asctime)s [%(levelname)s] %(message)s",
@@ -100,7 +100,10 @@ async def main() -> None:
                 logger.info("[%d/%d] house_id=%s id=%s → %s", i, total, house_id, cian_id, url)
             else:
                 failed += 1
-                logger.warning("[%d/%d] house_id=%s id=%s — could not resolve URL", i, total, house_id, cian_id)
+                logger.warning(
+                    "[%d/%d] house_id=%s id=%s — could not resolve URL",
+                    i, total, house_id, cian_id,
+                )
 
             if i < total:
                 await asyncio.sleep(DELAY_SEC)

tradein-mvp/scripts/cian_backfill_all.sh

@@ -46,13 +46,15 @@ while (( i < MAX_ITERS )); do
   resp=$(curl -fsS -X POST "$url")
   lt=$(echo "$resp" | jq -r '.listings_total // 0')
   vt=$(echo "$resp" | jq -r '.valuations_total // 0')
+  ht=$(echo "$resp" | jq -r '.houses_total // 0')
   ls=$(echo "$resp" | jq -r '.listings_succeeded // 0')
   vs=$(echo "$resp" | jq -r '.valuations_succeeded // 0')
+  hs=$(echo "$resp" | jq -r '.houses_succeeded // 0')
   dur=$(echo "$resp" | jq -r '.duration_sec // 0')
   cum_listings=$((cum_listings + ls))
   cum_valuations=$((cum_valuations + vs))
-  echo "[$i] listings=$lt (ok=$ls) valuations=$vt (ok=$vs) ${dur}s  cum: hist=$cum_listings val=$cum_valuations"
-  if (( lt == 0 && vt == 0 )); then
+  echo "[$i] listings=$lt (ok=$ls) valuations=$vt (ok=$vs) houses=$ht (ok=$hs) ${dur}s  cum: hist=$cum_listings val=$cum_valuations"
+  if (( lt == 0 && vt == 0 && ht == 0 )); then
     echo "Drained — stopping."
     break
   fi