lekss361/gendesign
1
0
Fork 0
Code Issues 57 Pull requests Projects Releases Packages Wiki Activity Actions

fix(tradein): SAVEPOINT-isolate houses backfill + ruff/shell fixes (#568)

Browse source
This commit is contained in:
Light1YT 2026-05-28 18:54:27 +05:00
parent c47a42c0a5
commit cfa24c3bcf
4 changed files with 22 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
tradein-mvp/backend/app/services/scrapers/cian_newbuilding.py
View file

@ -391,7 +391,7 @@ def _extract_nested_offers(offers_state: dict[str, Any]) -> list[dict[str, Any]]
# ---- save helpers ---- # ---- save helpers ----
async def save_newbuilding_enrichment( def save_newbuilding_enrichment(
db: Any, db: Any,
house_id: int, house_id: int,
enrichment: NewbuildingEnrichment, enrichment: NewbuildingEnrichment,

13
tradein-mvp/backend/app/tasks/cian_history_backfill.py
View file

@ -207,13 +207,24 @@ async def backfill_cian_history(
continue continue
try: try:
await save_newbuilding_enrichment(db, house_id, enrichment) save_newbuilding_enrichment(db, house_id, enrichment)
result.houses_succeeded += 1 result.houses_succeeded += 1
except Exception as exc: except Exception as exc:
logger.warning( logger.warning(
"cian_newbuilding save failed for house_id=%s: %s", house_id, exc "cian_newbuilding save failed for house_id=%s: %s", house_id, exc
) )
result.houses_failed_save += 1 result.houses_failed_save += 1
# Roll back to clean session state so next house can proceed.
# save_newbuilding_enrichment commits on success; on failure the
# transaction is left open/dirty — rollback to avoid session poison.
try:
db.rollback()
except Exception as rb_exc:
logger.warning(
"cian_newbuilding rollback failed for house_id=%s: %s",
house_id,
rb_exc,
)
await asyncio.sleep(delay) await asyncio.sleep(delay)

7
tradein-mvp/scripts/backfill_cian_zhk_url.py
View file

@ -32,7 +32,7 @@ import psycopg
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "backend")) sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "backend"))
from app.services.scrapers.cian_newbuilding import resolve_cian_zhk_url # noqa: E402 from app.services.scrapers.cian_newbuilding import resolve_cian_zhk_url
logging.basicConfig( logging.basicConfig(
format="%(asctime)s [%(levelname)s] %(message)s", format="%(asctime)s [%(levelname)s] %(message)s",
@ -100,7 +100,10 @@ async def main() -> None:
logger.info("[%d/%d] house_id=%s id=%s → %s", i, total, house_id, cian_id, url) logger.info("[%d/%d] house_id=%s id=%s → %s", i, total, house_id, cian_id, url)
else: else:
failed += 1 failed += 1
logger.warning("[%d/%d] house_id=%s id=%s — could not resolve URL", i, total, house_id, cian_id) logger.warning(
"[%d/%d] house_id=%s id=%s — could not resolve URL",
i, total, house_id, cian_id,
)
if i < total: if i < total:
await asyncio.sleep(DELAY_SEC) await asyncio.sleep(DELAY_SEC)

6
tradein-mvp/scripts/cian_backfill_all.sh
View file

@ -46,13 +46,15 @@ while (( i < MAX_ITERS )); do
resp=$(curl -fsS -X POST "$url") resp=$(curl -fsS -X POST "$url")
lt=$(echo "$resp" | jq -r '.listings_total // 0') lt=$(echo "$resp" | jq -r '.listings_total // 0')
vt=$(echo "$resp" | jq -r '.valuations_total // 0') vt=$(echo "$resp" | jq -r '.valuations_total // 0')
ht=$(echo "$resp" | jq -r '.houses_total // 0')
ls=$(echo "$resp" | jq -r '.listings_succeeded // 0') ls=$(echo "$resp" | jq -r '.listings_succeeded // 0')
vs=$(echo "$resp" | jq -r '.valuations_succeeded // 0') vs=$(echo "$resp" | jq -r '.valuations_succeeded // 0')
hs=$(echo "$resp" | jq -r '.houses_succeeded // 0')
dur=$(echo "$resp" | jq -r '.duration_sec // 0') dur=$(echo "$resp" | jq -r '.duration_sec // 0')
cum_listings=$((cum_listings + ls)) cum_listings=$((cum_listings + ls))
cum_valuations=$((cum_valuations + vs)) cum_valuations=$((cum_valuations + vs))
echo "[$i] listings=$lt (ok=$ls) valuations=$vt (ok=$vs) ${dur}s cum: hist=$cum_listings val=$cum_valuations" echo "[$i] listings=$lt (ok=$ls) valuations=$vt (ok=$vs) houses=$ht (ok=$hs) ${dur}s cum: hist=$cum_listings val=$cum_valuations"
if (( lt == 0 && vt == 0 )); then if (( lt == 0 && vt == 0 && ht == 0 )); then
echo "Drained — stopping." echo "Drained — stopping."
break break
fi fi