From fe7727e338c400b78b12a16fadd91deffe1b54e1 Mon Sep 17 00:00:00 2001
From: lekss361 <lekss361@gendsgn.local>
Date: Sat, 6 Jun 2026 21:47:35 +0000
Subject: [PATCH] =?UTF-8?q?fix(sf):=20per-quarter=20commit=20=D0=B2=20ird/?=
 =?UTF-8?q?opportunity=20harvest=20=E2=80=94=20=D0=BD=D0=B5=20=D1=82=D0=B5?=
 =?UTF-8?q?=D1=80=D1=8F=D1=82=D1=8C=20=D0=BF=D1=80=D0=BE=D0=B3=D1=80=D0=B5?=
 =?UTF-8?q?=D1=81=D1=81=20=D0=BF=D1=80=D0=B8=20=D0=BA=D1=80=D0=B0=D1=88?=
 =?UTF-8?q?=D0=B5=20(#1107)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

ird_harvest + opportunity_harvest: commit после КАЖДОГО квартала (внутри per-quarter try) + db.rollback() в except перед continue. Прод-баг: single end-commit терял весь прогресс многочасового grid-walk при краше/таймауте/WorkerLost. Теперь durable поквартально; SAVEPOINT per-row не тронут. +2 теста (commit-count durability, flaky-quarter rollback+continue).

Refs #1067.
Co-authored-by: lekss361 <lekss361@gendsgn.local>
Co-committed-by: lekss361 <lekss361@gendsgn.local>
---
 backend/app/workers/tasks/ird_harvest.py      |  6 ++-
 .../app/workers/tasks/opportunity_harvest.py  |  6 ++-
 backend/tests/workers/test_ird_harvest.py     | 38 +++++++++++++++++++
 3 files changed, 48 insertions(+), 2 deletions(-)

diff --git a/backend/app/workers/tasks/ird_harvest.py b/backend/app/workers/tasks/ird_harvest.py
index c721ed83..42f50a96 100644
--- a/backend/app/workers/tasks/ird_harvest.py
+++ b/backend/app/workers/tasks/ird_harvest.py
@@ -211,10 +211,14 @@ def harvest_ird_overlays(quarters: list[str] | None = None) -> dict[str, int]:
                             fetched_at=fetched_at,
                         ):
                             n_features += 1
+                # Durable per-quarter commit: длинный grid-walk (часы) не теряет прогресс при
+                # краше/таймауте/WorkerLost середины прогона (commit раз-в-конце терял ВСЁ).
+                db.commit()
             except Exception as exc:
                 logger.warning("ird_harvest: квартал %s failed: %s", quarter, exc)
+                db.rollback()  # сбросить незакоммиченный tx квартала перед следующим
                 continue
-        db.commit()
+        db.commit()  # финальный — на случай пустого списка / подстраховка
         logger.info("ird_harvest: готово, кварталов=%d фич=%d", n_quarters, n_features)
         return {"quarters": n_quarters, "features": n_features}
     finally:
diff --git a/backend/app/workers/tasks/opportunity_harvest.py b/backend/app/workers/tasks/opportunity_harvest.py
index 392dece9..5a7589c8 100644
--- a/backend/app/workers/tasks/opportunity_harvest.py
+++ b/backend/app/workers/tasks/opportunity_harvest.py
@@ -95,10 +95,14 @@ def harvest_opportunity_overlays(quarters: list[str] | None = None) -> dict[str,
                             fetched_at=fetched_at,
                         ):
                             n_features += 1
+                # Durable per-quarter commit: длинный grid-walk не теряет прогресс при
+                # краше/таймауте середины прогона (commit раз-в-конце терял ВСЁ).
+                db.commit()
             except Exception as exc:
                 logger.warning("opportunity_harvest: квартал %s failed: %s", quarter, exc)
+                db.rollback()  # сбросить незакоммиченный tx квартала перед следующим
                 continue
-        db.commit()
+        db.commit()  # финальный — на случай пустого списка / подстраховка
         logger.info("opportunity_harvest: готово, кварталов=%d фич=%d", n_quarters, n_features)
         return {"quarters": n_quarters, "features": n_features}
     finally:
diff --git a/backend/tests/workers/test_ird_harvest.py b/backend/tests/workers/test_ird_harvest.py
index 1d075055..bc7c6e64 100644
--- a/backend/tests/workers/test_ird_harvest.py
+++ b/backend/tests/workers/test_ird_harvest.py
@@ -55,6 +55,8 @@ class _FakeDB:
     def __init__(self) -> None:
         self.executed: list[tuple[Any, dict[str, Any] | None]] = []
         self.committed = False
+        self.commit_count = 0
+        self.rollback_count = 0
         self.closed = False
 
     @contextmanager
@@ -67,6 +69,10 @@ class _FakeDB:
 
     def commit(self) -> None:
         self.committed = True
+        self.commit_count += 1
+
+    def rollback(self) -> None:
+        self.rollback_count += 1
 
     def close(self) -> None:
         self.closed = True
@@ -131,3 +137,35 @@ def test_upsert_feature_skips_missing_geom_data_id(monkeypatch: Any) -> None:
         is False
     )
     assert db.executed == []
+
+
+def test_per_quarter_commit_durability(monkeypatch: Any) -> None:
+    """Каждый квартал коммитится отдельно → краш середины не теряет прошлые (фикс #1067)."""
+    db = _FakeDB()
+    _patch(monkeypatch, db)
+
+    result = ird_harvest.harvest_ird_overlays(quarters=["66:41:0000001", "66:41:0000002"])
+
+    assert result["quarters"] == 2
+    # commit per-quarter (2) + финальный = >=2; точно больше одного (не «раз в конце»)
+    assert db.commit_count >= 2
+
+
+def test_quarter_failure_rolls_back_and_continues(monkeypatch: Any) -> None:
+    """Сбой одного квартала → rollback + продолжение; остальные кварталы коммитятся."""
+    db = _FakeDB()
+    client = _patch(monkeypatch, db)
+
+    def _flaky(cad: str, thematic_id: int = 1) -> Any:
+        if cad == "66:41:0000001":
+            raise RuntimeError("NSPD timeout")
+        ring = [[0, 0], [100, 0], [100, 100], [0, 100], [0, 0]]
+        return _SearchRes(_Feat({"type": "Polygon", "coordinates": [ring]}, {"cad": cad}))
+
+    monkeypatch.setattr(client, "search_by_cad", _flaky)
+
+    result = ird_harvest.harvest_ird_overlays(quarters=["66:41:0000001", "66:41:0000002"])
+
+    assert db.rollback_count >= 1  # битый квартал откатился
+    assert result["quarters"] == 1  # второй квартал обработан
+    assert db.committed is True