From bfee5ea9168d30ea50a6823a168d1736f1da61e3 Mon Sep 17 00:00:00 2001
From: lekss361 <claudestars@proton.me>
Date: Mon, 27 Apr 2026 18:59:56 +0300
Subject: [PATCH] fix

---
 backend/app/services/scrapers/domrf_kn.py | 91 ++++++++++-------------
 1 file changed, 40 insertions(+), 51 deletions(-)

diff --git a/backend/app/services/scrapers/domrf_kn.py b/backend/app/services/scrapers/domrf_kn.py
index 846b402c..9db5c09a 100644
--- a/backend/app/services/scrapers/domrf_kn.py
+++ b/backend/app/services/scrapers/domrf_kn.py
@@ -501,6 +501,41 @@ def _body_of(e: Exception) -> str:
     return msg[:500]
 
 
+def _classify_and_log(
+    db: Session,
+    run_id: int | None,
+    obj_id: int | None,
+    endpoint: str,
+    full_url: str,
+    e: Exception,
+) -> None:
+    """Log endpoint failure at appropriate severity and persist to kn_scrape_failures.
+
+    404 from sale_graph/sales_agg is *expected* for сданные ЖК (no active sales) —
+    log as DEBUG so it doesn't pollute worker stdout / Sentry. Other errors stay
+    WARNING. Failures table records everything for the admin journal.
+    """
+    status = _http_status_of(e)
+    is_expected_404 = status == 404 and endpoint.startswith(("sale_graph", "sales_agg"))
+    if is_expected_404:
+        # У сданных ЖК нет активных продаж — 404 ожидаемое поведение API.
+        # Не пишем ни в logger.warning (Sentry/Docker), ни в kn_scrape_failures
+        # (admin UI). Только debug-уровень для редкого диагностирования.
+        logger.debug("%s obj=%s 404 (expected for сданные)", endpoint, obj_id)
+        return
+    logger.warning("%s obj=%s FAILED: %s\n  URL: %s", endpoint, obj_id, e, full_url)
+    _log_failure(
+        db,
+        run_id,
+        obj_id,
+        endpoint,
+        full_url,
+        status,
+        str(e),
+        _body_of(e),
+    )
+
+
 def _log_failure(
     db: Session,
     run_id: int | None,
@@ -936,22 +971,8 @@ async def run_region_sweep(
                             full_url = (
                                 f"{BASE_URL}{PATH_SALE_GRAPH.format(obj_id=obj_id)}?type={type_}"
                             )
-                            logger.warning(
-                                "sale_graph obj=%s type=%s FAILED: %s\n  URL: %s",
-                                obj_id,
-                                type_,
-                                e,
-                                full_url,
-                            )
-                            _log_failure(
-                                db,
-                                run_id,
-                                obj_id,
-                                f"sale_graph_{type_}",
-                                full_url,
-                                _http_status_of(e),
-                                str(e),
-                                _body_of(e),
+                            _classify_and_log(
+                                db, run_id, obj_id, f"sale_graph_{type_}", full_url, e
                             )
 
                     # sales_agg
@@ -962,19 +983,7 @@ async def run_region_sweep(
                         )
                     except Exception as e:
                         full_url = f"{BASE_URL}{PATH_SALES_AGG.format(obj_id=obj_id)}"
-                        logger.warning(
-                            "sales_agg obj=%s FAILED: %s\n  URL: %s", obj_id, e, full_url
-                        )
-                        _log_failure(
-                            db,
-                            run_id,
-                            obj_id,
-                            "sales_agg",
-                            full_url,
-                            _http_status_of(e),
-                            str(e),
-                            _body_of(e),
-                        )
+                        _classify_and_log(db, run_id, obj_id, "sales_agg", full_url, e)
 
                     # infrastructure
                     try:
@@ -984,17 +993,7 @@ async def run_region_sweep(
                         )
                     except Exception as e:
                         full_url = f"{BASE_URL}{PATH_INFRA.format(obj_id=obj_id)}"
-                        logger.warning("infra obj=%s FAILED: %s\n  URL: %s", obj_id, e, full_url)
-                        _log_failure(
-                            db,
-                            run_id,
-                            obj_id,
-                            "infrastructure",
-                            full_url,
-                            _http_status_of(e),
-                            str(e),
-                            _body_of(e),
-                        )
+                        _classify_and_log(db, run_id, obj_id, "infrastructure", full_url, e)
 
                     # photos (metadata + optional binary)
                     try:
@@ -1008,17 +1007,7 @@ async def run_region_sweep(
                         )
                     except Exception as e:
                         full_url = f"{BASE_URL}{PATH_PHOTOS.format(obj_id=obj_id)}"
-                        logger.warning("photos obj=%s FAILED: %s\n  URL: %s", obj_id, e, full_url)
-                        _log_failure(
-                            db,
-                            run_id,
-                            obj_id,
-                            "photos",
-                            full_url,
-                            _http_status_of(e),
-                            str(e),
-                            _body_of(e),
-                        )
+                        _classify_and_log(db, run_id, obj_id, "photos", full_url, e)
 
                     if (i + 1) % 10 == 0:
                         logger.info(