lekss361/gendesign
1
0
Fork 0
Code Issues 57 Pull requests Projects Releases Packages Wiki Activity Actions

fix

Browse source
This commit is contained in:
lekss361 2026-04-27 18:59:56 +03:00
parent 692a4fc5eb
commit bfee5ea916
1 changed files with 40 additions and 51 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

91
backend/app/services/scrapers/domrf_kn.py
View file

@ -501,6 +501,41 @@ def _body_of(e: Exception) -> str:
return msg[:500]
def _classify_and_log(
db: Session,
run_id: int | None,
obj_id: int | None,
endpoint: str,
full_url: str,
e: Exception,
) -> None:
"""Log endpoint failure at appropriate severity and persist to kn_scrape_failures.
404 from sale_graph/sales_agg is *expected* for сданные ЖК (no active sales) —
log as DEBUG so it doesn't pollute worker stdout / Sentry. Other errors stay
WARNING. Failures table records everything for the admin journal.
"""
status = _http_status_of(e)
is_expected_404 = status == 404 and endpoint.startswith(("sale_graph", "sales_agg"))
if is_expected_404:
# У сданных ЖК нет активных продаж — 404 ожидаемое поведение API.
# Не пишем ни в logger.warning (Sentry/Docker), ни в kn_scrape_failures
# (admin UI). Только debug-уровень для редкого диагностирования.
logger.debug("%s obj=%s 404 (expected for сданные)", endpoint, obj_id)
return
logger.warning("%s obj=%s FAILED: %s\n URL: %s", endpoint, obj_id, e, full_url)
_log_failure(
db,
run_id,
obj_id,
endpoint,
full_url,
status,
str(e),
_body_of(e),
)
def _log_failure(
db: Session,
run_id: int | None,
@ -936,22 +971,8 @@ async def run_region_sweep(
full_url = (
f"{BASE_URL}{PATH_SALE_GRAPH.format(obj_id=obj_id)}?type={type_}"
)
logger.warning(
"sale_graph obj=%s type=%s FAILED: %s\n URL: %s",
obj_id,
type_,
e,
full_url,
)
_log_failure(
db,
run_id,
obj_id,
f"sale_graph_{type_}",
full_url,
_http_status_of(e),
str(e),
_body_of(e),
_classify_and_log(
db, run_id, obj_id, f"sale_graph_{type_}", full_url, e
)
# sales_agg
@ -962,19 +983,7 @@ async def run_region_sweep(
)
except Exception as e:
full_url = f"{BASE_URL}{PATH_SALES_AGG.format(obj_id=obj_id)}"
logger.warning(
"sales_agg obj=%s FAILED: %s\n URL: %s", obj_id, e, full_url
)
_log_failure(
db,
run_id,
obj_id,
"sales_agg",
full_url,
_http_status_of(e),
str(e),
_body_of(e),
)
_classify_and_log(db, run_id, obj_id, "sales_agg", full_url, e)
# infrastructure
try:
@ -984,17 +993,7 @@ async def run_region_sweep(
)
except Exception as e:
full_url = f"{BASE_URL}{PATH_INFRA.format(obj_id=obj_id)}"
logger.warning("infra obj=%s FAILED: %s\n URL: %s", obj_id, e, full_url)
_log_failure(
db,
run_id,
obj_id,
"infrastructure",
full_url,
_http_status_of(e),
str(e),
_body_of(e),
)
_classify_and_log(db, run_id, obj_id, "infrastructure", full_url, e)
# photos (metadata + optional binary)
try:
@ -1008,17 +1007,7 @@ async def run_region_sweep(
)
except Exception as e:
full_url = f"{BASE_URL}{PATH_PHOTOS.format(obj_id=obj_id)}"
logger.warning("photos obj=%s FAILED: %s\n URL: %s", obj_id, e, full_url)
_log_failure(
db,
run_id,
obj_id,
"photos",
full_url,
_http_status_of(e),
str(e),
_body_of(e),
)
_classify_and_log(db, run_id, obj_id, "photos", full_url, e)
if (i + 1) % 10 == 0:
logger.info(