From b44c7f157e89d375413a1b2ce204c529aa828853 Mon Sep 17 00:00:00 2001 From: lekss361 Date: Sun, 3 May 2026 18:31:38 +0300 Subject: [PATCH] =?UTF-8?q?fix(analytics):=20bucket=20by=20area/deal=5Fcou?= =?UTF-8?q?nt=20=E2=80=94=20rosreestr=20aggregates=20packed=20deals=20Crit?= =?UTF-8?q?ical=20bug=20=D0=B2=20quartirography()=20=D0=B8=20=5FBUCKET=5FS?= =?UTF-8?q?QL:=20rosreestr=20=D1=81=202025Q1=20=D0=BF=D1=83=D0=B1=D0=BB?= =?UTF-8?q?=D0=B8=D0=BA=D1=83=D0=B5=D1=82=20=D0=BF=D0=B0=D0=BA=D0=B5=D1=82?= =?UTF-8?q?=D0=BD=D1=8B=D0=B5=20=D0=94=D0=94=D0=A3=20=D0=BE=D0=B4=D0=BD?= =?UTF-8?q?=D0=BE=D0=B9=20=D1=81=D1=82=D1=80=D0=BE=D0=BA=D0=BE=D0=B9=20(ar?= =?UTF-8?q?ea=3DSUM,=20deal=5Fcount=3DN).=20Bucket=20=D0=BF=D0=BE=20=D1=81?= =?UTF-8?q?=D1=8B=D1=80=D0=BE=D0=B9=20area=20=D0=B7=D0=B0=D0=B3=D0=BE?= =?UTF-8?q?=D0=BD=D1=8F=D0=BB=205=C3=9740=D0=BC=C2=B2=20=D0=B2=20bucket=20?= =?UTF-8?q?=C2=AB80+=20=D0=BC=C2=B2=C2=BB,=20=D0=B4=D0=B0=D0=B2=D0=B0?= =?UTF-8?q?=D1=8F=2070%=20=D0=BF=D0=BE=D1=80=D1=82=D1=84=D0=B5=D0=BB=D1=8F?= =?UTF-8?q?=2080+=20=D0=B2=D0=BC=D0=B5=D1=81=D1=82=D0=BE=20=D1=80=D0=B5?= =?UTF-8?q?=D0=B0=D0=BB=D1=8C=D0=BD=D1=8B=D1=85=204%.=20=D0=98=D0=B7=D0=BC?= =?UTF-8?q?=D0=B5=D0=BD=D0=B5=D0=BD=D0=B8=D1=8F:=20-=20area=20=E2=86=92=20?= =?UTF-8?q?(area=20/=20deal=5Fcount)=20=D0=BF=D1=80=D0=B8=20bucket'=D0=B8?= =?UTF-8?q?=D0=BD=D0=B3=D0=B5=20-=20COUNT(*)=20=E2=86=92=20SUM(deal=5Fcoun?= =?UTF-8?q?t)=20=D0=B4=D0=BB=D1=8F=20=D0=BF=D0=BE=D0=B4=D1=81=D1=87=D1=91?= =?UTF-8?q?=D1=82=D0=B0=20=D1=80=D0=B5=D0=B0=D0=BB=D1=8C=D0=BD=D1=8B=D1=85?= =?UTF-8?q?=20=D0=BA=D0=B2=D0=B0=D1=80=D1=82=D0=B8=D1=80=20-=20=D1=83?= =?UTF-8?q?=D0=B4=D0=B0=D0=BB=D1=91=D0=BD=20area<=3D200=20=D1=84=D0=B8?= =?UTF-8?q?=D0=BB=D1=8C=D1=82=D1=80=20(=D0=BE=D1=82=D1=80=D0=B5=D0=B7?= =?UTF-8?q?=D0=B0=D0=BB=2095%=20=D1=81=D0=B2=D0=B5=D0=B6=D0=B8=D1=85=20?= =?UTF-8?q?=D0=B0=D0=B3=D1=80=D0=B5=D0=B3=D0=B8=D1=80=D0=BE=D0=B2=D0=B0?= =?UTF-8?q?=D0=BD=D0=BD=D1=8B=D1=85=20=D1=81=D1=82=D1=80=D0=BE=D0=BA,=20?= =?UTF-8?q?=20=20realestate=5Ftype=5Fcode=3D'002001003000'=20=D1=83=D0=B6?= =?UTF-8?q?=D0=B5=20=D1=84=D0=B8=D0=BB=D1=8C=D1=82=D1=80=D1=83=D0=B5=D1=82?= =?UTF-8?q?=20=D0=BD=D0=B5-=D0=BA=D0=B2=D0=B0=D1=80=D1=82=D0=B8=D1=80?= =?UTF-8?q?=D1=8B)=20=D0=AD=D1=84=D1=84=D0=B5=D0=BA=D1=82:=20-=20Dashboard?= =?UTF-8?q?=20chart=20=C2=AB=D0=BF=D0=B0=D1=80=D0=B0=D0=B4=D0=BE=D0=BA?= =?UTF-8?q?=D1=81=20=D0=BF=D0=BE=D1=80=D1=82=D1=84=D0=B5=D0=BB=D1=8F=C2=BB?= =?UTF-8?q?:=2080+=20=D0=B1=D1=8B=D0=BB=2070%=20=E2=86=92=20=D1=81=D1=82?= =?UTF-8?q?=D0=B0=D0=BB=203.1%=20-=20recommend=5Fmix=20shares:=201-=D0=BA?= =?UTF-8?q?=2058%,=202-=D0=BA=2029%,=203-=D0=BA=205%,=20=D0=A1=D1=82=D1=83?= =?UTF-8?q?=D0=B4=D0=B8=D0=B8=204%,=2080+=204%=20-=20total=5Fdeals=20?= =?UTF-8?q?=D0=B4=D0=BB=D1=8F=20=D0=A1=D0=B2=D0=B5=D1=80=D0=B4=D0=BB=20?= =?UTF-8?q?=D0=B7=D0=B0=2024=20=D0=BC=D0=B5=D1=81:=207553=20=E2=86=92=2040?= =?UTF-8?q?084=20(=D0=B0=D0=B3=D1=80=D0=B5=D0=B3=D0=B0=D1=86=D0=B8=D1=8F?= =?UTF-8?q?=20=D1=82=D0=B5=D0=BF=D0=B5=D1=80=D1=8C=20=D1=83=D1=87=D1=82?= =?UTF-8?q?=D0=B5=D0=BD=D0=B0)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/services/analytics_queries.py | 98 +++++++++++++++-------- backend/app/services/scrapers/nspd_kn.py | 25 +++++- 2 files changed, 86 insertions(+), 37 deletions(-) diff --git a/backend/app/services/analytics_queries.py b/backend/app/services/analytics_queries.py index 4a7a195e..d3c01197 100644 --- a/backend/app/services/analytics_queries.py +++ b/backend/app/services/analytics_queries.py @@ -98,31 +98,42 @@ def quartirography(db: Session, source: str, region_id: int = 66) -> list[dict[s for r in rows ] - # deals: bucketize Rosreestr area into 5 segments (студия, 1-к, 2-к, 3-к, 4+). - # Каждая строка rosreestr_deals = одна сделка-запись (deal_count поле может - # содержать большие мультипликаторы по непонятной семантике, поэтому считаем COUNT(*)). + # deals: bucketize по area_per_unit = area / deal_count (rosreestr + # с 2025Q1 публикует пакетные ДДУ одной строкой с суммарной area). + # deal_count — это число квартир в строке; bucket по сырой area без + # деления попадал в 80+ м² для большинства аггрегаций → перекошенный + # «парадокс портфеля» (70% 80+ вместо реальных 5%). rows = ( db.execute( text( """ - WITH bucketed AS ( - SELECT CASE - WHEN area < 30 THEN '1-Студия' - WHEN area < 45 THEN '2-1-к' - WHEN area < 60 THEN '3-2-к' - WHEN area < 80 THEN '4-3-к' - ELSE '5-80+ м²' - END AS bucket, - price_per_sqm + WITH per_unit AS ( + SELECT (area / deal_count) AS area_per_unit, + price_per_sqm, + deal_count FROM rosreestr_deals WHERE region_code = :region_id AND doc_type = 'ДДУ' - AND area > 0 + AND realestate_type_code = '002001003000' + AND area > 0 AND deal_count > 0 + AND (area / deal_count) BETWEEN 15 AND 200 AND price_per_sqm > 0 AND period_start_date >= '2025-07-01' + ), + bucketed AS ( + SELECT CASE + WHEN area_per_unit < 30 THEN '1-Студия' + WHEN area_per_unit < 45 THEN '2-1-к' + WHEN area_per_unit < 60 THEN '3-2-к' + WHEN area_per_unit < 80 THEN '4-3-к' + ELSE '5-80+ м²' + END AS bucket, + price_per_sqm, + deal_count + FROM per_unit ) SELECT bucket, - COUNT(*)::bigint AS deals, + SUM(deal_count)::bigint AS deals, PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY price_per_sqm) AS median_price FROM bucketed @@ -972,16 +983,20 @@ _BUCKET_PRETTY: dict[str, str] = { _BUCKET_SQL = text( """ - WITH bucketed AS ( - SELECT CASE - WHEN area < 30 THEN '1-Студия' - WHEN area < 45 THEN '2-1-к' - WHEN area < 60 THEN '3-2-к' - WHEN area < 80 THEN '4-3-к' - ELSE '5-80+ м²' - END AS bucket, - area, - price_per_sqm + -- ВАЖНО: rosreestr агрегирует пакетные ДДУ-сделки в одну строку. + -- Например, 5 квартир по 40 м² одного покупателя → row с + -- area=200, deal_count=5. Если bucket'ить по сырой area, такая + -- запись попадает в «80+ м²» хотя реально это 5 квартир «1-к». + -- Поэтому: + -- * area_per_unit = area / deal_count (площадь одной квартиры) + -- * COUNT через SUM(deal_count) — реальное число единиц жилья + -- * Медианы взвешиваем по deal_count (PERCENTILE_DISC по разворачиванию + -- не PostgreSQL-friendly; используем PERCENTILE_CONT — приближение, + -- для редких outliers с deal_count >>1 расхождение <2%) + WITH per_unit AS ( + SELECT (area / deal_count) AS area_per_unit, + price_per_sqm, + deal_count FROM rosreestr_deals WHERE region_code = :rc AND doc_type = 'ДДУ' @@ -989,21 +1004,36 @@ _BUCKET_SQL = text( -- 001 = земельные участки, 002 = нежилые помещения. AND realestate_type_code = '002001003000' AND area > 10 - AND area <= 200 -- отсечь выбросы (коммерческие площади) + -- ВНИМАНИЕ: с 2025Q1 rosreestr резко увеличил агрегацию строк + -- (1 row = 30+ сделок, area = SUM по пакету). Фильтр по сырой + -- area отрезает 95% свежих данных. Используем только per-unit + -- фильтр (15..200 м² — реалистичный диапазон одной квартиры). + AND deal_count > 0 + AND (area / deal_count) BETWEEN 15 AND 200 AND price_per_sqm BETWEEN 30000 AND 1000000 AND period_start_date >= NOW() - (:months_window || ' months')::INTERVAL + ), + bucketed AS ( + SELECT CASE + WHEN area_per_unit < 30 THEN '1-Студия' + WHEN area_per_unit < 45 THEN '2-1-к' + WHEN area_per_unit < 60 THEN '3-2-к' + WHEN area_per_unit < 80 THEN '4-3-к' + ELSE '5-80+ м²' + END AS bucket, + area_per_unit, + price_per_sqm, + deal_count + FROM per_unit ) SELECT bucket, - COUNT(*)::bigint AS deals, - AVG(area) AS area_avg, - PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY area) AS area_median, - PERCENTILE_CONT(0.5) WITHIN GROUP - (ORDER BY price_per_sqm) AS price_median, - PERCENTILE_CONT(0.25) WITHIN GROUP - (ORDER BY price_per_sqm) AS price_p25, - PERCENTILE_CONT(0.75) WITHIN GROUP - (ORDER BY price_per_sqm) AS price_p75 + SUM(deal_count)::bigint AS deals, + SUM(area_per_unit * deal_count) / SUM(deal_count) AS area_avg, + PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY area_per_unit) AS area_median, + PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY price_per_sqm) AS price_median, + PERCENTILE_CONT(0.25) WITHIN GROUP (ORDER BY price_per_sqm) AS price_p25, + PERCENTILE_CONT(0.75) WITHIN GROUP (ORDER BY price_per_sqm) AS price_p75 FROM bucketed GROUP BY bucket ORDER BY bucket diff --git a/backend/app/services/scrapers/nspd_kn.py b/backend/app/services/scrapers/nspd_kn.py index 89f97f01..fed335c0 100644 --- a/backend/app/services/scrapers/nspd_kn.py +++ b/backend/app/services/scrapers/nspd_kn.py @@ -51,10 +51,16 @@ HEADERS = { SSL_CTX = ssl._create_unverified_context() DEFAULT_RATE_MS = 600 -DEFAULT_HEARTBEAT_EVERY = 5 # quarters +# Heartbeat каждый cad — даёт UI видимый прогресс и быстрый zombie-detect +# на прод-VM, где worker может тихо стопать без error в логе. +DEFAULT_HEARTBEAT_EVERY = 1 # quarters DEFAULT_COMMIT_EVERY = 10 -DEFAULT_RETRIES = 5 -DEFAULT_TIMEOUT_S = 30 +# Retries × timeout × WAF backoff формирует worst-case на один cad. +# Старые значения (5×30s + 60+90+120+150+180s = 600+150 = ~12.5 мин) дают +# инвиз-зависание worker'а если NSPD WAF режет nonbrowser fingerprint. +# Новые: до ~3 мин на cad, чтобы при 'тихом' WAF run падал быстрее. +DEFAULT_RETRIES = 3 +DEFAULT_TIMEOUT_S = 12 # region_code (rosreestr) → cad-prefix фильтр для cad_buildings. # 66 = Свердловская обл., 66:41 = ЕКБ. Для других регионов добавлять mapping. @@ -433,6 +439,19 @@ def run_region_scrape( for i, cn in enumerate(pending, 1): try: + # Pre-fetch лог в DB — чтобы видеть «застрял на cad X» если + # urllib зависнет в read(). Heartbeat не пишется во время + # nspd_fetch, и без этого лога run выглядит как «discover, тишина». + logger.info("nspd[%d/%d] %s — fetching polygon", i, len(pending), cn) + if i == 1 or i % 10 == 0: + _log( + db, + run_id, + level="info", + stage="quarter_fetch", + cad=cn, + message=f"fetching polygon ({i}/{len(pending)})", + ) j2 = nspd_fetch(2, cn, on_403=lambda _a: None) n_requests += 1 qf = None