fix(site-finder): apply 12mo cutoff to n_lots/sold_pct/price in enrich economics (#1513)

site-finder/08_enrich_economics.py

@@ -11,7 +11,11 @@ Per-flat is way richer than monthly Сводные:
 
 Replaces values in district_economics with `*_real` columns.
 """
-import sqlite3, pathlib, datetime as dt
+import datetime as dt
+import pathlib
+import sqlite3
+from collections import Counter
+
 DB = pathlib.Path(__file__).parent / "analysis.db"
 
 EXTRA = """
@@ -29,20 +33,39 @@ ALTER TABLE district_economics ADD COLUMN real_top_bank_share REAL;
 ALTER TABLE district_economics ADD COLUMN real_avg_readiness_pct REAL;
 """
 
+
 def safe_alter(conn, sql):
     for stmt in sql.strip().split(";"):
         s = stmt.strip()
-        if not s: continue
-        try: conn.execute(s)
+        if not s:
+            continue
+        try:
+            conn.execute(s)
         except sqlite3.OperationalError as e:
-            if "duplicate column" not in str(e): raise
+            if "duplicate column" not in str(e):
+                raise
+
 
 def percentile(vals, p):
-    if not vals: return None
+    if not vals:
+        return None
     vals = sorted(vals)
     k = (len(vals) - 1) * p
-    f = int(k); c = min(f+1, len(vals)-1)
-    return vals[f] + (vals[c]-vals[f]) * (k - f)
+    f = int(k)
+    c = min(f + 1, len(vals) - 1)
+    return vals[f] + (vals[c] - vals[f]) * (k - f)
+
+
+def _reg_date(it):
+    """Parse register_date from a lot tuple; returns None on missing/malformed."""
+    r = it[6]
+    if not r:
+        return None
+    try:
+        return dt.date.fromisoformat(r[:10])
+    except Exception:
+        return None
+
 
 def main():
     conn = sqlite3.connect(DB)
@@ -64,8 +87,16 @@ def main():
     cutoff_12mo = today - dt.timedelta(days=365)
 
     for d, items in by_d.items():
-        n_lots = len(items)
-        sold_items = [it for it in items if (it[1] or '').strip().lower() == 'да']
+        # Apply the same 12-month window to n_lots/sold_pct/prices as velocity uses.
+        # Keep unsold lots with no register_date (currently active in market).
+        # Exclude lots whose DDU was registered before the cutoff (stale history).
+        items_12mo = [
+            it for it in items
+            if (rd := _reg_date(it)) is None or rd >= cutoff_12mo
+        ]
+
+        n_lots = len(items_12mo)
+        sold_items = [it for it in items_12mo if (it[1] or '').strip().lower() == 'да']
         n_sold = len(sold_items)
         sold_pct = 100.0 * n_sold / n_lots if n_lots else None
 
@@ -75,36 +106,40 @@ def main():
         p75 = percentile(prices, 0.75) / 1000 if prices else None
 
         areas = [it[3] for it in sold_items if it[3] and it[3] > 0]
-        avg_area = sum(areas)/len(areas) if areas else None
+        avg_area = sum(areas) / len(areas) if areas else None
 
         # velocity: registered deals in last 12 mo / 12 / n_corpuses_in_district
+        # sold_items is already windowed to 12mo; the cutoff check is a safety guard.
         reg_dates = []
         for it in sold_items:
-            r = it[6]
-            if not r: continue
-            try:
-                rd = dt.date.fromisoformat(r[:10])
-                if rd >= cutoff_12mo: reg_dates.append(rd)
-            except: pass
-        # Distinct corpuses in district (from per-flat data)
-        n_corp = len({(it[0],) for it in items if it[0]})  # crude — but we want corpuses
-        n_corp_real = len({(it,) for it in conn.execute(
-            "SELECT DISTINCT project, corpus FROM objective_lots WHERE district=?", (d,)).fetchall()}) or 1
+            rd = _reg_date(it)
+            if rd is not None and rd >= cutoff_12mo:
+                reg_dates.append(rd)
+        # Distinct corpuses in district (lifetime count — denominator for velocity)
+        n_corp_real = (
+            len({it for it in conn.execute(
+                "SELECT DISTINCT project, corpus FROM objective_lots WHERE district=?",
+                (d,),
+            ).fetchall()})
+            or 1
+        )
         velocity = len(reg_dates) / 12.0 / n_corp_real if reg_dates else 0
 
+        # banks and readiness use full lifetime items — these reflect current market
+        # structure (which banks are active, current construction readiness).
         banks = [it[4] for it in items if it[4] and it[4].strip()]
         unique_banks = set(banks)
         top_bank, top_share = None, None
         if banks:
-            from collections import Counter
             c = Counter(banks)
             top_bank, top_n = c.most_common(1)[0]
             top_share = top_n / len(banks)
 
         ready_vals = [it[5] for it in items if it[5] is not None]
-        avg_ready = sum(ready_vals)/len(ready_vals) if ready_vals else None
+        avg_ready = sum(ready_vals) / len(ready_vals) if ready_vals else None
 
-        conn.execute("""UPDATE district_economics SET
+        conn.execute(
+            """UPDATE district_economics SET
             real_n_lots=?, real_n_sold=?, real_sold_pct=?,
             real_median_price_m2=?, real_p25_price_m2=?, real_p75_price_m2=?,
             real_avg_area_sold=?, real_velocity_per_month=?,
@@ -112,10 +147,14 @@ def main():
             real_avg_readiness_pct=?
             WHERE district=?""",
             (n_lots, n_sold, sold_pct, med, p25, p75, avg_area, velocity,
-             len(unique_banks), top_bank, top_share, avg_ready, d))
+             len(unique_banks), top_bank, top_share, avg_ready, d),
+        )
     conn.commit()
 
-    print(f"{'район':<22}{'лот':>6}{'прод':>7}{'sold%':>7}{'медцена':>9}{'площ':>6}{'vel':>6}{'банки':>6}{'top_bank':>22}{'%':>5}{'готовн':>7}")
+    print(
+        f"{'район':<22}{'лот':>6}{'прод':>7}{'sold%':>7}"
+        f"{'медцена':>9}{'площ':>6}{'vel':>6}{'банки':>6}{'top_bank':>22}{'%':>5}{'готовн':>7}"
+    )
     for r in conn.execute("""SELECT district, real_n_lots, real_n_sold, real_sold_pct,
                                     real_median_price_m2, real_avg_area_sold,
                                     real_velocity_per_month, real_n_banks,
@@ -124,9 +163,14 @@ def main():
                              WHERE real_n_lots>0
                              ORDER BY real_median_price_m2 DESC NULLS LAST""").fetchall():
         d, nl, ns, sp, mp, aa, v, nb, tb, ts, rd = r
-        print(f"{d:<22}{nl:>6}{ns:>7}{sp or 0:>6.1f}%{mp or 0:>9.1f}{aa or 0:>6.1f}{v:>6.2f}{nb or 0:>6}{(tb or '—')[:20]:>22}{(ts or 0)*100:>4.0f}%{rd or 0:>6.0f}%")
+        print(
+            f"{d:<22}{nl:>6}{ns:>7}{sp or 0:>6.1f}%{mp or 0:>9.1f}"
+            f"{aa or 0:>6.1f}{v:>6.2f}{nb or 0:>6}{(tb or '—')[:20]:>22}"
+            f"{(ts or 0)*100:>4.0f}%{rd or 0:>6.0f}%"
+        )
 
     conn.close()
 
+
 if __name__ == "__main__":
     main()