feat(avito): capture newbuilding ЖК id/url from SERP card; persist newbuilding_id/url

tradein-mvp/backend/app/services/scrapers/avito.py

@@ -28,7 +28,7 @@ import logging
 import re
 from datetime import date, datetime, timedelta, timezone
 from typing import Any
-from urllib.parse import urlencode, urljoin
+from urllib.parse import urlencode, urljoin, urlparse, urlunparse
 
 from curl_cffi.requests import AsyncSession
 from selectolax.parser import HTMLParser
@@ -750,6 +750,23 @@ class AvitoScraper(BaseScraper):
             dev_name = card.css_first('[data-marker="item-development-name"]')
             listing_segment = "novostroyki" if dev_name is not None else "vtorichka"
 
+            # Newbuilding (ЖК) link: новостройки несут якорь на ЖК-сабдомен
+            # <a href="https://zhk-<slug>-ekaterinburg.avito.ru?context=...">.
+            # newbuilding_url — host+path без ?context (mirror urlparse-cleaning),
+            # newbuilding_id — <slug> = host без 'zhk-' префикса и '.avito.ru' суффикса.
+            newbuilding_id: str | None = None
+            newbuilding_url: str | None = None
+            if dev_name is not None:
+                nb_link = card.css_first('a[href*="zhk-"][href*=".avito.ru"]')
+                if nb_link is not None:
+                    nb_href = nb_link.attributes.get("href", "") or ""
+                    parsed = urlparse(nb_href)
+                    host = parsed.netloc
+                    if host.startswith("zhk-") and host.endswith(".avito.ru"):
+                        # strip query/fragment, keep scheme+host+path
+                        newbuilding_url = urlunparse((parsed.scheme, host, parsed.path, "", "", ""))
+                        newbuilding_id = host[len("zhk-") : -len(".avito.ru")]
+
             return ScrapedLot(
                 source="avito",
                 source_url=url,
@@ -769,6 +786,8 @@ class AvitoScraper(BaseScraper):
                 house_ext_id=house_ext_id,
                 house_url=house_url,
                 listing_segment=listing_segment,
+                newbuilding_id=newbuilding_id,
+                newbuilding_url=newbuilding_url,
             )
         except Exception:
             # #823: логируем каждую сломанную карточку и считаем для observability.

tradein-mvp/backend/app/services/scrapers/base.py

@@ -89,6 +89,10 @@ class ScrapedLot(BaseModel):
     house_url: str | None = None  # full URL of /catalog/houses/...
     listing_segment: str | None = None  # 'vtorichka' / 'novostroyki'
 
+    # Newbuilding (ЖК) linking — slug-id + canonical ЖК-subdomain URL
+    newbuilding_id: str | None = None  # e.g. 'federatsiya-ekaterinburg'
+    newbuilding_url: str | None = None  # e.g. https://zhk-federatsiya-ekaterinburg.avito.ru
+
     # Цена (обязательно)
     price_rub: int = Field(gt=0)
     price_per_m2: int | None = None
@@ -262,6 +266,7 @@ def save_listings(
                     rooms, area_m2, floor, total_floors, year_built,
                     house_type, repair_state, has_balcony,
                     house_source, house_ext_id, house_url, listing_segment,
+                    newbuilding_id, newbuilding_url,
                     price_rub, price_per_m2,
                     listing_date, publish_date, days_on_market, description,
                     photo_urls, raw_payload,
@@ -280,6 +285,7 @@ def save_listings(
                     :rooms, :area_m2, :floor, :total_floors, :year_built,
                     :house_type, :repair_state, :has_balcony,
                     :house_source, :house_ext_id, :house_url, :listing_segment,
+                    :newbuilding_id, :newbuilding_url,
                     :price_rub, :ppm2,
                     :listing_date, :publish_date, :days_on_market, :description,
                     CAST(:photos AS jsonb),
@@ -340,6 +346,12 @@ def save_listings(
                         price_trend = COALESCE(EXCLUDED.price_trend, listings.price_trend),
                         price_previous_rub = COALESCE(
                             EXCLUDED.price_previous_rub, listings.price_previous_rub
+                        ),
+                        newbuilding_id = COALESCE(
+                            EXCLUDED.newbuilding_id, listings.newbuilding_id
+                        ),
+                        newbuilding_url = COALESCE(
+                            EXCLUDED.newbuilding_url, listings.newbuilding_url
                         )
                 RETURNING id, (xmax = 0) AS inserted
                 """
@@ -364,6 +376,8 @@ def save_listings(
                 "house_ext_id": lot.house_ext_id,
                 "house_url": lot.house_url,
                 "listing_segment": lot.listing_segment,
+                "newbuilding_id": lot.newbuilding_id,
+                "newbuilding_url": lot.newbuilding_url,
                 "price_rub": lot.price_rub,
                 "ppm2": ppm2,
                 "listing_date": lot.listing_date,

tradein-mvp/backend/tests/test_avito_serp_date.py

@@ -68,14 +68,20 @@ def test_lazy_card_date_comes_from_json_not_dom() -> None:
 # (название ЖК/застройщика рендерится только у карточек новостроек).
 
 
-def _serp_card(item_id: str, *, with_dev_name: bool) -> str:
+def _serp_card(item_id: str, *, with_dev_name: bool, with_zhk_anchor: bool = False) -> str:
     dev = '<div data-marker="item-development-name">ЖК «Федерация»</div>' if with_dev_name else ""
+    zhk = (
+        '<a href="https://zhk-fed-ekaterinburg.avito.ru?context=H4sIAAAA">ЖК Федерация</a>'
+        if with_zhk_anchor
+        else ""
+    )
     return f"""
     <div data-marker="item" data-item-id="{item_id}">
       <a data-marker="item-title" href="/ekaterinburg/kvartiry/2-k_kvartira_{item_id}">
         2-к. квартира, 50 м², 5/20 эт.</a>
       <meta itemprop="price" content="9500000"/>
       {dev}
+      {zhk}
     </div>
     """
 
@@ -94,3 +100,36 @@ def test_segment_vtorichka_when_no_development_name() -> None:
     lots = AvitoScraper()._parse_html(html, "https://www.avito.ru/ekaterinburg/kvartiry")
     assert len(lots) == 1
     assert lots[0].listing_segment == "vtorichka"
+
+
+# ── newbuilding (ЖК) id/url из zhk-сабдомена ──────────────────────────────────
+
+
+def test_newbuilding_captured_from_zhk_anchor() -> None:
+    """Новостройка с якорем zhk-*.avito.ru → newbuilding_url без ?context, id = slug."""
+    card = _serp_card("8043003287", with_dev_name=True, with_zhk_anchor=True)
+    html = f"<html><body>{card}</body></html>"
+    lots = AvitoScraper()._parse_html(html, "https://www.avito.ru/ekaterinburg/kvartiry")
+    assert len(lots) == 1
+    assert lots[0].newbuilding_url == "https://zhk-fed-ekaterinburg.avito.ru"
+    assert lots[0].newbuilding_id == "fed-ekaterinburg"
+
+
+def test_newbuilding_none_when_novostroyka_without_anchor() -> None:
+    """Новостройка без zhk-якоря → оба поля None (сегмент по dev-name остаётся)."""
+    card = _serp_card("8043003287", with_dev_name=True, with_zhk_anchor=False)
+    html = f"<html><body>{card}</body></html>"
+    lots = AvitoScraper()._parse_html(html, "https://www.avito.ru/ekaterinburg/kvartiry")
+    assert len(lots) == 1
+    assert lots[0].listing_segment == "novostroyki"
+    assert lots[0].newbuilding_id is None
+    assert lots[0].newbuilding_url is None
+
+
+def test_newbuilding_none_for_vtorichka() -> None:
+    """Вторичка → newbuilding_id/url оба None."""
+    html = f"<html><body>{_serp_card('8043936560', with_dev_name=False)}</body></html>"
+    lots = AvitoScraper()._parse_html(html, "https://www.avito.ru/ekaterinburg/kvartiry")
+    assert len(lots) == 1
+    assert lots[0].newbuilding_id is None
+    assert lots[0].newbuilding_url is None

tradein-mvp/backend/tests/test_base_save_listings.py

@@ -695,3 +695,49 @@ def test_save_listings_non_yandex_rich_cols_are_null():
     assert params["predicted_price_max"] is None
     assert params["price_trend"] is None
     assert params["price_previous_rub"] is None
+
+
+# ── newbuilding (ЖК) id/url persistence ──────────────────────────────────────
+
+
+def test_save_listings_persists_newbuilding_fields():
+    """Lot с newbuilding_id/url → оба попадают в params INSERT + есть в SQL."""
+    db = _mock_db(inserted=True)
+    lot = _base_lot(
+        source="avito",
+        source_id="nb_test",
+        newbuilding_id="federatsiya-ekaterinburg",
+        newbuilding_url="https://zhk-federatsiya-ekaterinburg.avito.ru",
+    )
+
+    save_listings(db, [lot])
+
+    params = _get_execute_params(db)
+    assert params["newbuilding_id"] == "federatsiya-ekaterinburg"
+    assert params["newbuilding_url"] == "https://zhk-federatsiya-ekaterinburg.avito.ru"
+
+    sql_text = _get_listings_insert_sql(db)
+    assert "newbuilding_id" in sql_text
+    assert "newbuilding_url" in sql_text
+    # COALESCE backfill in DO UPDATE SET (never wipe a prior value).
+    # SQL wraps long COALESCE calls; assert on a whitespace-normalised copy.
+    sql_norm = " ".join(sql_text.split())
+    assert (
+        "newbuilding_id = COALESCE( EXCLUDED.newbuilding_id, listings.newbuilding_id )" in sql_norm
+    )
+    assert (
+        "newbuilding_url = COALESCE( EXCLUDED.newbuilding_url, listings.newbuilding_url )"
+        in sql_norm
+    )
+
+
+def test_save_listings_newbuilding_fields_default_none():
+    """Lot без ЖК-полей → params newbuilding_id/url = None (backward compat)."""
+    db = _mock_db(inserted=True)
+    lot = _base_lot(source="avito", source_id="987654")
+
+    save_listings(db, [lot])
+
+    params = _get_execute_params(db)
+    assert params["newbuilding_id"] is None
+    assert params["newbuilding_url"] is None