diff --git a/tradein-mvp/backend/app/services/scrapers/avito.py b/tradein-mvp/backend/app/services/scrapers/avito.py index 38255dda..4edb6281 100644 --- a/tradein-mvp/backend/app/services/scrapers/avito.py +++ b/tradein-mvp/backend/app/services/scrapers/avito.py @@ -28,7 +28,7 @@ import logging import re from datetime import date, datetime, timedelta, timezone from typing import Any -from urllib.parse import urlencode, urljoin +from urllib.parse import urlencode, urljoin, urlparse, urlunparse from curl_cffi.requests import AsyncSession from selectolax.parser import HTMLParser @@ -750,6 +750,23 @@ class AvitoScraper(BaseScraper): dev_name = card.css_first('[data-marker="item-development-name"]') listing_segment = "novostroyki" if dev_name is not None else "vtorichka" + # Newbuilding (ЖК) link: новостройки несут якорь на ЖК-сабдомен + # . + # newbuilding_url — host+path без ?context (mirror urlparse-cleaning), + # newbuilding_id — = host без 'zhk-' префикса и '.avito.ru' суффикса. + newbuilding_id: str | None = None + newbuilding_url: str | None = None + if dev_name is not None: + nb_link = card.css_first('a[href*="zhk-"][href*=".avito.ru"]') + if nb_link is not None: + nb_href = nb_link.attributes.get("href", "") or "" + parsed = urlparse(nb_href) + host = parsed.netloc + if host.startswith("zhk-") and host.endswith(".avito.ru"): + # strip query/fragment, keep scheme+host+path + newbuilding_url = urlunparse((parsed.scheme, host, parsed.path, "", "", "")) + newbuilding_id = host[len("zhk-") : -len(".avito.ru")] + return ScrapedLot( source="avito", source_url=url, @@ -769,6 +786,8 @@ class AvitoScraper(BaseScraper): house_ext_id=house_ext_id, house_url=house_url, listing_segment=listing_segment, + newbuilding_id=newbuilding_id, + newbuilding_url=newbuilding_url, ) except Exception: # #823: логируем каждую сломанную карточку и считаем для observability. diff --git a/tradein-mvp/backend/app/services/scrapers/base.py b/tradein-mvp/backend/app/services/scrapers/base.py index 9fe05390..d0164f17 100644 --- a/tradein-mvp/backend/app/services/scrapers/base.py +++ b/tradein-mvp/backend/app/services/scrapers/base.py @@ -89,6 +89,10 @@ class ScrapedLot(BaseModel): house_url: str | None = None # full URL of /catalog/houses/... listing_segment: str | None = None # 'vtorichka' / 'novostroyki' + # Newbuilding (ЖК) linking — slug-id + canonical ЖК-subdomain URL + newbuilding_id: str | None = None # e.g. 'federatsiya-ekaterinburg' + newbuilding_url: str | None = None # e.g. https://zhk-federatsiya-ekaterinburg.avito.ru + # Цена (обязательно) price_rub: int = Field(gt=0) price_per_m2: int | None = None @@ -262,6 +266,7 @@ def save_listings( rooms, area_m2, floor, total_floors, year_built, house_type, repair_state, has_balcony, house_source, house_ext_id, house_url, listing_segment, + newbuilding_id, newbuilding_url, price_rub, price_per_m2, listing_date, publish_date, days_on_market, description, photo_urls, raw_payload, @@ -280,6 +285,7 @@ def save_listings( :rooms, :area_m2, :floor, :total_floors, :year_built, :house_type, :repair_state, :has_balcony, :house_source, :house_ext_id, :house_url, :listing_segment, + :newbuilding_id, :newbuilding_url, :price_rub, :ppm2, :listing_date, :publish_date, :days_on_market, :description, CAST(:photos AS jsonb), @@ -340,6 +346,12 @@ def save_listings( price_trend = COALESCE(EXCLUDED.price_trend, listings.price_trend), price_previous_rub = COALESCE( EXCLUDED.price_previous_rub, listings.price_previous_rub + ), + newbuilding_id = COALESCE( + EXCLUDED.newbuilding_id, listings.newbuilding_id + ), + newbuilding_url = COALESCE( + EXCLUDED.newbuilding_url, listings.newbuilding_url ) RETURNING id, (xmax = 0) AS inserted """ @@ -364,6 +376,8 @@ def save_listings( "house_ext_id": lot.house_ext_id, "house_url": lot.house_url, "listing_segment": lot.listing_segment, + "newbuilding_id": lot.newbuilding_id, + "newbuilding_url": lot.newbuilding_url, "price_rub": lot.price_rub, "ppm2": ppm2, "listing_date": lot.listing_date, diff --git a/tradein-mvp/backend/tests/test_avito_serp_date.py b/tradein-mvp/backend/tests/test_avito_serp_date.py index f6fa868f..71f2b546 100644 --- a/tradein-mvp/backend/tests/test_avito_serp_date.py +++ b/tradein-mvp/backend/tests/test_avito_serp_date.py @@ -68,14 +68,20 @@ def test_lazy_card_date_comes_from_json_not_dom() -> None: # (название ЖК/застройщика рендерится только у карточек новостроек). -def _serp_card(item_id: str, *, with_dev_name: bool) -> str: +def _serp_card(item_id: str, *, with_dev_name: bool, with_zhk_anchor: bool = False) -> str: dev = '
ЖК «Федерация»
' if with_dev_name else "" + zhk = ( + 'ЖК Федерация' + if with_zhk_anchor + else "" + ) return f"""
2-к. квартира, 50 м², 5/20 эт. {dev} + {zhk}
""" @@ -94,3 +100,36 @@ def test_segment_vtorichka_when_no_development_name() -> None: lots = AvitoScraper()._parse_html(html, "https://www.avito.ru/ekaterinburg/kvartiry") assert len(lots) == 1 assert lots[0].listing_segment == "vtorichka" + + +# ── newbuilding (ЖК) id/url из zhk-сабдомена ────────────────────────────────── + + +def test_newbuilding_captured_from_zhk_anchor() -> None: + """Новостройка с якорем zhk-*.avito.ru → newbuilding_url без ?context, id = slug.""" + card = _serp_card("8043003287", with_dev_name=True, with_zhk_anchor=True) + html = f"{card}" + lots = AvitoScraper()._parse_html(html, "https://www.avito.ru/ekaterinburg/kvartiry") + assert len(lots) == 1 + assert lots[0].newbuilding_url == "https://zhk-fed-ekaterinburg.avito.ru" + assert lots[0].newbuilding_id == "fed-ekaterinburg" + + +def test_newbuilding_none_when_novostroyka_without_anchor() -> None: + """Новостройка без zhk-якоря → оба поля None (сегмент по dev-name остаётся).""" + card = _serp_card("8043003287", with_dev_name=True, with_zhk_anchor=False) + html = f"{card}" + lots = AvitoScraper()._parse_html(html, "https://www.avito.ru/ekaterinburg/kvartiry") + assert len(lots) == 1 + assert lots[0].listing_segment == "novostroyki" + assert lots[0].newbuilding_id is None + assert lots[0].newbuilding_url is None + + +def test_newbuilding_none_for_vtorichka() -> None: + """Вторичка → newbuilding_id/url оба None.""" + html = f"{_serp_card('8043936560', with_dev_name=False)}" + lots = AvitoScraper()._parse_html(html, "https://www.avito.ru/ekaterinburg/kvartiry") + assert len(lots) == 1 + assert lots[0].newbuilding_id is None + assert lots[0].newbuilding_url is None diff --git a/tradein-mvp/backend/tests/test_base_save_listings.py b/tradein-mvp/backend/tests/test_base_save_listings.py index 1a38b7e8..816606e8 100644 --- a/tradein-mvp/backend/tests/test_base_save_listings.py +++ b/tradein-mvp/backend/tests/test_base_save_listings.py @@ -695,3 +695,49 @@ def test_save_listings_non_yandex_rich_cols_are_null(): assert params["predicted_price_max"] is None assert params["price_trend"] is None assert params["price_previous_rub"] is None + + +# ── newbuilding (ЖК) id/url persistence ────────────────────────────────────── + + +def test_save_listings_persists_newbuilding_fields(): + """Lot с newbuilding_id/url → оба попадают в params INSERT + есть в SQL.""" + db = _mock_db(inserted=True) + lot = _base_lot( + source="avito", + source_id="nb_test", + newbuilding_id="federatsiya-ekaterinburg", + newbuilding_url="https://zhk-federatsiya-ekaterinburg.avito.ru", + ) + + save_listings(db, [lot]) + + params = _get_execute_params(db) + assert params["newbuilding_id"] == "federatsiya-ekaterinburg" + assert params["newbuilding_url"] == "https://zhk-federatsiya-ekaterinburg.avito.ru" + + sql_text = _get_listings_insert_sql(db) + assert "newbuilding_id" in sql_text + assert "newbuilding_url" in sql_text + # COALESCE backfill in DO UPDATE SET (never wipe a prior value). + # SQL wraps long COALESCE calls; assert on a whitespace-normalised copy. + sql_norm = " ".join(sql_text.split()) + assert ( + "newbuilding_id = COALESCE( EXCLUDED.newbuilding_id, listings.newbuilding_id )" in sql_norm + ) + assert ( + "newbuilding_url = COALESCE( EXCLUDED.newbuilding_url, listings.newbuilding_url )" + in sql_norm + ) + + +def test_save_listings_newbuilding_fields_default_none(): + """Lot без ЖК-полей → params newbuilding_id/url = None (backward compat).""" + db = _mock_db(inserted=True) + lot = _base_lot(source="avito", source_id="987654") + + save_listings(db, [lot]) + + params = _get_execute_params(db) + assert params["newbuilding_id"] is None + assert params["newbuilding_url"] is None