refactor(tradein): drop dead schema — sale_type_text column + audit_log (#731) (#893)

Co-authored-by: bot-backend <bot-backend@gendsgn.local>
Co-committed-by: bot-backend <bot-backend@gendsgn.local>

tradein-mvp/backend/app/services/matching/conflict_resolution.py

@@ -6,6 +6,7 @@ Source-of-truth dicts read by merge logic in match_or_create_house/listing.
 Sources covered: avito (serp/detail/houses_catalog/domoteka/imv),
 cian (serp/bti/detail/stats/valuation), yandex (serp/detail/realty_nb/valuation).
 """
+
 from __future__ import annotations
 
 from typing import Any
@@ -18,7 +19,11 @@ HOUSE_FIELD_PRIORITY: dict[str, list[str] | str] = {
     "lat": ["cian_serp", "avito_houses_catalog", "yandex_realty_nb"],
     "lon": ["cian_serp", "avito_houses_catalog", "yandex_realty_nb"],
     "year_built": [
-        "cian_bti", "cian_serp", "avito_houses_catalog", "yandex_valuation", "yandex_realty_nb",
+        "cian_bti",
+        "cian_serp",
+        "avito_houses_catalog",
+        "yandex_valuation",
+        "yandex_realty_nb",
     ],
     "house_type": ["cian_bti", "cian_serp", "avito", "yandex_valuation", "yandex_realty_nb"],
     "series_name": ["cian_bti"],
@@ -34,7 +39,6 @@ HOUSE_FIELD_PRIORITY: dict[str, list[str] | str] = {
     "house_class": ["avito_houses_catalog", "cian", "yandex_realty_nb"],
     "rating_score": ["avito_houses_catalog", "cian", "yandex_realty_nb"],
     "reviews_count": ["avito_houses_catalog", "cian", "yandex_realty_nb"],
-
     # Yandex unique fields (newbuilding landing only)
     "text_reviews_count": ["yandex_realty_nb"],  # 353 text reviews — Yandex strongpoint
     "corpus_count": ["yandex_realty_nb"],  # "три башни" → 3
@@ -44,7 +48,6 @@ HOUSE_FIELD_PRIORITY: dict[str, list[str] | str] = {
     "developer_name": ["cian", "yandex_realty_nb"],
     "has_panorama": ["yandex_valuation"],  # Yandex 3D panorama flag
     "yandex_total_listings": ["yandex_valuation"],  # "N объектов" в истории
-
     # Yandex Valuation enrichment (existing house attrs)
     "has_lift": ["cian_bti", "cian_detail", "yandex_valuation"],
     "ceiling_height": ["cian_detail", "yandex_valuation"],
@@ -77,39 +80,30 @@ LISTING_FIELD_PRIORITY: dict[str, list[str] | str] = {
     "phones": ["cian_serp"],
     "description": ["cian_serp", "avito_detail", "yandex_detail"],
     "photo_urls": "union",
-
     # Avito Domoteka unique
     "owners_count": ["avito_domoteka"],
     "owners_at_least": ["avito_domoteka"],
     "last_owner_change_date": ["avito_domoteka"],
     "encumbrances_clean": ["avito_domoteka"],
     "registry_match": ["avito_domoteka"],
-
     # Cian-only
     "is_rosreestr_checked": ["cian_serp"],
     "is_layout_approved": ["cian_serp"],
     "is_commercial_ownership_verified": ["cian_serp"],
-
     # Cross-validation
     "price_rub": "cross_validate",
     "kadastr_num": "first_non_null",
     # NOTE: task description claims price_rub=['cian','avito','yandex']; vault sec 4.5
     # says 'cross_validate' (flag if diff > 10%). Following vault — change to list if
     # cross_validate semantics aren't desired at caller.
-
     # Yandex unique (agency block — OfferCardAuthorInfo)
     "agency_name": ["yandex_detail"],
     "agency_founded_year": ["yandex_detail"],
     "agency_objects_count": ["yandex_detail"],
-
     # Yandex parallel views column (NOT existing `views_total` which is Cian's)
     "views_total_yandex": ["yandex_detail"],
-
     # Yandex raw publish-date text (relative form)
     "publish_date_relative": ["yandex_detail"],
-
-    # Yandex raw RU sale-type phrase (vs existing `sale_type` enum)
-    "sale_type_text":        ["yandex_detail"],
 }
 
 
@@ -129,7 +123,7 @@ def _resolve(
             for v in candidates.values():
                 if v is None:
                     continue
-                if isinstance(v, (list, tuple, set)):
+                if isinstance(v, list | tuple | set):
                     out.extend(v)
                 else:
                     out.append(v)
@@ -156,7 +150,7 @@ def _resolve(
             return min(non_null) if non_null else None
         if rule == "cross_validate":
             # Caller decides — return median by default
-            nums = [v for v in candidates.values() if isinstance(v, (int, float))]
+            nums = [v for v in candidates.values() if isinstance(v, int | float)]
             if not nums:
                 return None
             nums.sort()
@@ -186,6 +180,7 @@ def resolve_listing_field(field: str, candidates: dict[str, Any]) -> Any:
 # Legacy stub — kept for backward compat with existing __init__.py and tests
 # ---------------------------------------------------------------------------
 
+
 def update_canonical_fields(
     db: Any,
     listing_id: int,

tradein-mvp/backend/app/services/scrapers/yandex_detail.py

@@ -64,9 +64,6 @@ class DetailEnrichment(BaseModel):
     # Description (full text)
     description: str | None = None
 
-    # Sale type — raw RU phrase ('свободная продажа' / 'альтернативная')
-    sale_type_text: str | None = None
-
     # Repair state — enum, inferred from description text (#622).
     # Yandex не отдаёт структурного поля ремонта, поэтому только инференс.
     repair_state: str | None = None
@@ -116,9 +113,7 @@ class YandexDetailScraper(BaseScraper):
         self.request_delay_sec = get_scraper_delay(self.name)
 
     # BaseScraper requires fetch_around — detail isn't geo-based, raise NotImplementedError
-    async def fetch_around(
-        self, lat: float, lon: float, radius_m: int = 1000
-    ) -> list:  # type: ignore[override]
+    async def fetch_around(self, lat: float, lon: float, radius_m: int = 1000) -> list:  # type: ignore[override]
         raise NotImplementedError(
             "YandexDetailScraper is offer-id-based; use fetch_detail(offer_url) instead."
         )
@@ -130,9 +125,7 @@ class YandexDetailScraper(BaseScraper):
             logger.exception("yandex detail fetch failed: %s", offer_url)
             return None
         if response.status_code != 200:
-            logger.warning(
-                "yandex detail returned %d for %s", response.status_code, offer_url
-            )
+            logger.warning("yandex detail returned %d for %s", response.status_code, offer_url)
             return None
         result = self.parse(response.text, offer_url=offer_url)
         await self.sleep_between_requests()
@@ -174,13 +167,6 @@ class YandexDetailScraper(BaseScraper):
         summary_node = tree.css_first('[data-test="OfferCardSummary"]')
         summary_text = summary_node.text(strip=True) if summary_node else ""
 
-        # Sale type — raw RU phrase
-        sale_type_text: str | None = None
-        for phrase in ("свободная продажа", "альтернативная"):
-            if phrase in summary_text.lower():
-                sale_type_text = phrase
-                break
-
         # Views + relative publish date from summary text
         views_match = RE_VIEWS.search(summary_text)
         views_total = int(views_match.group(1)) if views_match else None
@@ -235,9 +221,7 @@ class YandexDetailScraper(BaseScraper):
             if nb_match:
                 nb_id = nb_match.group(1)
                 nb_url = (
-                    nb_href
-                    if nb_href.startswith("http")
-                    else f"https://realty.yandex.ru{nb_href}"
+                    nb_href if nb_href.startswith("http") else f"https://realty.yandex.ru{nb_href}"
                 )
 
         # --- NLP best-effort from description ---
@@ -260,7 +244,6 @@ class YandexDetailScraper(BaseScraper):
             total_floors=total_floors,
             address=address,
             description=description,
-            sale_type_text=sale_type_text,
             repair_state=repair_state,
             views_total=views_total,
             publish_date=publish_date,

tradein-mvp/backend/data/sql/095_dead_schema.sql

@@ -0,0 +1,24 @@
+-- 095_dead_schema.sql
+-- Purpose: Удаление мёртвой схемы (верифицированный безопасный подмножество аудита TradeinDb_Audit_May30).
+--
+-- Контекст (Issue #731):
+--   sale_type_text: столбец 100% NULL в listings, code refs удалены в том же PR.
+--   audit_log: 0 строк, 0 python refs, триггеры не зависят (верифицировано).
+--
+-- Idempotency:
+--   ALTER TABLE ... DROP COLUMN IF EXISTS — безопасен при повторном запуске.
+--   DROP TABLE IF EXISTS — безопасен при повторном запуске.
+--
+-- Явно НЕ удаляем: houses_price_dynamics, agents, sellers, estimate_photos,
+--                  house_reviews, listings_snapshots.
+
+BEGIN;
+
+-- #731: drop dead schema (verified-safe subset of TradeinDb_Audit_May30)
+-- sale_type_text: 100% NULL in listings, code refs removed in same PR
+ALTER TABLE IF EXISTS listings DROP COLUMN IF EXISTS sale_type_text;
+
+-- audit_log: 0 rows, 0 python refs, no triggers depend on it (verified)
+DROP TABLE IF EXISTS audit_log;
+
+COMMIT;

tradein-mvp/backend/tests/matching/test_conflict_resolution.py

@@ -1,4 +1,5 @@
 """Per-field priority resolution tests."""
+
 from app.services.matching.conflict_resolution import (
     HOUSE_FIELD_PRIORITY,
     LISTING_FIELD_PRIORITY,
@@ -10,6 +11,7 @@ from app.services.matching.conflict_resolution import (
 # Pre-existing tests (preserved)
 # ---------------------------------------------------------------------------
 
+
 def test_house_year_built_prefers_cian_bti() -> None:
     out = resolve_house_field(
         "year_built",
@@ -29,7 +31,8 @@ def test_listing_owners_count_avito_domoteka_only() -> None:
 
 def test_listing_photo_urls_union() -> None:
     out = resolve_listing_field(
-        "photo_urls", {"avito": ["a.jpg", "b.jpg"], "cian": ["b.jpg", "c.jpg"]},
+        "photo_urls",
+        {"avito": ["a.jpg", "b.jpg"], "cian": ["b.jpg", "c.jpg"]},
     )
     assert set(out) == {"a.jpg", "b.jpg", "c.jpg"}
 
@@ -51,15 +54,14 @@ def test_listing_priority_dict_has_owners() -> None:
 # Yandex house priority tests
 # ---------------------------------------------------------------------------
 
+
 class TestYandexHousePriority:
     def test_house_lat_yandex_realty_nb_when_cian_missing(self) -> None:
         out = resolve_house_field("lat", {"yandex_realty_nb": 56.85})
         assert out == 56.85
 
     def test_house_lat_cian_preferred_over_yandex(self) -> None:
-        out = resolve_house_field(
-            "lat", {"cian_serp": 56.83, "yandex_realty_nb": 56.85}
-        )
+        out = resolve_house_field("lat", {"cian_serp": 56.83, "yandex_realty_nb": 56.85})
         assert out == 56.83
 
     def test_house_year_built_yandex_valuation_picked(self) -> None:
@@ -67,9 +69,7 @@ class TestYandexHousePriority:
         assert out == 1981
 
     def test_house_year_built_cian_bti_preferred(self) -> None:
-        out = resolve_house_field(
-            "year_built", {"cian_bti": 1980, "yandex_valuation": 1981}
-        )
+        out = resolve_house_field("year_built", {"cian_bti": 1980, "yandex_valuation": 1981})
         assert out == 1980
 
     def test_house_text_reviews_count_yandex_only(self) -> None:
@@ -81,9 +81,7 @@ class TestYandexHousePriority:
         assert out == 3
 
     def test_house_commission_year_cian_serp_preferred(self) -> None:
-        out = resolve_house_field(
-            "commission_year", {"cian_serp": 2022, "yandex_realty_nb": 2023}
-        )
+        out = resolve_house_field("commission_year", {"cian_serp": 2022, "yandex_realty_nb": 2023})
         assert out == 2022
 
     def test_house_commission_month_yandex_only(self) -> None:
@@ -98,15 +96,11 @@ class TestYandexHousePriority:
         assert out == "PRINZIP"
 
     def test_house_has_lift_cian_bti_preferred_over_yandex_valuation(self) -> None:
-        out = resolve_house_field(
-            "has_lift", {"cian_bti": True, "yandex_valuation": True}
-        )
+        out = resolve_house_field("has_lift", {"cian_bti": True, "yandex_valuation": True})
         assert out is True
 
     def test_house_ceiling_height_cian_detail_preferred(self) -> None:
-        out = resolve_house_field(
-            "ceiling_height", {"cian_detail": 2.7, "yandex_valuation": 2.5}
-        )
+        out = resolve_house_field("ceiling_height", {"cian_detail": 2.7, "yandex_valuation": 2.5})
         assert out == 2.7
 
     def test_house_has_panorama_yandex_valuation_only(self) -> None:
@@ -141,6 +135,7 @@ class TestYandexHousePriority:
 # Yandex listing priority tests
 # ---------------------------------------------------------------------------
 
+
 class TestYandexListingPriority:
     def test_listing_description_cian_preferred_over_yandex_detail(self) -> None:
         out = resolve_listing_field(
@@ -154,9 +149,7 @@ class TestYandexListingPriority:
         assert out == "brick"
 
     def test_listing_agency_name_yandex_only(self) -> None:
-        out = resolve_listing_field(
-            "agency_name", {"yandex_detail": "Агентство «Диал»"}
-        )
+        out = resolve_listing_field("agency_name", {"yandex_detail": "Агентство «Диал»"})
         assert out == "Агентство «Диал»"
 
     def test_listing_agency_founded_year_yandex_only(self) -> None:
@@ -172,17 +165,9 @@ class TestYandexListingPriority:
         assert out == 874
 
     def test_listing_publish_date_relative_yandex_only(self) -> None:
-        out = resolve_listing_field(
-            "publish_date_relative", {"yandex_detail": "3 дня назад"}
-        )
+        out = resolve_listing_field("publish_date_relative", {"yandex_detail": "3 дня назад"})
         assert out == "3 дня назад"
 
-    def test_listing_sale_type_text_yandex_only(self) -> None:
-        out = resolve_listing_field(
-            "sale_type_text", {"yandex_detail": "Прямая продажа"}
-        )
-        assert out == "Прямая продажа"
-
     def test_listing_agency_name_none_when_no_yandex(self) -> None:
         out = resolve_listing_field("agency_name", {"cian_serp": None})
         assert out is None
@@ -209,7 +194,6 @@ class TestYandexListingPriority:
             "agency_objects_count",
             "views_total_yandex",
             "publish_date_relative",
-            "sale_type_text",
         ]
         for key in yandex_keys:
             assert key in LISTING_FIELD_PRIORITY, f"{key!r} missing from LISTING_FIELD_PRIORITY"

tradein-mvp/backend/tests/test_yandex_detail.py

@@ -60,9 +60,7 @@ def _make_html(
 ) -> str:
     ld_block = ""
     if product_ld is not None:
-        ld_block = (
-            f'<script type="application/ld+json">{json.dumps(product_ld)}</script>'
-        )
+        ld_block = f'<script type="application/ld+json">{json.dumps(product_ld)}</script>'
 
     nb_link = ""
     if nb_href:
@@ -128,11 +126,6 @@ class TestFullDetailFixture:
         assert result.views_total == 342
         assert result.publish_date == date(2026, 4, 10)
 
-    def test_sale_type_free(self) -> None:
-        result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
-        assert result is not None
-        assert result.sale_type_text == "свободная продажа"
-
     def test_price_per_m2_from_summary(self) -> None:
         result = SCRAPER.parse(FULL_HTML, offer_url=_BASE_OFFER_URL)
         assert result is not None
@@ -196,7 +189,6 @@ class TestStudio:
         assert result.area_m2 == pytest.approx(28.0)
         assert result.floor == 2
         assert result.total_floors == 9
-        assert result.sale_type_text == "альтернативная"
 
 
 class TestNoAgencySection:
@@ -221,10 +213,7 @@ class TestAgencyBlock:
           <h2>АН Городской риелтор</h2>
           <p>Год основания 1998. Продали 150 объектов.</p>
         </div>"""
-        summary = (
-            "Иван Петров АН Городской риелтор • 50 просмотров"
-            " • опубликовано 3 марта 2025"
-        )
+        summary = "Иван Петров АН Городской риелтор • 50 просмотров" " • опубликовано 3 марта 2025"
         html = _make_html(author_block=author_html, summary_text=summary)
         result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
         assert result is not None
@@ -291,9 +280,7 @@ class TestMetroStations:
 
 class TestRelativeDate:
     def test_parse_relative_date_yesterday(self) -> None:
-        html = _make_html(
-            summary_text="Россия, ЕКБ, ул. Тест • 100 просмотров • вчера"
-        )
+        html = _make_html(summary_text="Россия, ЕКБ, ул. Тест • 100 просмотров • вчера")
         result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
         assert result is not None
         assert result.publish_date_relative == "вчера"
@@ -313,9 +300,7 @@ class TestRelativeDate:
 
 class TestPublishDate:
     def test_publish_date_ru_format(self) -> None:
-        html = _make_html(
-            summary_text="Россия, ЕКБ • 200 просмотров • опубликовано 5 января 2026"
-        )
+        html = _make_html(summary_text="Россия, ЕКБ • 200 просмотров • опубликовано 5 января 2026")
         result = SCRAPER.parse(html, offer_url=_BASE_OFFER_URL)
         assert result is not None
         from datetime import date