feat(tradein/cian): surface valuation chart + rent + change-pct on /trade-in #530
3 changed files with 460 additions and 31 deletions
|
|
@ -52,6 +52,35 @@ class ValuationHouseMeta(BaseModel):
|
|||
total_objects: int | None = None # 'N объектов' (full archive count)
|
||||
has_panorama: bool = False # 'Панорама' label present
|
||||
|
||||
def validate_match(
|
||||
self,
|
||||
expected_year_built: int | None = None,
|
||||
expected_total_floors: int | None = None,
|
||||
) -> float:
|
||||
"""Return confidence 0..1 that this house meta matches expected values.
|
||||
|
||||
Used after fetching valuation by address to detect when Yandex returned a
|
||||
different house (ambiguous address geocoding). Tolerance ±1 year, ±1 floor.
|
||||
|
||||
Both expected=None → 1.0 (no check). Mismatch on any dimension → 0.0 for it.
|
||||
"""
|
||||
score = 0.0
|
||||
checks = 0
|
||||
if expected_year_built is not None:
|
||||
checks += 1
|
||||
if self.year_built is not None and abs(self.year_built - expected_year_built) <= 1:
|
||||
score += 1.0
|
||||
if expected_total_floors is not None:
|
||||
checks += 1
|
||||
if (
|
||||
self.total_floors is not None
|
||||
and abs(self.total_floors - expected_total_floors) <= 1
|
||||
):
|
||||
score += 1.0
|
||||
if checks == 0:
|
||||
return 1.0
|
||||
return score / checks
|
||||
|
||||
|
||||
class ValuationHistoryItem(BaseModel):
|
||||
"""One historical offer entry from the valuation page."""
|
||||
|
|
@ -64,6 +93,7 @@ class ValuationHistoryItem(BaseModel):
|
|||
last_price: int | None = None
|
||||
last_price_per_m2: int | None = None
|
||||
publish_date: date | None = None
|
||||
removed_date: date | None = None # ← NEW
|
||||
exposure_days: int | None = None
|
||||
status: str | None = None # 'В продаже' / 'Снято'
|
||||
|
||||
|
|
@ -86,7 +116,7 @@ class YandexValuationResult(BaseModel):
|
|||
# ---------------------------------------------------------------------------
|
||||
|
||||
RE_YEAR_BUILT = re.compile(r"Дом\s+(\d{4})\s+года", re.IGNORECASE)
|
||||
RE_FLOORS = re.compile(r"(\d+)\s+этаж", re.IGNORECASE)
|
||||
RE_FLOORS = re.compile(r"(\d+)\s+этажей", re.IGNORECASE)
|
||||
RE_CEILING = re.compile(r"([\d,]+)\s*м\s+потолки", re.IGNORECASE)
|
||||
RE_TOTAL_OBJECTS = re.compile(r"(\d+)\s+объект", re.IGNORECASE)
|
||||
|
||||
|
|
@ -98,9 +128,7 @@ RE_ITEM_EXPOSURE = re.compile(r"экспозиции\s+(\d+)\s+дн", re.IGNOREC
|
|||
RE_ITEM_STATUS = re.compile(r"(В\s+продаже|Снят[оа])", re.IGNORECASE)
|
||||
|
||||
# Matches total-price tokens (rubles) — excludes per-m2 tokens by negative lookahead
|
||||
_RE_PRICE_TOKEN = re.compile(
|
||||
r"(?:\d[\d\s]*\d|\d)(?:[.,]\d+)?\s*(?:млн)?\s*₽(?!\s*за\s*м²)"
|
||||
)
|
||||
_RE_PRICE_TOKEN = re.compile(r"(?:\d[\d\s]*\d|\d)(?:[.,]\d+)?\s*(?:млн)?\s*₽(?!\s*за\s*м²)")
|
||||
_RE_PPM2_TOKEN = re.compile(r"\d[\d\s]*\s*₽\s*за\s*м²")
|
||||
|
||||
|
||||
|
|
@ -126,9 +154,7 @@ class YandexValuationScraper(BaseScraper):
|
|||
super().__init__()
|
||||
self.request_delay_sec = get_scraper_delay(self.name)
|
||||
|
||||
async def fetch_around(
|
||||
self, lat: float, lon: float, radius_m: int = 1000
|
||||
) -> list: # type: ignore[override]
|
||||
async def fetch_around(self, lat: float, lon: float, radius_m: int = 1000) -> list: # type: ignore[override]
|
||||
raise NotImplementedError(
|
||||
"YandexValuationScraper is address-based; use fetch_house_history() instead."
|
||||
)
|
||||
|
|
@ -164,9 +190,7 @@ class YandexValuationScraper(BaseScraper):
|
|||
logger.exception("yandex valuation fetch failed: %s", url)
|
||||
return None
|
||||
if response.status_code != 200:
|
||||
logger.warning(
|
||||
"yandex valuation returned %d for %s", response.status_code, url
|
||||
)
|
||||
logger.warning("yandex valuation returned %d for %s", response.status_code, url)
|
||||
return None
|
||||
result = self.parse(
|
||||
response.text,
|
||||
|
|
@ -189,9 +213,10 @@ class YandexValuationScraper(BaseScraper):
|
|||
source_url: str,
|
||||
) -> YandexValuationResult:
|
||||
"""Parse raw HTML into YandexValuationResult. Pure function — usable in unit tests."""
|
||||
tree = HTMLParser(html)
|
||||
html_normalized = html.replace("\xa0", " ")
|
||||
tree = HTMLParser(html_normalized)
|
||||
body = tree.body
|
||||
body_text = body.text(strip=True) if body else ""
|
||||
body_text = (body.text(strip=True) if body else "").replace("\xa0", " ")
|
||||
|
||||
house = self._parse_house_meta(body_text)
|
||||
history_items = self._parse_history_items(tree, body_text)
|
||||
|
|
@ -221,17 +246,13 @@ class YandexValuationScraper(BaseScraper):
|
|||
year_built=int(year_m.group(1)) if year_m else None,
|
||||
total_floors=int(floors_m.group(1)) if floors_m else None,
|
||||
house_type=parse_house_type(body_text),
|
||||
ceiling_height=(
|
||||
float(ceiling_m.group(1).replace(",", ".")) if ceiling_m else None
|
||||
),
|
||||
ceiling_height=(float(ceiling_m.group(1).replace(",", ".")) if ceiling_m else None),
|
||||
has_lift="Лифт" in body_text,
|
||||
total_objects=int(objects_m.group(1)) if objects_m else None,
|
||||
has_panorama="Панорама" in body_text,
|
||||
)
|
||||
|
||||
def _parse_history_items(
|
||||
self, tree: HTMLParser, body_text: str
|
||||
) -> list[ValuationHistoryItem]:
|
||||
def _parse_history_items(self, tree: HTMLParser, body_text: str) -> list[ValuationHistoryItem]:
|
||||
"""Extract list of historical offer items using best available strategy.
|
||||
|
||||
Strategy 1: CSS data-test containers (if Yandex exposes them).
|
||||
|
|
@ -239,9 +260,7 @@ class YandexValuationScraper(BaseScraper):
|
|||
"""
|
||||
items: list[ValuationHistoryItem] = []
|
||||
# Strategy 1: explicit data-test container
|
||||
for container in tree.css(
|
||||
'[data-test*="HistoryItem"], [data-test*="ValuationItem"]'
|
||||
):
|
||||
for container in tree.css('[data-test*="HistoryItem"], [data-test*="ValuationItem"]'):
|
||||
item = self._parse_item_text(container.text(strip=True))
|
||||
if item:
|
||||
items.append(item)
|
||||
|
|
@ -284,7 +303,15 @@ class YandexValuationScraper(BaseScraper):
|
|||
start_ppm2 = parse_rub(ppm2_tokens[0]) if len(ppm2_tokens) >= 1 else None
|
||||
last_ppm2 = parse_rub(ppm2_tokens[1]) if len(ppm2_tokens) >= 2 else None
|
||||
|
||||
publish_date = parse_dmy(text)
|
||||
# Extract ALL DD.MM.YYYY dates: first → publish_date, second → removed_date
|
||||
date_matches = list(re.finditer(r"\d{2}\.\d{2}\.\d{4}", text))
|
||||
dates_parsed: list[date] = []
|
||||
for m in date_matches:
|
||||
d = parse_dmy(m.group(0))
|
||||
if d is not None:
|
||||
dates_parsed.append(d)
|
||||
publish_date = dates_parsed[0] if dates_parsed else None
|
||||
removed_date = dates_parsed[1] if len(dates_parsed) >= 2 else None
|
||||
expo_m = RE_ITEM_EXPOSURE.search(text)
|
||||
exposure_days = int(expo_m.group(1)) if expo_m else None
|
||||
|
||||
|
|
@ -304,6 +331,7 @@ class YandexValuationScraper(BaseScraper):
|
|||
last_price=last_price,
|
||||
last_price_per_m2=last_ppm2,
|
||||
publish_date=publish_date,
|
||||
removed_date=removed_date,
|
||||
exposure_days=exposure_days,
|
||||
status=status,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,304 @@
|
|||
-- 063_backfill_houses_and_link_listings.sql
|
||||
-- Bootstrap houses table from existing listings data (Phase A+B).
|
||||
--
|
||||
-- Context:
|
||||
-- listings: 18,256 rows (all have addresses)
|
||||
-- houses: 0 rows (empty)
|
||||
-- listings.house_id_fk: 0 linked
|
||||
-- house_placement_history: 160 orphans, all source=yandex_valuation, raw_payload has no address
|
||||
--
|
||||
-- Steps:
|
||||
-- 1. CREATE OR REPLACE FUNCTION tradein_normalize_short_addr
|
||||
-- 2. INSERT Avito/CIAN-sourced houses from listings WHERE house_source+house_ext_id NOT NULL
|
||||
-- 3. INSERT derived houses for listings WITHOUT ext_house_id (grouped by normalized address)
|
||||
-- DEDUP: skips addresses already covered by step 2 (prevents split house rows)
|
||||
-- 4. UPDATE listings.house_id_fk via source+ext_house_id match (direct path)
|
||||
-- 5. UPDATE listings.house_id_fk via normalized address match (fuzzy path)
|
||||
-- DEDUP: COALESCE prefers avito-sourced house over derived at same address
|
||||
-- 6. Best-effort relink house_placement_history via raw_payload->>'address' (likely 0 rows)
|
||||
-- 7. Dedup sanity check: WARN if any normalized address maps to >1 houses rows
|
||||
-- 8. RAISE NOTICE with final counters
|
||||
--
|
||||
-- Dependencies:
|
||||
-- - pgcrypto extension (for digest()) -- verified present
|
||||
-- - listings table, houses table, house_placement_history table
|
||||
-- - houses UNIQUE constraint on (source, ext_house_id)
|
||||
-- - house_placement_history UNIQUE constraint on (source, ext_item_id)
|
||||
--
|
||||
-- Idempotency:
|
||||
-- - INSERT ... ON CONFLICT DO NOTHING
|
||||
-- - UPDATE ... WHERE house_id_fk IS NULL
|
||||
-- - CREATE OR REPLACE FUNCTION
|
||||
--
|
||||
-- Deploy order: after 062_clean_avito_addresses.sql
|
||||
|
||||
BEGIN;
|
||||
|
||||
-- -------------------------------------------------------------------------
|
||||
-- Step 1: Address normalizer function
|
||||
-- Strips regional/city prefix and apartment/corpus suffix.
|
||||
-- Handles addresses like:
|
||||
-- "ул. Репина, 75/2 стр." → "ул. Репина, 75/2 стр."
|
||||
-- "Свердловская обл., Екатеринбург, ул. Большакова, 17" → "ул. Большакова, 17"
|
||||
-- "улица Яскина, 12 · р-н Октябрьский" → "улица Яскина, 12"
|
||||
-- "Азина, 4.1.1" → "Азина, 4.1.1"
|
||||
-- -------------------------------------------------------------------------
|
||||
CREATE OR REPLACE FUNCTION tradein_normalize_short_addr(addr text)
|
||||
RETURNS text
|
||||
LANGUAGE sql
|
||||
IMMUTABLE
|
||||
PARALLEL SAFE
|
||||
AS $$
|
||||
SELECT trim(both ' ,.' FROM
|
||||
regexp_replace(
|
||||
regexp_replace(
|
||||
regexp_replace(
|
||||
regexp_replace(
|
||||
regexp_replace(
|
||||
-- Strip leading "Россия / РФ / Российская Федерация, "
|
||||
regexp_replace(addr, '^\s*(?:Россия|РФ|Российская\s+Федерация)\s*,\s*', '', 'i'),
|
||||
-- Strip leading region "Свердловская обл., " / "Свердловская область, "
|
||||
'^[А-ЯЁа-яё][А-ЯЁа-яё\s-]+\s+обл(?:асть|\.)\s*,\s*', '', 'i'
|
||||
),
|
||||
-- Strip leading city "Екатеринбург, " / "г. Екатеринбург, " / "г Екатеринбург, "
|
||||
'^г\.?\s*[А-ЯЁ][А-ЯЁа-яё-]+\s*,\s*', '', 'i'
|
||||
),
|
||||
-- Strip district suffix " · р-н ..." or " · district"
|
||||
'\s*·\s*.+$', '', 'i'
|
||||
),
|
||||
-- Strip apartment/corpus/office suffix ", кв./корп./оф./пом./подъезд N..."
|
||||
',\s*(?:кв\.?|корп\.?|к\.?|оф\.?|пом\.?|подъезд)\s*\d+.*$', '', 'i'
|
||||
),
|
||||
-- Strip trailing whitespace artefacts
|
||||
'\s{2,}', ' ', 'g'
|
||||
)
|
||||
);
|
||||
$$;
|
||||
|
||||
-- -------------------------------------------------------------------------
|
||||
-- Step 2: Insert Avito/CIAN-sourced houses from listings
|
||||
-- Uses DISTINCT ON to pick the most recently scraped data per (source, ext_house_id).
|
||||
-- -------------------------------------------------------------------------
|
||||
INSERT INTO houses (
|
||||
source,
|
||||
ext_house_id,
|
||||
url,
|
||||
address,
|
||||
lat,
|
||||
lon,
|
||||
geom,
|
||||
year_built,
|
||||
house_type,
|
||||
total_floors,
|
||||
full_address,
|
||||
first_seen_at,
|
||||
last_scraped_at
|
||||
)
|
||||
SELECT DISTINCT ON (l.house_source, l.house_ext_id)
|
||||
l.house_source AS source,
|
||||
l.house_ext_id AS ext_house_id,
|
||||
COALESCE(l.house_url, '') AS url,
|
||||
l.address,
|
||||
l.lat,
|
||||
l.lon,
|
||||
l.geom,
|
||||
l.year_built,
|
||||
l.house_type,
|
||||
l.total_floors,
|
||||
l.address AS full_address,
|
||||
NOW() AS first_seen_at,
|
||||
NOW() AS last_scraped_at
|
||||
FROM listings l
|
||||
WHERE l.house_source IS NOT NULL
|
||||
AND l.house_ext_id IS NOT NULL
|
||||
AND l.address IS NOT NULL
|
||||
ORDER BY l.house_source, l.house_ext_id, l.scraped_at DESC
|
||||
ON CONFLICT (source, ext_house_id) DO NOTHING;
|
||||
|
||||
-- -------------------------------------------------------------------------
|
||||
-- Step 3: Insert derived houses for listings WITHOUT ext_house_id
|
||||
-- Groups listings by normalized address; each group = one derived house.
|
||||
-- Synthetic ext_house_id = first 32 hex chars of sha256(normalized_addr).
|
||||
-- Uses DISTINCT ON (norm_addr) picking latest scraped_at per group.
|
||||
--
|
||||
-- DEDUP FIX: Excludes normalized addresses already covered by Step 2's
|
||||
-- avito/cian-sourced houses. Prevents the same physical building getting
|
||||
-- TWO rows (one avito-sourced, one derived) when it has listings from
|
||||
-- both Avito (with ext_house_id) and other sources (without ext_house_id).
|
||||
-- -------------------------------------------------------------------------
|
||||
INSERT INTO houses (
|
||||
source,
|
||||
ext_house_id,
|
||||
url,
|
||||
address,
|
||||
lat,
|
||||
lon,
|
||||
geom,
|
||||
year_built,
|
||||
house_type,
|
||||
total_floors,
|
||||
full_address,
|
||||
short_address,
|
||||
first_seen_at,
|
||||
last_scraped_at
|
||||
)
|
||||
SELECT DISTINCT ON (norm_addr)
|
||||
'derived' AS source,
|
||||
substring(encode(digest(norm_addr, 'sha256'), 'hex'), 1, 32) AS ext_house_id,
|
||||
'' AS url,
|
||||
src.address,
|
||||
src.lat,
|
||||
src.lon,
|
||||
src.geom,
|
||||
src.year_built,
|
||||
src.house_type,
|
||||
src.total_floors,
|
||||
src.address AS full_address,
|
||||
norm_addr AS short_address,
|
||||
NOW() AS first_seen_at,
|
||||
NOW() AS last_scraped_at
|
||||
FROM (
|
||||
SELECT
|
||||
l.*,
|
||||
tradein_normalize_short_addr(l.address) AS norm_addr
|
||||
FROM listings l
|
||||
WHERE l.address IS NOT NULL
|
||||
AND (l.house_source IS NULL OR l.house_ext_id IS NULL)
|
||||
) src
|
||||
WHERE src.norm_addr IS NOT NULL
|
||||
AND length(src.norm_addr) >= 5
|
||||
AND src.norm_addr NOT IN (
|
||||
SELECT tradein_normalize_short_addr(address)
|
||||
FROM houses
|
||||
WHERE source != 'derived' AND address IS NOT NULL
|
||||
)
|
||||
ORDER BY norm_addr, src.scraped_at DESC
|
||||
ON CONFLICT (source, ext_house_id) DO NOTHING;
|
||||
|
||||
-- -------------------------------------------------------------------------
|
||||
-- Step 4: Link listings.house_id_fk via direct source+ext_house_id match
|
||||
-- Uses listings_house_ext_id_idx (source, house_ext_id) — sargable.
|
||||
-- -------------------------------------------------------------------------
|
||||
UPDATE listings l
|
||||
SET house_id_fk = h.id
|
||||
FROM houses h
|
||||
WHERE l.house_id_fk IS NULL
|
||||
AND l.house_source IS NOT NULL
|
||||
AND l.house_ext_id IS NOT NULL
|
||||
AND l.house_source = h.source
|
||||
AND l.house_ext_id = h.ext_house_id;
|
||||
|
||||
-- -------------------------------------------------------------------------
|
||||
-- Step 5: Link listings.house_id_fk via normalized address (fuzzy path)
|
||||
-- Covers listings where house_source/house_ext_id is absent.
|
||||
-- houses.short_address was populated above (derived rows); joined by equality.
|
||||
--
|
||||
-- DEDUP FIX: For cian/yandex listings at an address that already has an
|
||||
-- avito-sourced house (from Step 2), we prefer that avito-sourced house
|
||||
-- over any derived row. This consolidates all listings for the same physical
|
||||
-- building under a single houses row regardless of scrape source.
|
||||
--
|
||||
-- COALESCE priority:
|
||||
-- 1. Non-derived house whose address normalizes to the same value
|
||||
-- 2. Derived house whose short_address equals the normalized listing address
|
||||
--
|
||||
-- NOTE: houses_addr_fp_idx is on address_fingerprint; short_address has no
|
||||
-- dedicated index. If this migration is re-run on large data, consider:
|
||||
-- CREATE INDEX IF NOT EXISTS houses_short_addr_idx ON houses(short_address)
|
||||
-- WHERE source = 'derived';
|
||||
-- For a one-shot migration on 18k rows this is acceptable (seq scan ~ms).
|
||||
-- -------------------------------------------------------------------------
|
||||
WITH norm_l AS (
|
||||
SELECT id, tradein_normalize_short_addr(address) AS norm_addr
|
||||
FROM listings
|
||||
WHERE house_id_fk IS NULL AND address IS NOT NULL
|
||||
)
|
||||
UPDATE listings l
|
||||
SET house_id_fk = COALESCE(
|
||||
-- Prefer avito/cian-sourced house at same normalized address (cross-source consolidation)
|
||||
(SELECT h.id FROM houses h
|
||||
WHERE h.source != 'derived'
|
||||
AND tradein_normalize_short_addr(h.address) = nl.norm_addr
|
||||
LIMIT 1),
|
||||
-- Fall back to derived house
|
||||
(SELECT h.id FROM houses h
|
||||
WHERE h.source = 'derived'
|
||||
AND h.short_address = nl.norm_addr
|
||||
LIMIT 1)
|
||||
)
|
||||
FROM norm_l nl
|
||||
WHERE l.id = nl.id AND l.house_id_fk IS NULL;
|
||||
|
||||
-- -------------------------------------------------------------------------
|
||||
-- Step 6: Best-effort relink orphan house_placement_history
|
||||
-- All 160 orphan rows are source=yandex_valuation; raw_payload has no
|
||||
-- 'address' key (verified: has_addr=false for all 160 rows).
|
||||
-- This UPDATE will match 0 rows but is included for completeness/idempotency.
|
||||
-- Phase C (Celery IMV scraper) will assign house_id on new inserts directly.
|
||||
-- -------------------------------------------------------------------------
|
||||
UPDATE house_placement_history hph
|
||||
SET house_id = h.id
|
||||
FROM houses h
|
||||
WHERE hph.house_id IS NULL
|
||||
AND (hph.raw_payload ->> 'address') IS NOT NULL
|
||||
AND h.source = 'derived'
|
||||
AND h.short_address = tradein_normalize_short_addr(hph.raw_payload ->> 'address');
|
||||
|
||||
-- -------------------------------------------------------------------------
|
||||
-- Step 7: Dedup sanity check
|
||||
-- Warn if any normalized address maps to more than one houses row.
|
||||
-- A non-zero count after the dedup fix above means manual review is needed.
|
||||
-- -------------------------------------------------------------------------
|
||||
DO $$
|
||||
DECLARE
|
||||
dup_count int;
|
||||
BEGIN
|
||||
SELECT count(*) INTO dup_count FROM (
|
||||
SELECT tradein_normalize_short_addr(address) AS norm, count(*) AS n
|
||||
FROM houses
|
||||
WHERE address IS NOT NULL
|
||||
GROUP BY tradein_normalize_short_addr(address)
|
||||
HAVING count(*) > 1
|
||||
) dup;
|
||||
IF dup_count > 0 THEN
|
||||
RAISE WARNING 'backfill 063: % normalized addresses still have multiple houses rows', dup_count;
|
||||
ELSE
|
||||
RAISE NOTICE 'backfill 063: dedup OK — 1 house per normalized address';
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
-- -------------------------------------------------------------------------
|
||||
-- Step 8: Final counters via RAISE NOTICE
|
||||
-- -------------------------------------------------------------------------
|
||||
DO $$
|
||||
DECLARE
|
||||
v_houses_total bigint;
|
||||
v_houses_derived bigint;
|
||||
v_houses_direct bigint;
|
||||
v_listings_total bigint;
|
||||
v_listings_linked bigint;
|
||||
v_listings_addr bigint;
|
||||
v_history_total bigint;
|
||||
v_history_linked bigint;
|
||||
BEGIN
|
||||
SELECT count(*) INTO v_houses_total FROM houses;
|
||||
SELECT count(*) FILTER (WHERE source = 'derived') INTO v_houses_derived FROM houses;
|
||||
SELECT count(*) FILTER (WHERE source != 'derived') INTO v_houses_direct FROM houses;
|
||||
|
||||
SELECT count(*) INTO v_listings_total FROM listings;
|
||||
SELECT count(*) FILTER (WHERE house_id_fk IS NOT NULL) INTO v_listings_linked FROM listings;
|
||||
SELECT count(*) FILTER (WHERE address IS NOT NULL) INTO v_listings_addr FROM listings;
|
||||
|
||||
SELECT count(*) INTO v_history_total FROM house_placement_history;
|
||||
SELECT count(*) FILTER (WHERE house_id IS NOT NULL) INTO v_history_linked FROM house_placement_history;
|
||||
|
||||
RAISE NOTICE
|
||||
'backfill 063 final: houses=% (direct=%, derived=%)'
|
||||
' | listings_linked=%/% (addr_coverage=%)'
|
||||
' | history_linked=%/%',
|
||||
v_houses_total, v_houses_direct, v_houses_derived,
|
||||
v_listings_linked, v_listings_total, v_listings_addr,
|
||||
v_history_linked, v_history_total;
|
||||
END $$;
|
||||
|
||||
COMMIT;
|
||||
|
|
@ -20,9 +20,7 @@ from app.services.scrapers.yandex_valuation import (
|
|||
# Fixture helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_HOUSE_META_BLOCK = (
|
||||
"12 объектов Дом 1981 года 9 этажей Панельное здание 2,50 м потолки Лифт"
|
||||
)
|
||||
_HOUSE_META_BLOCK = "12 объектов Дом 1981 года 9 этажей Панельное здание 2,50 м потолки Лифт"
|
||||
|
||||
_ITEM_1 = (
|
||||
"2-комнатная квартира 45,3 м² 4 этаж "
|
||||
|
|
@ -52,9 +50,7 @@ def _make_full_html(body_content: str) -> str:
|
|||
return f"<html><body>{body_content}</body></html>"
|
||||
|
||||
|
||||
FULL_FIXTURE_HTML = _make_full_html(
|
||||
f"{_HOUSE_META_BLOCK}\n{_ITEM_1}\n{_ITEM_2}\n{_ITEM_3}"
|
||||
)
|
||||
FULL_FIXTURE_HTML = _make_full_html(f"{_HOUSE_META_BLOCK}\n{_ITEM_1}\n{_ITEM_2}\n{_ITEM_3}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -155,9 +151,7 @@ def test_parse_items_chunked_dedup():
|
|||
duplicate_block = f"{_ITEM_1}\n{_ITEM_1}"
|
||||
items = YandexValuationScraper._parse_items_from_chunked_text(duplicate_block)
|
||||
# Both items reference 15.03.2023 + 45.3 m2 + floor 4 — must dedup to 1
|
||||
matching = [
|
||||
i for i in items if i.area_m2 == 45.3 and i.publish_date == date(2023, 3, 15)
|
||||
]
|
||||
matching = [i for i in items if i.area_m2 == 45.3 and i.publish_date == date(2023, 3, 15)]
|
||||
assert len(matching) == 1
|
||||
|
||||
|
||||
|
|
@ -240,3 +234,106 @@ def test_parse_ceiling_height_comma():
|
|||
text = "Дом 1999 года 5 этажей Кирпичное здание 2,70 м потолки Лифт"
|
||||
meta = YandexValuationScraper._parse_house_meta(text)
|
||||
assert meta.ceiling_height == 2.7
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Regression tests — parser fixes (2026-05-24)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_parse_item_extracts_removed_date():
|
||||
"""Two dates in chunk → first = publish_date, second = removed_date."""
|
||||
text = "2-комнатная квартира 50,0 м² 3 этаж 10.05.2024 В экспозиции 30 дней 09.06.2024"
|
||||
item = YandexValuationScraper._parse_item_text(text)
|
||||
assert item is not None
|
||||
assert item.publish_date == date(2024, 5, 10)
|
||||
assert item.removed_date == date(2024, 6, 9)
|
||||
|
||||
|
||||
def test_parse_item_in_sale_has_no_removed_date():
|
||||
"""Single date + 'В продаже' → removed_date is None."""
|
||||
text = "1-комнатная квартира 40 м² 5 этаж 10.05.2024 В экспозиции 30 дней В продаже"
|
||||
item = YandexValuationScraper._parse_item_text(text)
|
||||
assert item is not None
|
||||
assert item.publish_date == date(2024, 5, 10)
|
||||
assert item.removed_date is None
|
||||
|
||||
|
||||
def test_total_floors_extracted_from_dom_meta_not_items():
|
||||
"""Regression: 'N этаж' inside each item must NOT be matched as dom total_floors.
|
||||
Real Yandex page has 'M этажей' (plural) in dom-meta and 'N этаж' (singular) per item.
|
||||
"""
|
||||
text = (
|
||||
"Дом 2025 года Панорама 25 этажей Монолитное здание 2,7 м потолки Лифт "
|
||||
"1-комнатная 40 м² 3 этаж 10.01.2026 В экспозиции 5 дней В продаже "
|
||||
"2-комнатная 55 м² 17 этаж 05.01.2026 В экспозиции 10 дней В продаже"
|
||||
)
|
||||
meta = YandexValuationScraper._parse_house_meta(text)
|
||||
assert meta.year_built == 2025
|
||||
assert meta.total_floors == 25
|
||||
assert meta.house_type == "monolith"
|
||||
|
||||
|
||||
def test_parse_normalizes_nbsp_for_price_per_m2():
|
||||
"""NBSP (\\xa0) in price tokens must not break price_per_m2 regex.
|
||||
Real Yandex SSR HTML uses NBSP between digit groups and before ₽.
|
||||
"""
|
||||
html = (
|
||||
"<html><body>"
|
||||
"Дом 2020 года 16 этажей Монолитное здание "
|
||||
"2-комнатная 50,0 м² 3 этаж "
|
||||
"10.05.2024 В экспозиции 30 дней В продаже "
|
||||
"5\xa0000\xa0000 ₽ 100\xa0000 ₽ за м² "
|
||||
"4\xa0900\xa0000 ₽ 98\xa0000 ₽ за м² "
|
||||
"</body></html>"
|
||||
)
|
||||
scraper = YandexValuationScraper()
|
||||
result = scraper.parse(
|
||||
html,
|
||||
address="test",
|
||||
offer_category="APARTMENT",
|
||||
offer_type="SELL",
|
||||
page=1,
|
||||
source_url="http://test/",
|
||||
)
|
||||
assert len(result.history_items) >= 1
|
||||
item = result.history_items[0]
|
||||
assert item.start_price == 5_000_000
|
||||
assert item.start_price_per_m2 == 100_000
|
||||
|
||||
|
||||
def test_validate_match_full_match():
|
||||
from app.services.scrapers.yandex_valuation import ValuationHouseMeta
|
||||
|
||||
meta = ValuationHouseMeta(year_built=2025, total_floors=25)
|
||||
assert meta.validate_match(2025, 25) == 1.0
|
||||
|
||||
|
||||
def test_validate_match_year_mismatch():
|
||||
from app.services.scrapers.yandex_valuation import ValuationHouseMeta
|
||||
|
||||
meta = ValuationHouseMeta(year_built=1984, total_floors=9)
|
||||
assert meta.validate_match(2025, 25) == 0.0
|
||||
|
||||
|
||||
def test_validate_match_partial():
|
||||
from app.services.scrapers.yandex_valuation import ValuationHouseMeta
|
||||
|
||||
meta = ValuationHouseMeta(year_built=2025, total_floors=10)
|
||||
# year matches, floors mismatch (>1 tolerance) → 0.5
|
||||
assert meta.validate_match(2025, 25) == 0.5
|
||||
|
||||
|
||||
def test_validate_match_tolerance_one_year():
|
||||
from app.services.scrapers.yandex_valuation import ValuationHouseMeta
|
||||
|
||||
meta = ValuationHouseMeta(year_built=2024, total_floors=25)
|
||||
# ±1 year tolerance
|
||||
assert meta.validate_match(2025, 25) == 1.0
|
||||
|
||||
|
||||
def test_validate_match_no_expected_returns_one():
|
||||
from app.services.scrapers.yandex_valuation import ValuationHouseMeta
|
||||
|
||||
meta = ValuationHouseMeta(year_built=None, total_floors=None)
|
||||
assert meta.validate_match(None, None) == 1.0
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue