feat(tradein/avito): house-поля с детальной страницы доезжают в houses (#3036) #3040
3 changed files with 278 additions and 4 deletions
|
|
@ -105,3 +105,7 @@ tests/test_2992_upsert_unchanged_gate.py::test_coalesce_backfill_still_updates
|
|||
tests/test_2992_upsert_unchanged_gate.py::test_next_day_rescrape_updates_even_if_unchanged
|
||||
tests/test_2992_upsert_unchanged_gate.py::test_skipped_row_still_yields_listing_id_for_downstream
|
||||
tests/test_2992_upsert_unchanged_gate.py::test_listing_sources_unchanged_rescrape_same_day_does_not_update
|
||||
# #3036 — house-поля с детальной страницы Авито → houses (fill-only). Live-тест ходит в
|
||||
# настоящую БД (в CI она есть, #2745), локально без TEST_DATABASE_URL пропускается. Строки
|
||||
# t3036-* тест удаляет в finally.
|
||||
tests/test_3036_detail_house_params_to_houses.py::test_live_fill_only_then_keep_then_unlinked_untouched
|
||||
|
|
|
|||
|
|
@ -0,0 +1,208 @@
|
|||
"""Детальная страница Авито больше не выбрасывает house-поля (#3036).
|
||||
|
||||
`save_detail_enrichment` парсит has_concierge/closed_yard/total_floors_house/house_type на
|
||||
каждой карточке и до правки писал в listings только house_type/house_url — остальное
|
||||
терялось. На проде при 10 051 обогащённых карточках с привязкой к дому has_concierge был
|
||||
заполнен у 5 домов из 10 131, closed_yard — у 19. Теперь вторым оператором (fill-only,
|
||||
через listings.house_id_fk) пустые поля дома заполняются; заполненные не трогаются.
|
||||
|
||||
Юнит — подменный Session: на main красный по значению (UPDATE houses не исполняется).
|
||||
Live-тест (skipif без БД): дом + листинг, fill, затем «не затирает», затем «без привязки
|
||||
— дом не тронут».
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import text
|
||||
|
||||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||||
|
||||
from scraper_kit.providers.avito.detail import (
|
||||
DetailEnrichment,
|
||||
save_detail_enrichment,
|
||||
)
|
||||
|
||||
|
||||
def _enrichment(**over: Any) -> DetailEnrichment:
|
||||
base: dict[str, Any] = {
|
||||
"item_id": "t3036-1",
|
||||
"source_url": "https://www.avito.ru/ekaterinburg/kvartiry/x_t3036-1",
|
||||
"has_concierge": True,
|
||||
"closed_yard": False,
|
||||
"total_floors_house": 9,
|
||||
"house_type": "panel",
|
||||
}
|
||||
base.update(over)
|
||||
return DetailEnrichment(**base)
|
||||
|
||||
|
||||
class _FakeNested:
|
||||
def __enter__(self) -> None:
|
||||
return None
|
||||
|
||||
def __exit__(self, *exc: object) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
class _FakeSession:
|
||||
"""Пишет все execute(sql, params); rowcount=1 для обоих операторов."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.calls: list[tuple[str, dict[str, Any]]] = []
|
||||
self.nested = 0
|
||||
self.committed = 0
|
||||
|
||||
def execute(self, stmt: Any, params: dict[str, Any] | None = None) -> Any:
|
||||
self.calls.append((str(stmt), params or {}))
|
||||
return SimpleNamespace(rowcount=1)
|
||||
|
||||
def begin_nested(self) -> _FakeNested:
|
||||
self.nested += 1
|
||||
return _FakeNested()
|
||||
|
||||
def commit(self) -> None:
|
||||
self.committed += 1
|
||||
|
||||
|
||||
def test_house_params_written_fill_only_via_house_id_fk() -> None:
|
||||
"""Головной: за UPDATE listings идёт UPDATE houses … FROM listings … house_id_fk, fill-only."""
|
||||
db = _FakeSession()
|
||||
assert save_detail_enrichment(db, _enrichment()) is True # type: ignore[arg-type]
|
||||
assert len(db.calls) == 2, [c[0][:40] for c in db.calls]
|
||||
sql, params = db.calls[1]
|
||||
assert "UPDATE houses" in sql and "l.house_id_fk = h.id" in sql
|
||||
for col in ("has_concierge", "closed_yard", "total_floors", "house_type"):
|
||||
assert f"COALESCE(h.{col}," in sql, f"{col} не fill-only"
|
||||
assert params["has_concierge"] is True and params["closed_yard"] is False
|
||||
assert params["total_floors_house"] == 9 and params["house_type"] == "panel"
|
||||
assert params["item_id"] == "t3036-1"
|
||||
assert db.nested == 1, "UPDATE houses обязан идти под SAVEPOINT"
|
||||
assert db.committed == 1
|
||||
|
||||
|
||||
def test_lifts_are_not_written_no_such_columns() -> None:
|
||||
"""Контроль: лифты не пишем — колонок в houses нет (conflict_resolution их объявляет зря)."""
|
||||
db = _FakeSession()
|
||||
save_detail_enrichment(db, _enrichment(passenger_elevators=2, cargo_elevators=1)) # type: ignore[arg-type]
|
||||
sql = db.calls[1][0]
|
||||
assert "elevator" not in sql and "lifts" not in sql
|
||||
|
||||
|
||||
def test_nothing_parsed_means_no_second_statement() -> None:
|
||||
"""Контроль: если с карточки house-полей не пришло — второй оператор не исполняется."""
|
||||
db = _FakeSession()
|
||||
save_detail_enrichment( # type: ignore[arg-type]
|
||||
db,
|
||||
_enrichment(has_concierge=None, closed_yard=None, total_floors_house=None, house_type=None),
|
||||
)
|
||||
assert len(db.calls) == 1 and db.nested == 0
|
||||
|
||||
|
||||
def test_listing_not_found_skips_houses() -> None:
|
||||
"""Контроль: листинга нет (rowcount 0) — дом не трогаем, False наружу как раньше."""
|
||||
|
||||
class _NotFound(_FakeSession):
|
||||
def execute(self, stmt: Any, params: dict[str, Any] | None = None) -> Any:
|
||||
self.calls.append((str(stmt), params or {}))
|
||||
return SimpleNamespace(rowcount=0)
|
||||
|
||||
db = _NotFound()
|
||||
assert save_detail_enrichment(db, _enrichment()) is False # type: ignore[arg-type]
|
||||
assert len(db.calls) == 1
|
||||
|
||||
|
||||
# ── live DB ──────────────────────────────────────────────────────────────────
|
||||
def _live_session() -> Any | None:
|
||||
try:
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
dsn = os.environ.get("TEST_DATABASE_URL") or os.environ.get("DATABASE_URL", "")
|
||||
if not dsn or "localhost:5432/test" in dsn:
|
||||
return None
|
||||
engine = create_engine(dsn, future=True)
|
||||
conn = engine.connect()
|
||||
conn.execute(text("SELECT 1"))
|
||||
conn.close()
|
||||
return sessionmaker(bind=engine, future=True)()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _cleanup(db: Any) -> None:
|
||||
try:
|
||||
db.rollback()
|
||||
db.execute(text("DELETE FROM listings WHERE source='avito' AND source_id LIKE 't3036-%'"))
|
||||
db.execute(text("DELETE FROM houses WHERE source='test' AND ext_house_id LIKE 't3036-%'"))
|
||||
db.commit()
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
@pytest.mark.skipif(_live_session() is None, reason="no reachable Postgres test DB")
|
||||
def test_live_fill_only_then_keep_then_unlinked_untouched() -> None:
|
||||
db = _live_session()
|
||||
assert db is not None
|
||||
try:
|
||||
_cleanup_inline = text(
|
||||
"DELETE FROM listings WHERE source='avito' AND source_id LIKE 't3036-%'"
|
||||
)
|
||||
db.execute(_cleanup_inline)
|
||||
db.execute(text("DELETE FROM houses WHERE source='test' AND ext_house_id LIKE 't3036-%'"))
|
||||
hid = db.execute(
|
||||
text(
|
||||
"INSERT INTO houses (source, ext_house_id, url, address) "
|
||||
"VALUES ('test', 't3036-h1', 'https://example.test/houses/t3036-h1', "
|
||||
"'ул. Тестовая, 1') RETURNING id"
|
||||
)
|
||||
).scalar_one()
|
||||
db.execute(
|
||||
text(
|
||||
"INSERT INTO listings "
|
||||
"(source, source_id, source_url, dedup_hash, address, price_rub, "
|
||||
" house_id_fk, is_active) "
|
||||
"VALUES ('avito', 't3036-1', 'https://www.avito.ru/x/t3036-1', 't3036-dh-1', "
|
||||
" 'ул. Тестовая, 1', 5000000, :hid, true), "
|
||||
" ('avito', 't3036-2', 'https://www.avito.ru/x/t3036-2', 't3036-dh-2', "
|
||||
" 'ул. Тестовая, 2', 5000000, NULL, true)"
|
||||
),
|
||||
{"hid": hid},
|
||||
)
|
||||
db.commit()
|
||||
# 1) fill
|
||||
assert save_detail_enrichment(db, _enrichment()) is True
|
||||
row = db.execute(
|
||||
text(
|
||||
"SELECT has_concierge, closed_yard, total_floors, house_type "
|
||||
"FROM houses WHERE id=:h"
|
||||
),
|
||||
{"h": hid},
|
||||
).one()
|
||||
assert tuple(row) == (True, False, 9, "panel"), tuple(row)
|
||||
# 2) fill-only: другое значение с другой карточки того же дома не затирает
|
||||
assert (
|
||||
save_detail_enrichment(db, _enrichment(has_concierge=False, total_floors_house=16))
|
||||
is True
|
||||
)
|
||||
row = db.execute(
|
||||
text("SELECT has_concierge, total_floors FROM houses WHERE id=:h"), {"h": hid}
|
||||
).one()
|
||||
assert tuple(row) == (True, 9), tuple(row)
|
||||
# 3) листинг без привязки — дом не трогается, листинг обогащается
|
||||
before = db.execute(
|
||||
text("SELECT count(*) FROM houses WHERE house_type='brick' AND source='test'")
|
||||
).scalar_one()
|
||||
assert (
|
||||
save_detail_enrichment(db, _enrichment(item_id="t3036-2", house_type="brick")) is True
|
||||
)
|
||||
after = db.execute(
|
||||
text("SELECT count(*) FROM houses WHERE house_type='brick' AND source='test'")
|
||||
).scalar_one()
|
||||
assert after == before
|
||||
finally:
|
||||
_cleanup(db)
|
||||
|
|
@ -6,7 +6,8 @@
|
|||
bathroom_type, windows_view, repair_state, sale_type, mortgage_available)
|
||||
- Location (lat, lon, avito_location_id, metro_stations[], address_full)
|
||||
- House params (house_type, total_floors_house, lifts, concierge, closed_yard,
|
||||
house_catalog_url) — собираются но НЕ сохраняются в БД (Stage 2c)
|
||||
house_catalog_url) — house_type/total_floors/concierge/closed_yard доезжают в
|
||||
houses (fill-only, через listings.house_id_fk, #3036); лифты — нет колонок в houses
|
||||
- Description
|
||||
- Domoteka 5 полей (owners_count, owners_at_least, last_owner_change_date,
|
||||
encumbrances_clean, registry_match)
|
||||
|
|
@ -802,6 +803,60 @@ def parse_detail_html(html: str, source_url: str) -> DetailEnrichment:
|
|||
|
||||
|
||||
# ── save_detail_enrichment ────────────────────────────────────────────────────
|
||||
# #3036: house-level поля с детальной страницы → houses (fill-only). Колонок под лифты
|
||||
# в houses нет — passenger/cargo_elevators намеренно не в списке.
|
||||
_HOUSE_PARAMS_SQL = text("""
|
||||
UPDATE houses h SET
|
||||
has_concierge = COALESCE(h.has_concierge, CAST(:has_concierge AS boolean)),
|
||||
closed_yard = COALESCE(h.closed_yard, CAST(:closed_yard AS boolean)),
|
||||
total_floors = COALESCE(h.total_floors, CAST(:total_floors_house AS integer)),
|
||||
house_type = COALESCE(h.house_type, CAST(:house_type AS text))
|
||||
FROM listings l
|
||||
WHERE l.source = 'avito' AND l.source_id = :item_id
|
||||
AND l.house_id_fk = h.id
|
||||
AND (h.has_concierge IS NULL AND CAST(:has_concierge AS boolean) IS NOT NULL
|
||||
OR h.closed_yard IS NULL AND CAST(:closed_yard AS boolean) IS NOT NULL
|
||||
OR h.total_floors IS NULL AND CAST(:total_floors_house AS integer) IS NOT NULL
|
||||
OR h.house_type IS NULL AND CAST(:house_type AS text) IS NOT NULL)
|
||||
""")
|
||||
|
||||
|
||||
def _fill_house_params_from_detail(db: Session, e: DetailEnrichment) -> int:
|
||||
"""Заполнить пустые house-поля дома листинга значениями с детальной страницы (#3036).
|
||||
|
||||
Возвращает число обновлённых домов (0 — листинг без house_id_fk, у дома всё уже
|
||||
заполнено, либо с карточки ничего не пришло). Ошибка оператора глушится под
|
||||
SAVEPOINT с warning: обогащение листинга важнее.
|
||||
"""
|
||||
if (
|
||||
e.has_concierge is None
|
||||
and e.closed_yard is None
|
||||
and e.total_floors_house is None
|
||||
and e.house_type is None
|
||||
):
|
||||
return 0
|
||||
try:
|
||||
with db.begin_nested():
|
||||
r = db.execute(
|
||||
_HOUSE_PARAMS_SQL,
|
||||
{
|
||||
"item_id": e.item_id,
|
||||
"has_concierge": e.has_concierge,
|
||||
"closed_yard": e.closed_yard,
|
||||
"total_floors_house": e.total_floors_house,
|
||||
"house_type": e.house_type,
|
||||
},
|
||||
)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"save_detail_enrichment: house params not saved for item_id=%s",
|
||||
e.item_id,
|
||||
exc_info=True,
|
||||
)
|
||||
return 0
|
||||
return int(r.rowcount or 0)
|
||||
|
||||
|
||||
def save_detail_enrichment(db: Session, e: DetailEnrichment) -> bool:
|
||||
"""UPDATE listings SET <25+ cols> WHERE source='avito' AND source_id=:item_id.
|
||||
|
||||
|
|
@ -814,9 +869,14 @@ def save_detail_enrichment(db: Session, e: DetailEnrichment) -> bool:
|
|||
house_catalog_url колонки нет. house_type — COALESCE existing-first (не затираем
|
||||
canonical из Houses Catalog Stage 2c); house_url — new-first additive (avito-owned
|
||||
поле, нет конкурирующего canonical-писателя).
|
||||
Остальные house-level поля (passenger_elevators, cargo_elevators, has_concierge,
|
||||
closed_yard, total_floors_house) по-прежнему опускаются — нет колонок в listings,
|
||||
canonical приходит из Houses Catalog Stage 2c (отдельный follow-up).
|
||||
House-level поля (has_concierge, closed_yard, total_floors_house, house_type)
|
||||
пишутся в houses ЧЕРЕЗ listings.house_id_fk вторым оператором (#3036), fill-only:
|
||||
COALESCE(houses.X, новое) — не затираем канон из Houses Catalog Stage 2c, а
|
||||
заполняем пустое. До этого они парсились на каждой карточке и выбрасывались:
|
||||
на проде has_concierge был заполнен у 5 домов из 10 131, closed_yard — у 19,
|
||||
при 10 051 обогащённых карточках Авито с привязкой к дому. Лифты
|
||||
(passenger_elevators/cargo_elevators) по-прежнему опускаются — в houses нет колонок.
|
||||
Под SAVEPOINT: отказ этого оператора не должен ронять обогащение самого листинга.
|
||||
"""
|
||||
result = db.execute(
|
||||
text("""
|
||||
|
|
@ -894,6 +954,8 @@ def save_detail_enrichment(db: Session, e: DetailEnrichment) -> bool:
|
|||
"house_catalog_url": e.house_catalog_url,
|
||||
},
|
||||
)
|
||||
if result.rowcount > 0:
|
||||
_fill_house_params_from_detail(db, e)
|
||||
db.commit()
|
||||
found = result.rowcount > 0
|
||||
if not found:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue