diff --git a/tradein-mvp/deploy/import-rosreestr.sh b/tradein-mvp/deploy/import-rosreestr.sh new file mode 100755 index 00000000..09c8ac1f --- /dev/null +++ b/tradein-mvp/deploy/import-rosreestr.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# Импорт реальных сделок Росреестра из gendesign-БД в tradein deals. +# +# Источник: gendesign-postgres-1 / rosreestr_deals (6.8М строк, партиц.). +# Берём ЕКБ квартиры (realestate_type_code=002001003000) за последние ~18 мес. +# rooms выводим из площади (Росреестр не отдаёт кол-во комнат). +# Координаты — NULL, проставляются отдельно (geocode по street). +# +# Запуск на прод-хосте: ./import-rosreestr.sh +# Повторяемо: дедуп по dedup_hash, новый запуск подтянет свежие кварталы. + +set -euo pipefail + +SRC_PG="${SRC_PG:-gendesign-postgres-1}" +DST_PG="${DST_PG:-tradein-postgres}" +SRC_DB="${SRC_DB:-gendesign}" +SRC_USER="${SRC_USER:-gendesign}" +SINCE="${SINCE:-2024-01-01}" + +echo "[$(date -u +%H:%M:%S)] import-rosreestr: ЕКБ квартиры с $SINCE" + +# 1. Staging-таблица в tradein. +docker exec "$DST_PG" psql -U tradein -d tradein -v ON_ERROR_STOP=on -c " + CREATE TABLE IF NOT EXISTS deals_ros_staging ( + dedup_hash text PRIMARY KEY, address text, rooms int, area_m2 numeric, + floor int, year_built int, price_rub bigint, price_per_m2 int, deal_date date + ); + TRUNCATE deals_ros_staging; +" + +# 2. Трансформируем на gendesign-стороне, пайпим CSV в staging. +docker exec "$SRC_PG" psql -U "$SRC_USER" -d "$SRC_DB" -v ON_ERROR_STOP=on -c " + COPY ( + SELECT + md5('ros:' || id::text) AS dedup_hash, + 'Екатеринбург, ' || trim(street) AS address, + CASE + WHEN area < 30 THEN 0 WHEN area < 44 THEN 1 + WHEN area < 62 THEN 2 WHEN area < 85 THEN 3 + ELSE 4 END AS rooms, + round(area, 2) AS area_m2, + LEAST(NULLIF(substring(floor from '[0-9]+'), '')::int, 100) AS floor, + year_build AS year_built, + round(deal_price)::bigint AS price_rub, + round(price_per_sqm)::int AS price_per_m2, + period_start_date AS deal_date + FROM rosreestr_deals + WHERE region_code = 66 + AND city ILIKE '%катеринбург%' + AND realestate_type_code = '002001003000' + AND area BETWEEN 18 AND 200 + AND deal_price BETWEEN 1000000 AND 100000000 + AND street IS NOT NULL AND trim(street) <> '' + AND period_start_date >= '$SINCE' + ) TO STDOUT WITH CSV +" | docker exec -i "$DST_PG" psql -U tradein -d tradein -v ON_ERROR_STOP=on -c " + COPY deals_ros_staging FROM STDIN WITH CSV +" + +STAGED=$(docker exec "$DST_PG" psql -U tradein -d tradein -tAc "SELECT count(*) FROM deals_ros_staging") +echo "[$(date -u +%H:%M:%S)] staged: $STAGED строк" + +# 3. Удаляем старые синтетические сделки, вставляем реальные (дедуп). +docker exec "$DST_PG" psql -U tradein -d tradein -v ON_ERROR_STOP=on -c " + DELETE FROM deals WHERE address = 'Екатеринбург, реальная сделка'; + + INSERT INTO deals ( + source, dedup_hash, address, rooms, area_m2, floor, + year_built, price_rub, price_per_m2, deal_date + ) + SELECT 'rosreestr', dedup_hash, address, rooms, area_m2, floor, + year_built, price_rub, price_per_m2, deal_date + FROM deals_ros_staging + ON CONFLICT (dedup_hash) DO NOTHING; + + DROP TABLE deals_ros_staging; +" + +TOTAL=$(docker exec "$DST_PG" psql -U tradein -d tradein -tAc "SELECT count(*) FROM deals") +echo "[$(date -u +%H:%M:%S)] deals всего: $TOTAL" +echo "Координаты проставит geocode-deals (отдельный шаг)."