96 lines
3.8 KiB
Bash
96 lines
3.8 KiB
Bash
#!/usr/bin/env bash
|
||
# Loads all 7 quarters of dataset_СДЕЛКИ into rosreestr_deals via staging.
|
||
# Q3 2024 uses ';' separator, all later quarters use '~'.
|
||
#
|
||
# Usage:
|
||
# bash 02_load_all_quarters.sh # local test container (gd_test_pg)
|
||
# PG_HOST=localhost PG_PORT=15432 PGPASSWORD=... bash 02_load_all_quarters.sh # remote (via SSH tunnel)
|
||
set -euo pipefail
|
||
|
||
if [[ -n "${PG_HOST:-}" ]]; then
|
||
# Remote / TCP mode (e.g. prod via SSH tunnel)
|
||
PG_PORT="${PG_PORT:-5432}"
|
||
PG_USER="${PG_USER:-gendesign}"
|
||
PG_DB="${PG_DB:-gendesign}"
|
||
PSQL="docker run --rm -i -e PGPASSWORD postgres:16-alpine psql -h host.docker.internal -p $PG_PORT -U $PG_USER -d $PG_DB -v ON_ERROR_STOP=1"
|
||
PSQL_PIPE="docker run --rm -i -e PGPASSWORD postgres:16-alpine psql -h host.docker.internal -p $PG_PORT -U $PG_USER -d $PG_DB -v ON_ERROR_STOP=1"
|
||
else
|
||
# Local docker exec into the test container
|
||
PG_CONTAINER="${PG_CONTAINER:-gd_test_pg}"
|
||
PSQL="docker exec -i $PG_CONTAINER psql -U gendesign -d gendesign -v ON_ERROR_STOP=1"
|
||
PSQL_PIPE="$PSQL"
|
||
fi
|
||
|
||
cd "$(dirname "$0")/../raw"
|
||
|
||
declare -a JOBS=(
|
||
"2024Q3:2024-07-01:dataset_СДЕЛКИ_r_all_q_3.csv:;"
|
||
"2024Q4:2024-10-01:dataset_СДЕЛКИ_r-r_01-92_y_2024_q_4.csv:~"
|
||
"2025Q1:2025-01-01:dataset_СДЕЛКИ_r-r_01-92_y_2025_q_1.csv:~"
|
||
"2025Q2:2025-04-01:dataset_СДЕЛКИ_r-r_01-92_y_2025_q_2.csv:~"
|
||
"2025Q3:2025-07-01:dataset_СДЕЛКИ_r-r_01-92_y_2025_q_3.csv:~"
|
||
"2025Q4:2025-10-01:dataset_СДЕЛКИ_r-r_01-92_y_2025_q_4.csv:~"
|
||
"2026Q1:2026-01-01:dataset_СДЕЛКИ_r-r_01-92_y_2026_q_1.csv:~"
|
||
)
|
||
|
||
for job in "${JOBS[@]}"; do
|
||
IFS=':' read -r SRC_Q PERIOD FILE SEP <<< "$job"
|
||
echo "=== $SRC_Q ($FILE, sep='$SEP') ==="
|
||
$PSQL -c "TRUNCATE rosreestr_deals_staging;"
|
||
start=$(date +%s)
|
||
cat "$FILE" | $PSQL -c "\\copy rosreestr_deals_staging FROM stdin (FORMAT csv, HEADER true, DELIMITER '$SEP', QUOTE '\"', NULL '')"
|
||
$PSQL <<SQL
|
||
INSERT INTO rosreestr_deals (
|
||
source_quarter, period_start_date,
|
||
region_code, okato, district, city, quarter_cad_number, street,
|
||
realestate_type_code, wall_material_code, purpose_code,
|
||
year_build, floor, area,
|
||
doc_type, deal_price, currency, deal_count
|
||
)
|
||
SELECT
|
||
'$SRC_Q', period_start_date,
|
||
region_code, NULLIF(okato, '-'), NULLIF(district, ''), NULLIF(city, ''),
|
||
quarter_cad_number, NULLIF(street, ''),
|
||
NULLIF(realestate_type_code, ''),
|
||
NULLIF(wall_material_code, ''),
|
||
NULLIF(purpose_code, ''),
|
||
-- year_build can contain garbage like '09.07.1988' — keep only plausible 4-digit years
|
||
CASE WHEN year_build ~ '^[12][0-9]{3}$' THEN year_build::INT ELSE NULL END,
|
||
NULLIF(floor, ''),
|
||
-- discard absurd area/price (data-quality outliers from source)
|
||
CASE WHEN area > 0 AND area < 1e8 THEN area ELSE NULL END,
|
||
NULLIF(doc_type, ''),
|
||
CASE WHEN deal_price > 0 AND deal_price < 1e15 THEN deal_price ELSE NULL END,
|
||
COALESCE(NULLIF(currency, ''), 'рубль'),
|
||
COALESCE(deal_count, 1)
|
||
FROM rosreestr_deals_staging
|
||
WHERE region_code IS NOT NULL
|
||
AND quarter_cad_number IS NOT NULL
|
||
AND quarter_cad_number <> ''
|
||
AND doc_type IS NOT NULL
|
||
AND doc_type <> ''
|
||
AND period_start_date IS NOT NULL;
|
||
SQL
|
||
end=$(date +%s)
|
||
echo " loaded in $((end-start))s"
|
||
done
|
||
|
||
$PSQL <<'SQL'
|
||
SELECT
|
||
source_quarter,
|
||
COUNT(*) AS rows_loaded,
|
||
SUM(deal_count) AS deals,
|
||
SUM(deal_count) FILTER (WHERE region_code = 66) AS sverdl_deals,
|
||
pg_size_pretty(pg_relation_size('rosreestr_deals_'||lower(source_quarter))) AS partition_size
|
||
FROM rosreestr_deals
|
||
GROUP BY source_quarter
|
||
ORDER BY source_quarter;
|
||
|
||
SELECT
|
||
'TOTAL' AS quarter,
|
||
COUNT(*) AS rows,
|
||
SUM(deal_count) AS deals,
|
||
SUM(deal_count) FILTER (WHERE region_code = 66) AS sverdl_deals,
|
||
pg_size_pretty(pg_total_relation_size('rosreestr_deals')) AS total_size
|
||
FROM rosreestr_deals;
|
||
SQL
|