fix repeatable migrations
This commit is contained in:
Dusan Vojacek
209
db/routines/R__031_fn_ote_import.sql
Normal file
209
db/routines/R__031_fn_ote_import.sql
Normal file
@@ -0,0 +1,209 @@
|
||||
-- =============================================================
|
||||
-- R__031_fn_ote_import.sql
|
||||
-- OTE CZ import – parser a import funkce
|
||||
-- Repeatable migration – při změně funkce stačí upravit tento soubor
|
||||
-- =============================================================
|
||||
|
||||
-- Parser: raw jsonb → 96 (nebo 92/100 při DST) cenových řádků po 15 min
|
||||
CREATE OR REPLACE FUNCTION ems.fn_ote_parse_15m_price_json(
|
||||
in_payload jsonb,
|
||||
in_czk_per_eur numeric DEFAULT 25.000
|
||||
)
|
||||
RETURNS TABLE (
|
||||
interval_start timestamptz,
|
||||
interval_end timestamptz,
|
||||
raw_price_czk_kwh numeric(10,6)
|
||||
)
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
DECLARE
|
||||
v_date_text text;
|
||||
v_market_date date;
|
||||
v_anchor timestamptz;
|
||||
v_dl jsonb;
|
||||
v_npts int;
|
||||
BEGIN
|
||||
IF in_payload IS NULL THEN
|
||||
RAISE EXCEPTION 'in_payload must not be null';
|
||||
END IF;
|
||||
IF in_czk_per_eur IS NULL OR in_czk_per_eur <= 0 THEN
|
||||
RAISE EXCEPTION 'in_czk_per_eur must be > 0, got: %', in_czk_per_eur;
|
||||
END IF;
|
||||
|
||||
-- Datum z graph.title ve formátu "... DD.MM.YYYY"
|
||||
v_date_text := substring(
|
||||
in_payload #>> '{graph,title}'
|
||||
FROM '([0-9]{2}\.[0-9]{2}\.[0-9]{4})'
|
||||
);
|
||||
IF v_date_text IS NULL THEN
|
||||
RAISE EXCEPTION 'cannot parse date from graph.title: %',
|
||||
in_payload #>> '{graph,title}';
|
||||
END IF;
|
||||
v_market_date := to_date(v_date_text, 'DD.MM.YYYY');
|
||||
-- Začátek obchodního dne v Praze jako absolutní okamžik; pak +15 min v UTC.
|
||||
-- Starý vzor (timestamp + interval) AT TIME ZONE při přechodu na letní čas
|
||||
-- slučoval různé sloty → duplicitní interval_start a chyba ON CONFLICT.
|
||||
v_anchor := v_market_date::timestamp AT TIME ZONE 'Europe/Prague';
|
||||
|
||||
-- OTE mění strukturu: novější data = 15min série (tooltip flash_chart_01_y_15m_*),
|
||||
-- starší / jiná odpověď = jen 24 hodinových bodů s tooltip "Cena".
|
||||
SELECT t.dl INTO v_dl
|
||||
FROM jsonb_array_elements(in_payload #> '{data,dataLine}') AS t(dl)
|
||||
WHERE dl ->> 'tooltip' = 'flash_chart_01_y_15m_price_tooltip'
|
||||
AND jsonb_array_length(dl -> 'point') >= 90
|
||||
LIMIT 1;
|
||||
|
||||
IF v_dl IS NULL THEN
|
||||
SELECT t.dl INTO v_dl
|
||||
FROM jsonb_array_elements(in_payload #> '{data,dataLine}') AS t(dl)
|
||||
WHERE dl ->> 'tooltip' = 'flash_chart_01_y_60m_price_tooltip'
|
||||
AND jsonb_array_length(dl -> 'point') >= 90
|
||||
LIMIT 1;
|
||||
END IF;
|
||||
|
||||
IF v_dl IS NULL THEN
|
||||
SELECT t.dl INTO v_dl
|
||||
FROM jsonb_array_elements(in_payload #> '{data,dataLine}') AS t(dl)
|
||||
WHERE dl ->> 'tooltip' = 'Cena'
|
||||
AND jsonb_array_length(dl -> 'point') >= 90
|
||||
LIMIT 1;
|
||||
END IF;
|
||||
|
||||
IF v_dl IS NULL THEN
|
||||
SELECT t.dl INTO v_dl
|
||||
FROM jsonb_array_elements(in_payload #> '{data,dataLine}') AS t(dl)
|
||||
WHERE dl ->> 'tooltip' = 'Cena'
|
||||
AND jsonb_array_length(dl -> 'point') BETWEEN 2 AND 28
|
||||
LIMIT 1;
|
||||
END IF;
|
||||
|
||||
IF v_dl IS NULL THEN
|
||||
SELECT t.dl INTO v_dl
|
||||
FROM jsonb_array_elements(in_payload #> '{data,dataLine}') AS t(dl)
|
||||
WHERE dl ->> 'tooltip' = 'flash_chart_01_y_15m_price_tooltip'
|
||||
AND jsonb_array_length(dl -> 'point') BETWEEN 2 AND 28
|
||||
LIMIT 1;
|
||||
END IF;
|
||||
|
||||
IF v_dl IS NULL THEN
|
||||
RAISE EXCEPTION
|
||||
'OTE price dataLine not found (očekáváno 15min flash_* nebo hodinová Cena); '
|
||||
'dostupné tooltips: %',
|
||||
(SELECT jsonb_agg(dl ->> 'tooltip' ORDER BY dl ->> 'tooltip')
|
||||
FROM jsonb_array_elements(in_payload #> '{data,dataLine}') dl);
|
||||
END IF;
|
||||
|
||||
v_npts := jsonb_array_length(v_dl -> 'point');
|
||||
|
||||
IF v_npts >= 90 THEN
|
||||
-- x = 1..N = 15min bloky (96 běžně; 92 při zkráceném dni při přechodu na letní čas)
|
||||
RETURN QUERY
|
||||
SELECT s.interval_start, s.interval_end, s.raw_price_czk_kwh
|
||||
FROM (
|
||||
SELECT
|
||||
(v_anchor + ((block_no - 1) * INTERVAL '15 minutes')) AS interval_start,
|
||||
(v_anchor + (block_no * INTERVAL '15 minutes')) AS interval_end,
|
||||
ROUND(
|
||||
(price_eur_mwh * in_czk_per_eur / 1000.0)::numeric, 6
|
||||
)::numeric(10,6) AS raw_price_czk_kwh,
|
||||
block_no
|
||||
FROM (
|
||||
SELECT
|
||||
(p ->> 'x')::int AS block_no,
|
||||
(p ->> 'y')::numeric AS price_eur_mwh
|
||||
FROM jsonb_array_elements(v_dl -> 'point') AS p
|
||||
) pts
|
||||
) AS s
|
||||
ORDER BY s.block_no;
|
||||
ELSIF v_npts BETWEEN 2 AND 28 THEN
|
||||
-- x = 1..H = hodiny (ne nutně celý den); každou hodinovou cenu → 4× 15min slot
|
||||
RETURN QUERY
|
||||
SELECT s.interval_start, s.interval_end, s.raw_price_czk_kwh
|
||||
FROM (
|
||||
SELECT
|
||||
(v_anchor + (((hour_no - 1) * 4 + qix) * INTERVAL '15 minutes'))
|
||||
AS interval_start,
|
||||
(v_anchor + (((hour_no - 1) * 4 + qix) * INTERVAL '15 minutes')
|
||||
+ INTERVAL '15 minutes')
|
||||
AS interval_end,
|
||||
ROUND(
|
||||
(price_eur_mwh * in_czk_per_eur / 1000.0)::numeric, 6
|
||||
)::numeric(10,6) AS raw_price_czk_kwh,
|
||||
(hour_no - 1) * 4 + qix + 1 AS sort_key
|
||||
FROM (
|
||||
SELECT (p ->> 'x')::int AS hour_no, (p ->> 'y')::numeric AS price_eur_mwh
|
||||
FROM jsonb_array_elements(v_dl -> 'point') AS p
|
||||
) h
|
||||
CROSS JOIN (VALUES (0), (1), (2), (3)) AS q(qix)
|
||||
) AS s
|
||||
ORDER BY s.sort_key;
|
||||
ELSE
|
||||
RAISE EXCEPTION
|
||||
'OTE price series: unexpected point count % (tooltip %)',
|
||||
v_npts,
|
||||
v_dl ->> 'tooltip';
|
||||
END IF;
|
||||
|
||||
IF NOT FOUND THEN
|
||||
RAISE EXCEPTION 'OTE price series had no points after parse';
|
||||
END IF;
|
||||
END;
|
||||
$$;
|
||||
|
||||
COMMENT ON FUNCTION ems.fn_ote_parse_15m_price_json(jsonb, numeric) IS
|
||||
'Parsuje raw JSON z OTE @@chart-data?time_resolution=PT15M.
|
||||
Datum z graph.title (DD.MM.YYYY). Sloty: kotva půlnoci Europe/Prague jako timestamptz + násobky 15 min (bez DST duplicit).
|
||||
Výběr série: flash_15m ≥90, flash_60m ≥90, Cena ≥90, Cena 2–28 h, flash_15m 2–28 h.
|
||||
EUR/MWh → Kč/kWh. Typicky 96 řádků, 92 při zkráceném dni (letní čas).
|
||||
Testování přímo v DB:
|
||||
SELECT * FROM ems.fn_ote_parse_15m_price_json(pg_read_file(''/tmp/ote.json'')::jsonb, 25.0) LIMIT 5;';
|
||||
|
||||
|
||||
CREATE OR REPLACE FUNCTION ems.fn_ote_import_from_json(
|
||||
in_payload jsonb,
|
||||
in_czk_per_eur numeric DEFAULT 25.000
|
||||
)
|
||||
RETURNS integer
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
DECLARE
|
||||
v_rowcount integer;
|
||||
BEGIN
|
||||
INSERT INTO ems.market_interval_price (
|
||||
market_source,
|
||||
interval_start,
|
||||
interval_end,
|
||||
buy_raw_price_czk_kwh,
|
||||
sell_raw_price_czk_kwh,
|
||||
currency,
|
||||
imported_at
|
||||
)
|
||||
SELECT DISTINCT ON (p.interval_start)
|
||||
'OTE_CZ',
|
||||
p.interval_start,
|
||||
p.interval_end,
|
||||
p.raw_price_czk_kwh,
|
||||
p.raw_price_czk_kwh, -- spot trh: buy = sell
|
||||
'CZK',
|
||||
now()
|
||||
FROM ems.fn_ote_parse_15m_price_json(in_payload, in_czk_per_eur) AS p
|
||||
ORDER BY p.interval_start, p.interval_end DESC
|
||||
ON CONFLICT (market_source, interval_start) DO UPDATE SET
|
||||
interval_end = EXCLUDED.interval_end,
|
||||
buy_raw_price_czk_kwh = EXCLUDED.buy_raw_price_czk_kwh,
|
||||
sell_raw_price_czk_kwh = EXCLUDED.sell_raw_price_czk_kwh,
|
||||
imported_at = EXCLUDED.imported_at;
|
||||
|
||||
GET DIAGNOSTICS v_rowcount = ROW_COUNT;
|
||||
RETURN v_rowcount;
|
||||
END;
|
||||
$$;
|
||||
|
||||
COMMENT ON FUNCTION ems.fn_ote_import_from_json(jsonb, numeric) IS
|
||||
'Uloží výstup fn_ote_parse_15m_price_json do ems.market_interval_price.
|
||||
Python předá raw jsonb z HTTP response + kurz EUR/CZK.
|
||||
Vrátí počet upsertnutých řádků (očekáváno 96).
|
||||
Testování přímo v DB:
|
||||
SELECT ems.fn_ote_import_from_json(
|
||||
pg_read_file(''/tmp/ote.json'')::jsonb, 25.0
|
||||
);';
|
||||
Reference in New Issue
Block a user