second version

backend/services/price_importer.py

@@ -1,11 +1,10 @@
-"""OTE CZ DAM spot price import (15min slots, shared market table)."""
-
+"""OTE CZ price import – Python dělá pouze HTTP fetch, logika je v PostgreSQL."""
 from __future__ import annotations
 
+import asyncio
 import json
 import logging
-from datetime import date, datetime, timedelta, timezone
-from typing import Any
+from datetime import date, datetime, timedelta
 from zoneinfo import ZoneInfo
 
 import httpx
@@ -14,167 +13,178 @@ from app.config import get_settings
 
 logger = logging.getLogger(__name__)
 
-MARKET_SOURCE = "OTE_CZ"
+OTE_URL = (
+    "https://www.ote-cr.cz/cs/kratkodobe-trhy/elektrina/denni-trh/"
+    "@@chart-data?report_date={date}&time_resolution=PT15M"
+)
+
+
+def _is_retryable_status(status_code: int) -> bool:
+    return status_code in {408, 425, 429, 500, 502, 503, 504}
+
+
+async def _fetch_ote_json(date_str: str) -> tuple[dict | None, str | None]:
+    url = OTE_URL.format(date=date_str)
+    timeout = httpx.Timeout(connect=10.0, read=45.0, write=10.0, pool=10.0)
+    headers = {
+        "User-Agent": "Mozilla/5.0 (compatible; EMS/1.0; +https://www.ote-cr.cz)",
+        "Accept": "application/json, text/plain, */*",
+        "Accept-Language": "cs-CZ,cs;q=0.9,en;q=0.8",
+        "Connection": "keep-alive",
+    }
+    max_attempts = 4
+    backoff_s = 1.0
+    last_err: str | None = None
+
+    async with httpx.AsyncClient(
+        timeout=timeout,
+        headers=headers,
+        follow_redirects=True,
+    ) as client:
+        for attempt in range(1, max_attempts + 1):
+            try:
+                logger.info("OTE fetch %s attempt %s/%s", date_str, attempt, max_attempts)
+                resp = await client.get(url)
+                if _is_retryable_status(resp.status_code) and attempt < max_attempts:
+                    last_err = f"http_status:{resp.status_code}"
+                    logger.warning(
+                        "OTE temporary HTTP %s for %s (attempt %s/%s), retrying",
+                        resp.status_code,
+                        date_str,
+                        attempt,
+                        max_attempts,
+                    )
+                    await asyncio.sleep(backoff_s)
+                    backoff_s *= 2.0
+                    continue
+                resp.raise_for_status()
+                return resp.json(), None
+            except (httpx.ConnectError, httpx.ReadTimeout, httpx.WriteTimeout, httpx.PoolTimeout) as e:
+                last_err = f"timeout_or_connect:{e.__class__.__name__}"
+                if attempt < max_attempts:
+                    logger.warning(
+                        "OTE request failed for %s (%s), retrying %s/%s",
+                        date_str,
+                        e.__class__.__name__,
+                        attempt,
+                        max_attempts,
+                    )
+                    await asyncio.sleep(backoff_s)
+                    backoff_s *= 2.0
+                    continue
+                logger.error("OTE fetch failed for %s after retries: %s", date_str, e)
+            except httpx.HTTPStatusError as e:
+                code = e.response.status_code if e.response is not None else "unknown"
+                last_err = f"http_status:{code}"
+                logger.error("OTE HTTP error for %s: %s", date_str, code)
+                break
+            except json.JSONDecodeError as e:
+                last_err = f"invalid_json:{e.__class__.__name__}"
+                logger.error("OTE invalid JSON for %s: %s", date_str, e)
+                break
+            except Exception as e:
+                last_err = f"unexpected:{e.__class__.__name__}"
+                logger.error("OTE fetch unexpected error for %s: %s", date_str, e)
+                break
+
+    return None, last_err
 
 
 async def import_ote_prices(
     site_id: int,
     db,
     target_date: date | None = None,
-) -> tuple[int, str, float]:
+) -> tuple[int, str, float, str | None]:
     """
-    Stáhne DAM ceny OTE pro zvolený den (nebo „zítřek“ v TZ lokality), uloží 96 slotů (15 min).
-
-    Schéma DB: ``ems.market_interval_price`` má PK ``(market_source, interval_start)``;
-    ceny v ``buy_raw_price_czk_kwh`` / ``sell_raw_price_czk_kwh`` (pro OTE stejné).
-
-    Returns:
-        ``(počet_slotů, datum_YMD, první_cena_kč_kwh)``. Počet 96 při úspěchu, -1 při chybě.
-        První cena je cena prvního 15min slotu dne; při chybě 0.0.
-        Datum je prázdný řetězec jen pokud site neexistuje nebo je neplatná timezone.
+    Stáhne OTE JSON a předá ho PostgreSQL funkci ems.fn_ote_import_from_json.
+    Python nedělá žádné parsování ani přepočty – vše je v DB funkcích.
+    Returns: (počet_slotů, datum_str, první_cena_kč_kwh, error_code)
+             (-1, datum_str, 0.0, error_code) při chybě
     """
+    settings = get_settings()
+
     row = await db.fetchrow(
-        "SELECT timezone FROM ems.site WHERE id = $1",
-        site_id,
+        "SELECT timezone FROM ems.site WHERE id = $1", site_id
     )
     if row is None:
-        logger.error("import_ote_prices: site id=%s nenalezen", site_id)
-        return -1, "", 0.0
+        logger.error("OTE import: site id=%s nenalezen", site_id)
+        return -1, "", 0.0, "site_not_found"
 
-    tz_name: str = row["timezone"] or "Europe/Prague"
-    try:
-        site_tz = ZoneInfo(tz_name)
-    except Exception as e:
-        logger.error("import_ote_prices: neplatná timezone %r: %s", tz_name, e)
-        return -1, "", 0.0
+    site_tz = ZoneInfo(row["timezone"] or "Europe/Prague")
+    now_site = datetime.now(site_tz)
+    today_site = now_site.date()
+    tomorrow_site = today_site + timedelta(days=1)
+    candidate_days = [target_date] if target_date is not None else [tomorrow_site, today_site]
+
+    payload: dict | None = None
+    fetch_error: str | None = None
+    target_day = candidate_days[0]
+
+    # Varování před 13:30 CET při implicitním (zítra) importu.
+    if target_date is None:
+        now_cet = datetime.now(ZoneInfo("Europe/Prague"))
+        if now_cet.hour < 13 or (now_cet.hour == 13 and now_cet.minute < 30):
+            logger.warning(
+                "OTE: ceny pro %s nemusí být dostupné (před 13:30 CET), použiji fallback na dnešek",
+                tomorrow_site.isoformat(),
+            )
+
+    for day in candidate_days:
+        day_str = day.isoformat()
+        payload, fetch_error = await _fetch_ote_json(day_str)
+        if payload is not None:
+            target_day = day
+            break
+        logger.warning("OTE fetch selhal pro %s (err=%s)", day_str, fetch_error)
+
+    if payload is None:
+        return -1, candidate_days[0].isoformat(), 0.0, fetch_error or "fetch_failed"
 
-    if target_date is not None:
-        target_day = target_date
-    else:
-        now_local = datetime.now(site_tz)
-        target_day = (now_local + timedelta(days=1)).date()
     date_str = target_day.isoformat()
 
-    cet = ZoneInfo("Europe/Prague")
-    now_cet = datetime.now(cet)
-    tomorrow_cet = (now_cet + timedelta(days=1)).date()
-    if target_day == tomorrow_cet:
-        cutoff = now_cet.replace(hour=13, minute=30, second=0, microsecond=0)
-        if now_cet < cutoff:
-            logger.warning(
-                "OTE prices for tomorrow may not be available yet (before 13:30 CET)"
-            )
-
-    settings = get_settings()
-    base_url = settings.ote_api_url.rstrip("/")
-    url = f"{base_url}?date={date_str}"
+    # Vše ostatní řeší PostgreSQL funkce
     eur_czk = float(settings.eur_czk_rate)
-
     try:
-        async with httpx.AsyncClient(timeout=30.0) as client:
-            resp = await client.get(url)
-            resp.raise_for_status()
-            body = resp.json()
-    except httpx.TimeoutException:
-        logger.warning("import_ote_prices: timeout při GET %s", url)
-        return -1, date_str, 0.0
-    except httpx.HTTPStatusError as e:
-        logger.warning(
-            "import_ote_prices: HTTP %s při GET %s: %s",
-            e.response.status_code,
-            url,
-            e.response.text[:500],
+        n = await db.fetchval(
+            "SELECT ems.fn_ote_import_from_json($1::jsonb, $2)",
+            json.dumps(payload),
+            eur_czk,
         )
-        return -1, date_str, 0.0
-    except httpx.HTTPError as e:
-        logger.warning("import_ote_prices: HTTP chyba při GET %s: %s", url, e)
-        return -1, date_str, 0.0
-    except Exception as e:
-        logger.warning("import_ote_prices: neočekávaná chyba při stahování: %s", e)
-        return -1, date_str, 0.0
-
-    hourly_eur_mwh: dict[int, float] | None = None
-    try:
-        points: list[dict[str, Any]] = body["data"]["dataLine"][0]["point"]
-        hourly_eur_mwh = {}
-        for p in points:
-            x = int(p["x"])
-            y = float(p["y"])
-            hourly_eur_mwh[x] = y
-    except (KeyError, TypeError, ValueError, IndexError):
-        snippet = json.dumps(body, ensure_ascii=False)[:500]
-        logger.error("import_ote_prices: neočekádaná struktura OTE, začátek: %s", snippet)
-        return -1, date_str, 0.0
-
-    if len(hourly_eur_mwh) != 24 or set(hourly_eur_mwh.keys()) != set(range(1, 25)):
-        logger.error(
-            "import_ote_prices: očekáváno 24 bodů x=1..24, dostáno klíče %s",
-            sorted(hourly_eur_mwh.keys()),
-        )
-        return -1, date_str, 0.0
-
-    slots: list[tuple[datetime, datetime, float]] = []
-    for h in range(24):
-        x = h + 1
-        eur_mwh = hourly_eur_mwh[x]
-        price_czk_kwh = eur_mwh * eur_czk / 1000.0
-        for minute in (0, 15, 30, 45):
-            interval_start_local = datetime(
-                target_day.year,
-                target_day.month,
-                target_day.day,
-                h,
-                minute,
-                tzinfo=site_tz,
-            )
-            interval_start_utc = interval_start_local.astimezone(timezone.utc)
-            interval_end_utc = interval_start_utc + timedelta(minutes=15)
-            slots.append((interval_start_utc, interval_end_utc, price_czk_kwh))
-
-    for interval_start_utc, interval_end_utc, price in slots:
-        await db.execute(
+        first_price = await db.fetchval(
             """
-            INSERT INTO ems.market_interval_price (
-                market_source,
-                interval_start,
-                interval_end,
-                buy_raw_price_czk_kwh,
-                sell_raw_price_czk_kwh,
-                currency,
-                imported_at
-            )
-            VALUES ($1, $2, $3, $4, $5, 'CZK', now())
-            ON CONFLICT (market_source, interval_start)
-            DO UPDATE SET
-                interval_end = EXCLUDED.interval_end,
-                buy_raw_price_czk_kwh = EXCLUDED.buy_raw_price_czk_kwh,
-                sell_raw_price_czk_kwh = EXCLUDED.sell_raw_price_czk_kwh,
-                imported_at = now()
+            SELECT buy_raw_price_czk_kwh
+            FROM ems.market_interval_price
+            WHERE market_source = 'OTE_CZ'
+              AND interval_start::date = $1::date
+            ORDER BY interval_start
+            LIMIT 1
             """,
-            MARKET_SOURCE,
-            interval_start_utc,
-            interval_end_utc,
-            price,
-            price,
+            target_day,
         )
-
-    first_price = float(slots[0][2]) if slots else 0.0
-    return len(slots), date_str, first_price
-
-
-if __name__ == "__main__":
-    import asyncio
-    import os
-
-    import asyncpg
-    from dotenv import load_dotenv
-
-    load_dotenv()
-
-    async def test():
-        conn = await asyncpg.connect(os.getenv("DATABASE_URL"))
-        n, d, fp = await import_ote_prices(1, conn)
-        print(f"Uloženo {n} slotů pro {d}, první cena {fp}")
-        await conn.close()
-
-    asyncio.run(test())
+        n_imported = await db.fetchval(
+            """
+            SELECT COUNT(*)::int
+            FROM ems.market_interval_price
+            WHERE market_source = 'OTE_CZ'
+              AND interval_start::date = $1::date
+            """,
+            target_day,
+        )
+        incomplete = (n_imported or 0) < 96
+        if incomplete:
+            now_p = datetime.now(ZoneInfo("Europe/Prague"))
+            tomorrow_p = (now_p + timedelta(days=1)).date()
+            # Stejná logika jako dashboard: neúplný D+1 před 14:30 je očekávaný
+            if not (
+                target_day == tomorrow_p
+                and (now_p.hour, now_p.minute) < (14, 30)
+            ):
+                logger.warning("OTE: jen %s/96 slotů pro %s", n_imported, date_str)
+        logger.info(
+            "OTE import OK: %s slotů pro %s, první cena %.4f Kč/kWh",
+            n, date_str, float(first_price or 0),
+        )
+        return int(n), date_str, float(first_price or 0.0), None
+    except Exception as e:
+        logger.error("OTE import DB error: %s", e)
+        return -1, date_str, 0.0, f"db_import:{e.__class__.__name__}"