Source code for energy_data_retrieval.sources.rte.client

"""Consolidated half-hourly French production by technology from ODRÉ.

The dataset also has quarter-hour forecasts. Their empty generation fields
are not actual observations and are excluded without filling them with zero.
Only the eight mutually exclusive top-level production categories are used;
subcategories, pumping, consumption and imports are never double-counted.
"""

from __future__ import annotations

import math
from dataclasses import dataclass
from datetime import UTC, date, datetime, time, timedelta
from zoneinfo import ZoneInfo

import pandas as pd

from energy_data_retrieval.generation import GenerationResult
from energy_data_retrieval.transport import (
    HTTPClient,
    JSONValue,
    RequestRecord,
    RetrievalError,
    array_value,
    object_value,
)

RTE_HOSTS = frozenset({"odre.opendatasoft.com"})
DATASET = "eco2mix-national-cons-def"
GENERATION_ENDPOINT = (
    f"https://odre.opendatasoft.com/api/explore/v2.1/catalog/datasets/{DATASET}/records"
)
REFERENCE_IDS = ("rte_eco2mix_consolidated", "rte_eco2mix_api")
TECHNOLOGIES = (
    "fioul",
    "charbon",
    "gaz",
    "nucleaire",
    "eolien",
    "solaire",
    "hydraulique",
    "bioenergies",
)
_PARIS = ZoneInfo("Europe/Paris")
_COLUMNS = [
    "timestamp",
    "source",
    "area",
    "technology",
    "value",
    "unit",
    "aggregation",
    "nature",
    "source_datetime",
]


[docs] def normalize_generation(payload: JSONValue, *, maximum_rows: int = 1_000_000) -> pd.DataFrame: """Parse an Explore API 2.1 page of actual half-hourly generation in MW. Source ``date_heure`` has an explicit UTC offset. Forecast-only quarter-hour rows are counted in metadata and omitted; null generation in a genuine half-hour row stays missing. Source nature distinguishes consolidated and definitive observations. Neither the technology detail nor a national total is summed into the top-level production categories. """ if type(maximum_rows) is not int or not 1 <= maximum_rows <= 10_000_000: raise ValueError("maximum_rows must be in 1..10000000") root = object_value(payload) if "error" in root or "errors" in root: raise RetrievalError("RTE returned an API error document") raw_rows = array_value(root.get("results")) if len(raw_rows) * len(TECHNOLOGIES) > maximum_rows: raise RetrievalError("RTE page exceeds maximum_rows") rows: list[dict[str, object]] = [] skipped = 0 for raw in raw_rows: item = object_value(raw) label = item.get("date_heure") if not isinstance(label, str): raise RetrievalError("RTE omitted its offset-bearing timestamp") try: stamp = datetime.fromisoformat(label) except ValueError: raise RetrievalError("Invalid RTE date_heure") from None if stamp.tzinfo is None or stamp.utcoffset() is None: raise RetrievalError("RTE timestamps must include a UTC offset") if item.get("perimetre") != "France": raise RetrievalError("Expected France's national RTE generation dataset") if stamp.second or stamp.microsecond or stamp.minute not in {0, 15, 30, 45}: raise RetrievalError("RTE timestamp is not on the published quarter-hour grid") if any(key not in item for key in TECHNOLOGIES): raise RetrievalError("RTE omitted a top-level generation field") if stamp.minute in {15, 45}: if any(item[key] is not None for key in TECHNOLOGIES): raise RetrievalError( "Unexpected actual generation on an RTE quarter-hour forecast row" ) skipped += 1 continue nature = item.get("nature") if nature not in {"Données consolidées", "Données définitives"}: raise RetrievalError("Expected consolidated or definitive RTE observations") for technology in TECHNOLOGIES: raw_value = item[technology] if isinstance(raw_value, bool) or not isinstance( raw_value, (int, float, str, type(None)) ): raise RetrievalError("RTE generation must be numeric or null") try: value = float("nan") if raw_value is None else float(raw_value) except ValueError: raise RetrievalError("RTE generation is not numeric") from None if raw_value is not None and not math.isfinite(value): raise RetrievalError("RTE generation contains a non-finite value") rows.append( { "timestamp": stamp, "source": "rte", "area": "FR", "technology": technology, "value": value, "unit": "MW", "aggregation": "half_hourly_reported_power", "nature": nature, "source_datetime": label, } ) frame = pd.DataFrame(rows, columns=_COLUMNS) frame["timestamp"] = pd.to_datetime(frame["timestamp"], utc=True) frame = frame.astype({"value": "float64"}).drop_duplicates() if frame.duplicated(["timestamp", "technology"]).any(): raise RetrievalError("Conflicting RTE generation observations") frame = frame.sort_values(["timestamp", "technology"]).reset_index(drop=True) frame.attrs.update( source="RTE ODRÉ éCO2mix national consolidated/definitive", unit="MW", interval_minutes=30, dataset=DATASET, excluded_forecast_only_rows=skipped, scope="National generation by top-level technology; not plant telemetry", missing_policy="Null remains missing; forecast-only quarter-hours excluded; no filling", timestamp_semantics="Provider's offset-bearing half-hour label converted to UTC", reference_ids=list(REFERENCE_IDS), ) return frame
[docs] @dataclass(slots=True) class RTEClient: """Public Explore API retrieval, paginated separately for each French civil day.""" http: HTTPClient maximum_requests: int = 64 maximum_rows: int = 1_000_000 page_size: int = 100 def __post_init__(self) -> None: for value, name, upper in ( (self.maximum_requests, "maximum_requests", 1000), (self.maximum_rows, "maximum_rows", 10_000_000), (self.page_size, "page_size", 100), ): if type(value) is not int or not 1 <= value <= upper: raise ValueError(f"{name} must be in 1..{upper}")
[docs] def generation( self, start: date, end: date, *, dataset: str = "consolidated" ) -> GenerationResult: """Fetch inclusive French civil dates from the consolidated national dataset. Live telemetry is a different product and is deliberately not aliased to consolidated generation. Pagination must account for all advertised records; short, repeated or changing pages fail rather than truncate. """ if type(start) is not date or type(end) is not date or start > end or end.year > 9998: raise ValueError("Use ordered inclusive dates with end year <= 9998") if dataset != "consolidated": raise ValueError("This connector supports dataset='consolidated' only") if (end - start).days + 1 > self.maximum_requests: raise ValueError("RTE date range exceeds maximum_requests") frames: list[pd.DataFrame] = [] provenance: list[RequestRecord] = [] day = start count = 0 skipped = 0 while day <= end: begin = datetime.combine(day, time(), _PARIS).astimezone(UTC) until = datetime.combine(day + timedelta(days=1), time(), _PARIS).astimezone(UTC) where = ( f"date_heure >= date'{begin.isoformat()}' " f"and date_heure < date'{until.isoformat()}'" ) offset = 0 expected: int | None = None seen: set[str] = set() while True: if len(provenance) >= self.maximum_requests: raise RetrievalError("RTE pagination exceeds maximum_requests") response = self.http.get_json( GENERATION_ENDPOINT, { "where": where, "order_by": "date_heure", "limit": str(self.page_size), "offset": str(offset), }, ) root = object_value(response.value) total = root.get("total_count") if ( type(total) is not int or total < 0 or (expected is not None and total != expected) ): raise RetrievalError("RTE pagination total_count is missing or changed") expected = total entries = array_value(root.get("results")) if len(entries) > self.page_size or offset + len(entries) > total: raise RetrievalError("RTE pagination returned an invalid page length") if not entries and offset < total: raise RetrievalError("RTE pagination ended before total_count") for raw in entries: item = object_value(raw) stamp = item.get("date_heure") if not isinstance(stamp, str) or stamp in seen: raise RetrievalError("RTE pagination repeated an observation") seen.add(stamp) try: parsed = datetime.fromisoformat(stamp) valid = parsed.tzinfo is not None and begin <= parsed < until except ValueError: valid = False if not valid: raise RetrievalError("RTE observation outside the requested civil day") frame = normalize_generation(response.value, maximum_rows=self.maximum_rows) count += len(frame) if count > self.maximum_rows: raise RetrievalError("RTE retrieval exceeds maximum_rows") skipped += int(frame.attrs["excluded_forecast_only_rows"]) frames.append(frame) provenance.append(response.record) offset += len(entries) if offset == total: break day += timedelta(days=1) metadata = frames[0].attrs.copy() combined = pd.concat(frames, ignore_index=True).sort_values(["timestamp", "technology"]) if combined.duplicated(["timestamp", "technology"]).any(): raise RetrievalError("RTE returned duplicate observations across pages") combined = combined.reset_index(drop=True) combined.attrs.update(metadata) combined.attrs["excluded_forecast_only_rows"] = skipped return GenerationResult(combined, tuple(provenance))