Source code for energy_data_retrieval.sources.entsoe.client

"""ENTSO-E actual generation, with UTC intervals and explicit consumption flows.

The Transparency Platform reports interval-average power in MW. A03 points are
step changes, not a sparse list of independent samples. Missing quantities remain
missing, including when the missing block extends across several positions.
"""

from __future__ import annotations

import math
import os
import re
import xml.etree.ElementTree as ET
from dataclasses import dataclass, field
from datetime import UTC, datetime, timedelta

import pandas as pd

from energy_data_retrieval.transport import HTTPClient, RequestRecord, RetrievalError

ENTSOE_ENDPOINT = "https://web-api.tp.entsoe.eu/api"
ENTSOE_HOSTS = frozenset({"web-api.tp.entsoe.eu"})
REFERENCE_IDS = (
    "entsoe_transparency_api",
    "entsoe_generation_data_descriptions",
    "entsoe_curve_types",
)
PSR_TYPES = {
    "B01": "Biomass",
    "B02": "Fossil Brown coal/Lignite",
    "B03": "Fossil Coal-derived gas",
    "B04": "Fossil Gas",
    "B05": "Fossil Hard coal",
    "B06": "Fossil Oil",
    "B07": "Fossil Oil shale",
    "B08": "Fossil Peat",
    "B09": "Geothermal",
    "B10": "Hydro Pumped Storage",
    "B11": "Hydro Run-of-river and poundage",
    "B12": "Hydro Water Reservoir",
    "B13": "Marine",
    "B14": "Nuclear",
    "B15": "Other renewable",
    "B16": "Solar",
    "B17": "Waste",
    "B18": "Wind Offshore",
    "B19": "Wind Onshore",
    "B20": "Other",
    "B25": "Energy storage",
}
_COLUMNS = (
    "timestamp",
    "interval_end",
    "duration_hours",
    "source",
    "area",
    "unit_id",
    "unit_name",
    "technology",
    "technology_name",
    "flow",
    "value",
    "unit",
    "native_unit",
    "document_type",
    "document_id",
    "revision",
    "series_id",
    "curve_type",
    "business_type",
    "missing_reason",
)
_KEYS = ["timestamp", "area", "unit_id", "technology", "flow"]
_DURATION = re.compile(r"P(?:(\d+)D)?(?:T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?)?\Z")


def _integer(value: int, label: str, maximum: int) -> None:
    if type(value) is not int or not 1 <= value <= maximum:
        raise ValueError(f"{label} must be an integer in 1..{maximum}")


def _eic(value: str) -> str:
    if not isinstance(value, str) or re.fullmatch(r"[A-Z0-9-]{16}", value) is None:
        raise ValueError("Use a 16-character EIC from ENTSO-E's official area or unit list")
    return value


def _utc(value: datetime) -> datetime:
    if not isinstance(value, datetime) or value.tzinfo is None or value.utcoffset() is None:
        raise ValueError("ENTSO-E boundaries must be timezone-aware datetimes")
    if value.second or value.microsecond:
        raise ValueError("ENTSO-E request boundaries must align to whole minutes")
    return value.astimezone(UTC)


[docs] def generation_requests( area: str, start: datetime, end: datetime, *, per_unit: bool = False, psr_type: str | None = None, unit_id: str | None = None, maximum_requests: int = 366, ) -> tuple[dict[str, str], ...]: """Plan non-secret requests for ``[start, end)`` in UTC. Aggregate queries use conservative 31-day chunks; per-unit queries use one-day chunks. EICs are explicit because countries and bidding zones are not always interchangeable. This function makes no network requests. """ _eic(area) _integer(maximum_requests, "maximum_requests", 10000) first, last = _utc(start), _utc(end) if first >= last: raise ValueError("ENTSO-E end must be strictly after start (exclusive end)") if psr_type is not None and psr_type not in PSR_TYPES: raise ValueError("psr_type must be a supported ENTSO-E PSR code, for example B16") if unit_id is not None: _eic(unit_id) if not per_unit: raise ValueError("unit_id is only valid for generation per unit") step = timedelta(days=1 if per_unit else 31) if math.ceil((last - first) / step) > maximum_requests: raise ValueError("ENTSO-E date range exceeds maximum_requests; request a smaller range") requests: list[dict[str, str]] = [] cursor = first while cursor < last: chunk_end = cursor + min(step, last - cursor) parameters = { "documentType": "A73" if per_unit else "A75", "processType": "A16", "in_Domain": area, "periodStart": cursor.strftime("%Y%m%d%H%M"), "periodEnd": chunk_end.strftime("%Y%m%d%H%M"), } if psr_type is not None: parameters["psrType"] = psr_type if unit_id is not None: parameters["registeredResource"] = unit_id requests.append(parameters) cursor = chunk_end return tuple(requests)
def _text(element: ET.Element, path: str, *, required: bool = False) -> str: found = element.find("/".join("{*}" + name for name in path.split("/"))) result = found.text.strip() if found is not None and found.text else "" if required and not result: raise RetrievalError(f"ENTSO-E document lacks required {path}") return result def _stamp(value: str) -> datetime: try: stamp = datetime.fromisoformat(value.replace("Z", "+00:00")) if stamp.tzinfo is None or stamp.utcoffset() != timedelta(0): raise ValueError return stamp.astimezone(UTC) except (ValueError, OverflowError): raise RetrievalError("ENTSO-E XML time boundaries must be valid UTC timestamps") from None def _resolution(value: str) -> timedelta: match = _DURATION.fullmatch(value) if match is None or not any(match.groups()): raise RetrievalError("ENTSO-E has an unsupported fixed-duration resolution") days, hours, minutes, seconds = (int(part or 0) for part in match.groups()) total = days * 86400 + hours * 3600 + minutes * 60 + seconds if not 1 <= total <= 86400: raise RetrievalError("ENTSO-E generation resolution must be between one second and one day") return timedelta(seconds=total) def _empty(*, no_data: bool = False) -> pd.DataFrame: frame = pd.DataFrame(columns=list(_COLUMNS)) frame["timestamp"] = pd.to_datetime(frame["timestamp"], utc=True) frame["interval_end"] = pd.to_datetime(frame["interval_end"], utc=True) frame["value"] = frame["value"].astype(float) frame["duration_hours"] = frame["duration_hours"].astype(float) frame.attrs.update( source="entsoe", no_data=no_data, timestamp_convention="interval_start", reference_ids=list(REFERENCE_IDS), ) return frame def _deduplicate(frame: pd.DataFrame) -> pd.DataFrame: """Drop equal repeats; never silently sum revised or overlapping observations.""" compare = [*_KEYS, "interval_end", "value", "missing_reason"] result = frame.drop_duplicates(compare) if result.duplicated(_KEYS).any(): raise RetrievalError("ENTSO-E returned conflicting duplicate or overlapping intervals") result = result.sort_values(_KEYS).reset_index(drop=True) for _, group in result.groupby(_KEYS[1:], sort=False, dropna=False): if ( group["timestamp"].iloc[1:].to_numpy() < group["interval_end"].iloc[:-1].to_numpy() ).any(): raise RetrievalError( "ENTSO-E returned overlapping intervals with different resolutions" ) return result
[docs] def normalize_generation( xml: str | bytes, *, maximum_rows: int = 1_000_000, maximum_bytes: int = 32 * 1024 * 1024, ) -> pd.DataFrame: """Parse an A75/A73 realised-generation XML document without network access. A01 absent positions are represented by NaN. A03 changes are expanded only within their own Period; an explicit missing quantity stops the prior value. Generation and consumption remain separate positive/negative-as-published flows. Values are never clipped, imputed, or summed across these flows. An acknowledgement with reason 999 and the provider's explicit no-matching- data message returns an empty frame. Other acknowledgements fail, without echoing provider text that might contain credential-bearing request URLs. """ _integer(maximum_rows, "maximum_rows", 10_000_000) _integer(maximum_bytes, "maximum_bytes", 256 * 1024 * 1024) raw = xml.encode("utf-8") if isinstance(xml, str) else xml if len(raw) > maximum_bytes: raise RetrievalError("ENTSO-E XML exceeded the configured byte limit") try: text = raw.decode("utf-8-sig") except UnicodeDecodeError: raise RetrievalError("ENTSO-E XML must use UTF-8 encoding") from None # Generation exports need neither DTDs nor entities. Refuse both before parsing. if "<!DOCTYPE" in text.upper() or "<!ENTITY" in text.upper() or "\x00" in text: raise RetrievalError("ENTSO-E XML must not contain a DTD or entity declaration") try: root = ET.fromstring(text) except (ET.ParseError, ValueError): raise RetrievalError( "ENTSO-E returned invalid XML; compressed exports are unsupported" ) from None name = root.tag.rsplit("}", 1)[-1] if name == "Acknowledgement_MarketDocument": reasons = root.findall("{*}Reason") if reasons and all( _text(reason, "code") == "999" and "no matching data" in _text(reason, "text").lower() for reason in reasons ): return _empty(no_data=True) raise RetrievalError("ENTSO-E rejected the query; check token, area, dates and parameters") document_type = _text(root, "type", required=True) if ( name != "GL_MarketDocument" or document_type not in {"A73", "A75"} or _text(root, "process.processType", required=True) != "A16" ): raise RetrievalError("Expected ENTSO-E realised generation document A73 or A75 / A16") document_id = _text(root, "mRID", required=True) revision = _text(root, "revisionNumber", required=True) rows: list[dict[str, object]] = [] series_list = root.findall("{*}TimeSeries") if not series_list: raise RetrievalError("ENTSO-E generation document contains no TimeSeries") for series in series_list: if _text(series, "cancelledTS"): raise RetrievalError("ENTSO-E returned a cancelled TimeSeries; inspect its revision") curve = _text(series, "curveType", required=True) if curve not in {"A01", "A03"}: raise RetrievalError("ENTSO-E generation supports A01 and A03 curves only") native_unit = _text(series, "quantity_Measure_Unit.name", required=True) if native_unit != "MAW": raise RetrievalError("ENTSO-E generation must use the native MAW (MW) unit") in_area = _text(series, "inBiddingZone_Domain.mRID") out_area = _text(series, "outBiddingZone_Domain.mRID") if bool(in_area) == bool(out_area): raise RetrievalError("ENTSO-E generation requires one unambiguous in/out domain") area = in_area or out_area technology = _text(series, "MktPSRType/psrType", required=True) resource = "MktPSRType/PowerSystemResources/" unit_id = _text(series, resource + "mRID") if document_type == "A73" and not unit_id: raise RetrievalError("ENTSO-E per-unit generation lacks its resource EIC") base: dict[str, object] = { "source": "entsoe", "area": area, "unit_id": unit_id, "unit_name": _text(series, resource + "name"), "technology": technology, "technology_name": PSR_TYPES.get(technology, technology), "flow": "generation" if in_area else "consumption", "unit": "MW", "native_unit": native_unit, "document_type": document_type, "document_id": document_id, "revision": revision, "series_id": _text(series, "mRID", required=True), "curve_type": curve, "business_type": _text(series, "businessType"), } periods = series.findall("{*}Period") if not periods: raise RetrievalError("ENTSO-E TimeSeries contains no Period") for period in periods: first = _stamp(_text(period, "timeInterval/start", required=True)) last = _stamp(_text(period, "timeInterval/end", required=True)) step = _resolution(_text(period, "resolution", required=True)) count, remainder = divmod(last - first, step) if count < 1 or remainder != timedelta(0): raise RetrievalError("ENTSO-E Period boundaries do not match its resolution") if len(rows) + count > maximum_rows: raise RetrievalError("ENTSO-E expanded data exceeded maximum_rows") points: dict[int, tuple[float, str]] = {} for point in period.findall("{*}Point"): try: position = int(_text(point, "position", required=True)) except ValueError: raise RetrievalError("ENTSO-E Point has an invalid position") from None if not 1 <= position <= count or position in points: raise RetrievalError( "ENTSO-E Point position is duplicated or outside its Period" ) quantity = _text(point, "quantity") reason = _text(point, "Reason/code") if quantity: try: value = float(quantity) except ValueError: raise RetrievalError( "ENTSO-E Point contains a nonnumeric quantity" ) from None if not math.isfinite(value): raise RetrievalError("ENTSO-E Point contains a non-finite quantity") points[position] = value, reason else: points[position] = math.nan, reason or "not_provided" current = (math.nan, "not_provided") for position in range(1, count + 1): if curve == "A01": current = points.get(position, (math.nan, "not_provided")) elif position in points: current = points[position] timestamp = first + (position - 1) * step rows.append( base | { "timestamp": timestamp, "interval_end": timestamp + step, "duration_hours": step.total_seconds() / 3600, "value": current[0], "missing_reason": current[1], } ) frame = _deduplicate(pd.DataFrame(rows, columns=list(_COLUMNS))) frame["timestamp"] = pd.to_datetime(frame["timestamp"], utc=True) frame["interval_end"] = pd.to_datetime(frame["interval_end"], utc=True) frame.attrs.update( source="entsoe", no_data=False, timestamp_convention="interval_start", quantity="interval_average_net_power", missing_values_filled=False, a03_expansion="constant blocks within each Period; explicit missing blocks preserved", reference_ids=list(REFERENCE_IDS), ) return frame
[docs] @dataclass(frozen=True, slots=True) class ENTSOEGeneration: """Long-form interval-average power and sanitized HTTP request provenance.""" data: pd.DataFrame requests: tuple[RequestRecord, ...]
[docs] @dataclass(slots=True) class ENTSOEClient: """Authenticated actual-generation retrieval; never requests forecasts.""" http: HTTPClient token: str = field(default="", repr=False) maximum_requests: int = 366 maximum_rows: int = 1_000_000 def __post_init__(self) -> None: _integer(self.maximum_requests, "maximum_requests", 10000) _integer(self.maximum_rows, "maximum_rows", 10_000_000)
[docs] @classmethod def from_environment(cls, http: HTTPClient) -> ENTSOEClient: return cls(http=http, token=os.environ.get("ENTSOE_API_TOKEN", ""))
[docs] def generation_by_type( self, area: str, start: datetime, end: datetime, *, psr_type: str | None = None, ) -> ENTSOEGeneration: """Actual generation per production type (A75/A16), with consumption separate.""" return self._generation(area, start, end, psr_type=psr_type)
[docs] def generation_by_unit( self, area: str, start: datetime, end: datetime, *, psr_type: str | None = None, unit_id: str | None = None, ) -> ENTSOEGeneration: """Actual generation per unit (A73/A16), queried in one-day chunks.""" return self._generation(area, start, end, per_unit=True, psr_type=psr_type, unit_id=unit_id)
def _generation( self, area: str, start: datetime, end: datetime, *, per_unit: bool = False, psr_type: str | None = None, unit_id: str | None = None, ) -> ENTSOEGeneration: planned = generation_requests( area, start, end, per_unit=per_unit, psr_type=psr_type, unit_id=unit_id, maximum_requests=self.maximum_requests, ) if not self.token: raise ValueError("ENTSOE_API_TOKEN must contain a personal token with API access") first, last = _utc(start), _utc(end) frames: list[pd.DataFrame] = [] records: list[RequestRecord] = [] row_count = 0 for parameters in planned: response = self.http.get_text( ENTSOE_ENDPOINT, parameters, secrets={"securityToken": self.token}, use_cache=False, ) frame = normalize_generation(response.value, maximum_rows=self.maximum_rows) if not frame.empty: if ( not frame["area"].eq(area).all() or not frame["document_type"].eq(parameters["documentType"]).all() or (psr_type is not None and not frame["technology"].eq(psr_type).all()) or (unit_id is not None and not frame["unit_id"].eq(unit_id).all()) ): raise RetrievalError( "ENTSO-E returned data outside the requested area or resource" ) # Clip only to the caller's outer boundaries. ENTSO-E may # return whole MTUs overlapping an artificial chunk seam; # clipping to each chunk would discard such an MTU twice. # Equal overlap is removed once when all chunks are combined. frame = frame.loc[ (frame["timestamp"] >= first) & (frame["interval_end"] <= last) ].copy() row_count += len(frame) if row_count > self.maximum_rows: raise RetrievalError("ENTSO-E query exceeded maximum_rows across its chunks") if not frame.empty: frames.append(frame) records.append(response.record) result = ( _deduplicate(pd.concat(frames, ignore_index=True)) if frames else _empty(no_data=True) ) result.attrs.update( source="entsoe", timestamp_convention="interval_start", quantity="interval_average_net_power", requested_start_utc=first.isoformat(), requested_end_utc=last.isoformat(), interval_selection="only whole native intervals contained in the requested range", reference_ids=list(REFERENCE_IDS), ) return ENTSOEGeneration(result, tuple(records))