Source code for energy_data_retrieval.sources.elexon.client

"""Half-hourly BM-unit metered energy from Elexon Insights B1610.

B1610 quantity is MWh, including negative demand and zero. It is published
through settlement, not live SCADA. Dates and settlement periods identify
46/48/50 half-hour civil days in Great Britain; source end labels are UTC.
"""

from __future__ import annotations

import math
import re
from dataclasses import dataclass
from datetime import UTC, date, datetime, time, timedelta
from zoneinfo import ZoneInfo

import pandas as pd

from energy_data_retrieval.generation import GenerationResult
from energy_data_retrieval.transport import (
    HTTPClient,
    JSONValue,
    RequestRecord,
    RetrievalError,
    array_value,
    object_value,
)

ELEXON_HOSTS = frozenset({"data.elexon.co.uk"})
GENERATION_ENDPOINT = "https://data.elexon.co.uk/bmrs/api/v1/datasets/B1610/stream"
REFERENCE_IDS = ("elexon_insights_api", "elexon_b1610_metered_volumes")
_LONDON = ZoneInfo("Europe/London")
_COLUMNS = [
    "timestamp",
    "interval_start",
    "interval_end",
    "source",
    "area",
    "unit_id",
    "national_grid_bm_unit_id",
    "value",
    "unit",
    "flow",
    "aggregation",
    "settlement_date",
    "settlement_period",
    "settlement_run",
    "source_end_time",
    "source_type",
]


def _midnight(day: date) -> datetime:
    return datetime.combine(day, time(), _LONDON).astimezone(UTC)


[docs] def normalize_generation(payload: JSONValue, *, maximum_rows: int = 1_000_000) -> pd.DataFrame: """Normalize a native B1610 stream without guessing away settlement revisions. One row identifies a BM unit and settlement interval. Exact duplicate rows collapse, but conflicting values or different runs for that same key raise: the endpoint normally returns the latest available run. All quantities are signed metered MWh; the API's generic ``psrType`` is not a fuel classification. """ if type(maximum_rows) is not int or not 1 <= maximum_rows <= 10_000_000: raise ValueError("maximum_rows must be in 1..10000000") observations = array_value(payload) if len(observations) > maximum_rows: raise RetrievalError("Elexon response exceeds maximum_rows") rows: list[dict[str, object]] = [] for raw in observations: item = object_value(raw) if item.get("dataset") != "B1610": raise RetrievalError("Expected Elexon B1610 actual metered volumes") label, identifier = item.get("settlementDate"), item.get("bmUnit") if not isinstance(label, str) or not isinstance(identifier, str) or not identifier: raise RetrievalError("Elexon omitted settlement date or BM unit") try: day = date.fromisoformat(label) start = _midnight(day) next_day = _midnight(day + timedelta(days=1)) except (ValueError, OverflowError): raise RetrievalError("Elexon returned an invalid settlement date") from None period = item.get("settlementPeriod") count = int((next_day - start).total_seconds() / 1800) if type(period) is not int or not 1 <= period <= count: raise RetrievalError("Elexon settlement period is invalid for the UK civil day") interval_start = start + timedelta(minutes=30 * (period - 1)) interval_end = interval_start + timedelta(minutes=30) end_label = item.get("halfHourEndTime") if not isinstance(end_label, str): raise RetrievalError("Elexon omitted halfHourEndTime") try: end_time = datetime.fromisoformat(end_label) except ValueError: raise RetrievalError("Elexon returned an invalid halfHourEndTime") from None # The native B1610 schema labels this UTC field without a trailing Z. if end_time.tzinfo is None: end_time = end_time.replace(tzinfo=UTC) if end_time.astimezone(UTC) != interval_end: raise RetrievalError("Elexon end label disagrees with the settlement period") if "quantity" not in item: raise RetrievalError("Elexon omitted quantity") value = item["quantity"] if value is not None and ( isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) ): raise RetrievalError("Elexon quantity must be finite numeric MWh or null") run = item.get("settlementRunType") if not isinstance(run, str) or not run: raise RetrievalError("Elexon omitted settlement run") rows.append( { "timestamp": interval_start, "interval_start": interval_start, "interval_end": interval_end, "source": "elexon", "area": "GB", "unit_id": identifier, "national_grid_bm_unit_id": item.get("nationalGridBmUnitId"), "value": float("nan") if value is None else float(value), "unit": "MWh", "flow": "net_generation_or_demand", "aggregation": "settlement_interval_energy", "settlement_date": label, "settlement_period": period, "settlement_run": run, "source_end_time": end_label, "source_type": item.get("psrType"), } ) frame = pd.DataFrame(rows, columns=_COLUMNS) for column in ("timestamp", "interval_start", "interval_end"): frame[column] = pd.to_datetime(frame[column], utc=True) frame = frame.astype({"value": "float64"}).drop_duplicates() if frame.duplicated(["unit_id", "timestamp"]).any(): raise RetrievalError("Conflicting Elexon observations or settlement revisions for one key") frame = frame.sort_values(["timestamp", "unit_id"]).reset_index(drop=True) frame.attrs.update( source="Elexon Insights B1610", unit="MWh", interval_minutes=30, scope="BM-unit signed metered energy, including consuming BM units; not live SCADA", timestamp_semantics="UTC interval start derived from UK settlement date and period", revision_policy="Latest available API settlement run retained; conflicting keys rejected", missing_policy="Null remains missing; no absent interval is filled", reference_ids=list(REFERENCE_IDS), ) return frame
[docs] @dataclass(slots=True) class ElexonClient: """No-key B1610 retrieval, bounded to six-hour UTC request chunks.""" http: HTTPClient maximum_requests: int = 64 maximum_rows: int = 1_000_000 def __post_init__(self) -> None: if type(self.maximum_requests) is not int or not 1 <= self.maximum_requests <= 1000: raise ValueError("maximum_requests must be in 1..1000") if type(self.maximum_rows) is not int or not 1 <= self.maximum_rows <= 10_000_000: raise ValueError("maximum_rows must be in 1..10000000")
[docs] def generation(self, start: date, end: date, *, bm_unit: str | None = None) -> GenerationResult: """Fetch inclusive UK settlement dates, optionally selecting one exact BM unit. The API uses inclusive interval-start bounds. Six-hour requests avoid one unbounded all-unit response; maximum_rows also limits the combined result. No BM-unit registry or generation technology is inferred. """ if type(start) is not date or type(end) is not date or start > end or end.year > 9998: raise ValueError("Use ordered inclusive dates with end year <= 9998") if bm_unit is not None and not re.fullmatch(r"[A-Za-z0-9_-]{1,64}", bm_unit): raise ValueError("bm_unit must be an exact Elexon BM-unit identifier") first, until = _midnight(start), _midnight(end + timedelta(days=1)) chunks = math.ceil((until - first).total_seconds() / 21600) if chunks > self.maximum_requests: raise ValueError("Elexon date range exceeds maximum_requests") frames: list[pd.DataFrame] = [] records: list[RequestRecord] = [] count = 0 current = first while current < until: stop = min(current + timedelta(hours=6), until) query = {"from": current.isoformat(), "to": (stop - timedelta(minutes=30)).isoformat()} if bm_unit is not None: query["bmUnit"] = bm_unit response = self.http.get_json(GENERATION_ENDPOINT, query) frame = normalize_generation(response.value, maximum_rows=self.maximum_rows) if not frame.empty and ( not frame["timestamp"].between(current, stop, inclusive="left").all() or (bm_unit is not None and not frame["unit_id"].eq(bm_unit).all()) ): raise RetrievalError("Elexon returned observations outside the requested selection") count += len(frame) if count > self.maximum_rows: raise RetrievalError("Elexon retrieval exceeds maximum_rows") frames.append(frame) records.append(response.record) current = stop metadata = frames[0].attrs.copy() combined = pd.concat(frames, ignore_index=True).sort_values(["timestamp", "unit_id"]) if combined.duplicated(["timestamp", "unit_id"]).any(): raise RetrievalError("Elexon returned overlapping chunk observations") combined = combined.reset_index(drop=True) combined.attrs.update(metadata) return GenerationResult(combined, tuple(records))