Source code for energy_data_retrieval.sources.ieso.client

"""Ontario's public hourly generator output, distinct from capability/forecasts."""

from __future__ import annotations

import math
import xml.etree.ElementTree as ET
from dataclasses import dataclass
from datetime import UTC, date, datetime, timedelta, timezone

import pandas as pd

from energy_data_retrieval.generation import GenerationResult
from energy_data_retrieval.transport import HTTPClient, RequestRecord, RetrievalError

IESO_HOSTS = frozenset({"reports-public.ieso.ca"})
REPORT_ROOT = "https://reports-public.ieso.ca/public/GenOutputCapability"
REFERENCE_IDS = ("ieso_generator_output", "ieso_report_directory")
_EST = timezone(timedelta(hours=-5))
_NS = {"i": "http://www.theIMO.com/schema"}
_COLUMNS = [
    "timestamp",
    "interval_end",
    "source",
    "area",
    "unit_id",
    "technology",
    "value",
    "unit",
    "source_date",
    "hour_ending",
    "source_created_at",
    "quality",
    "aggregation",
]


def _limit(value: int, name: str, upper: int) -> None:
    if type(value) is not int or not 1 <= value <= upper:
        raise ValueError(f"{name} must be an integer in 1..{upper}")


def _text(parent: ET.Element, path: str) -> str:
    value = parent.findtext(path, namespaces=_NS)
    if value is None or not value.strip():
        raise RetrievalError(f"IESO report omitted {path}")
    return value.strip()


[docs] def parse_generation_xml(payload: str, *, maximum_rows: int = 1_000_000) -> pd.DataFrame: """Parse only ``Outputs/Output/EnergyMW`` from a native IESO daily XML report. Source Hour 1 covers 00:00-01:00 and Hour 24 covers 23:00-24:00, in fixed Eastern Standard Time (UTC-05), including summer. ``timestamp`` is the UTC interval start. Published values are hourly averages of operational MW telemetry, not settlement-meter MWh. Missing telemetry remains NaN. """ _limit(maximum_rows, "maximum_rows", 10_000_000) if "<!DOCTYPE" in payload.upper() or "<!ENTITY" in payload.upper(): raise RetrievalError("IESO XML declarations/entities are not accepted") try: root = ET.fromstring(payload) except ET.ParseError: raise RetrievalError("IESO returned malformed XML") from None if root.get("docID") != "GenOutputCapability": raise RetrievalError("Expected IESO GenOutputCapability report") body = root.find("i:IMODocBody", _NS) if body is None: raise RetrievalError("IESO report has no body") label = _text(body, "i:Date") try: day = date.fromisoformat(label) except ValueError: raise RetrievalError("IESO report has an invalid date") from None origin = datetime.combine(day, datetime.min.time(), _EST) created = root.findtext("i:IMODocHeader/i:CreatedAt", default="", namespaces=_NS) generators = body.find("i:Generators", _NS) if generators is None: raise RetrievalError("IESO report omitted Generators") rows: list[dict[str, object]] = [] for generator in generators.findall("i:Generator", _NS): identifier = _text(generator, "i:GeneratorName") technology = _text(generator, "i:FuelType") outputs = generator.find("i:Outputs", _NS) if outputs is None: raise RetrievalError("IESO generator omitted actual Outputs") for output in outputs.findall("i:Output", _NS): if len(rows) >= maximum_rows: raise RetrievalError("IESO response exceeds maximum_rows") try: hour = int(_text(output, "i:Hour")) except ValueError: raise RetrievalError("IESO hour must be 1..24") from None if not 1 <= hour <= 24: raise RetrievalError("IESO hour must be 1..24") node = output.find("i:EnergyMW", _NS) # IESO publishes Hour-only Output elements for unavailable telemetry. # Their absence is distinct from an omitted Outputs container. raw = "" if node is None else (node.text or "").strip() if raw in {"", "N/A"}: value = float("nan") else: try: value = float(raw) except ValueError: raise RetrievalError("IESO EnergyMW must be numeric or N/A") from None if not math.isfinite(value): raise RetrievalError("IESO EnergyMW must be finite") rows.append( { "timestamp": (origin + timedelta(hours=hour - 1)).astimezone(UTC), "interval_end": (origin + timedelta(hours=hour)).astimezone(UTC), "source": "ieso", "area": "Ontario", "unit_id": identifier, "technology": technology, "value": value, "unit": "MW", "source_date": label, "hour_ending": hour, "source_created_at": created, "quality": "missing" if math.isnan(value) else "operational_telemetry", "aggregation": "hourly_mean", } ) frame = pd.DataFrame(rows, columns=_COLUMNS) for column in ("timestamp", "interval_end"): frame[column] = pd.to_datetime(frame[column], utc=True) frame["value"] = frame["value"].astype(float) frame = frame.drop_duplicates() if frame.duplicated(["unit_id", "timestamp"]).any(): raise RetrievalError("IESO returned conflicting unit/hour observations") frame = frame.sort_values(["timestamp", "unit_id"]).reset_index(drop=True) frame.attrs.update( source_timezone="UTC-05:00 (IESO fixed EST)", timestamp_semantics="UTC interval start derived from source hour ending", coverage="IESO participating facilities with maximum capability of at least 20 MW", missing_policy="N/A and empty telemetry remain missing; no gap filling", source_report_date=label, reference_ids=list(REFERENCE_IDS), ) return frame
[docs] @dataclass(slots=True) class IESOClient: """Fetch bounded daily reports from IESO's public recent XML archive.""" http: HTTPClient maximum_requests: int = 31 maximum_rows: int = 1_000_000 def __post_init__(self) -> None: _limit(self.maximum_requests, "maximum_requests", 1000) _limit(self.maximum_rows, "maximum_rows", 10_000_000)
[docs] def generation( self, start: date, end: date, *, unit_ids: list[str] | None = None ) -> GenerationResult: """Retrieve inclusive EST report dates; archive retention is provider-limited. The public XML archive normally covers 90 days. Older monthly CSV files are a separate product and are not silently substituted. A missing report raises a retrieval error instead of manufacturing zero output. """ if type(start) is not date or type(end) is not date or not start <= end < date(9999, 1, 1): raise ValueError("IESO requires ordered inclusive date objects") days = (end - start).days + 1 if days > self.maximum_requests: raise ValueError("IESO date range exceeds maximum_requests") if unit_ids is not None and ( not unit_ids or any(not isinstance(item, str) or not item for item in unit_ids) ): raise ValueError("unit_ids must be nonempty identifiers") frames: list[pd.DataFrame] = [] records: list[RequestRecord] = [] total = 0 for offset in range(days): day = start + timedelta(days=offset) response = self.http.get_text(f"{REPORT_ROOT}/PUB_GenOutputCapability_{day:%Y%m%d}.xml") frame = parse_generation_xml(response.value, maximum_rows=self.maximum_rows) if frame.attrs["source_report_date"] != day.isoformat(): raise RetrievalError("IESO returned a report outside the requested date") total += len(frame) if total > self.maximum_rows: raise RetrievalError("IESO query exceeds maximum_rows") frames.append(frame) records.append(response.record) result = pd.concat(frames, ignore_index=True) result.attrs.update(frames[0].attrs) if unit_ids is not None: if set(unit_ids) - set(result["unit_id"]): raise RetrievalError( "IESO requested unit_ids were absent from the returned reports" ) result = result.loc[result["unit_id"].isin(unit_ids)].reset_index(drop=True) return GenerationResult(result, tuple(records))