Source code for energy_data_retrieval.sources.eskom.client

"""Discover current Eskom hourly renewable exports without frozen upload URLs."""

from __future__ import annotations

import re
from dataclasses import dataclass
from datetime import date
from urllib.parse import urlsplit

import pandas as pd

from energy_data_retrieval.generation import GenerationResult
from energy_data_retrieval.sources._generation_utils import (
    bounded_csv,
    numeric,
    unique_observations,
    utc_labels,
)
from energy_data_retrieval.transport import HTTPClient, RetrievalError, array_value, object_value

ESKOM_HOSTS = frozenset({"www.eskom.co.za"})
MEDIA_ENDPOINT = "https://www.eskom.co.za/dataportal/wp-json/wp/v2/media"
DATASET_URL = (
    "https://www.eskom.co.za/dataportal/renewables-performance/hourly-renewable-generation/"
)
REFERENCE_IDS = ("eskom_hourly_renewables", "eskom_generation_statistics")
_TECHNOLOGIES = {
    "Wind": "wind",
    "PV": "solar_pv",
    "CSP": "solar_thermal",
    "Other RE": "other_renewable",
}


[docs] def parse_generation_csv(content: str | bytes, *, maximum_rows: int = 100_000) -> pd.DataFrame: """Read hourly reported MW, labelled at hour beginning in South African time. The product covers Eskom-owned/contracted renewables and excludes embedded generation. No energy integration or more precise sampling claim is made. """ wide = bounded_csv(content, maximum_rows=maximum_rows) label = "Date Time Hour Beginning" if not {label, *_TECHNOLOGIES}.issubset(wide.columns): raise RetrievalError("Unexpected Eskom hourly renewable export schema") stamp = utc_labels(wide[label], source_timezone="Africa/Johannesburg") if (stamp.dt.minute.ne(0) | stamp.dt.second.ne(0)).any(): raise RetrievalError("Eskom hourly labels must fall on the hour") frames = [] for column, technology in _TECHNOLOGIES.items(): frames.append( pd.DataFrame( { "timestamp": stamp, "source_datetime": wide[label], "source": "eskom", "area": "South Africa: Eskom-owned/contracted renewables", "technology": technology, "value": numeric(wide[column], column), "unit": "MW", "interval_minutes": 60, "measurement_kind": "reported_hourly_power", } ) ) result = unique_observations(pd.concat(frames, ignore_index=True), ["timestamp", "technology"]) result.attrs.update( source_url=DATASET_URL, timestamp_semantics="Hour beginning, Africa/Johannesburg (UTC+02:00)", scope="Eskom-owned or contracted renewables; embedded generation excluded", units_basis="MW contribution convention in Eskom renewable system-operator statistics", reference_ids=list(REFERENCE_IDS), ) return result
[docs] @dataclass(frozen=True, slots=True) class EskomClient: """Public media discovery and a bounded latest-export read. The provider's chart page can retain a stale monthly upload URL. The public WordPress media catalogue supplies current exact filenames. Discovery selects only Hourly_Generation exports, never the separate Total_Hourly product. """ http: HTTPClient maximum_rows: int = 100_000
[docs] def generation( self, start: date | None = None, end: date | None = None, *, url: str | None = None ) -> GenerationResult: """Fetch latest published file, optionally filter inclusive local dates. This endpoint does not promise an arbitrary historical archive. Missing requested days remain absent; frame metadata reports the returned range. An explicit URL permits an independently discovered official archive. """ if (start is None) != (end is None) or ( start is not None and end is not None and start > end ): raise ValueError("Supply both inclusive dates with start <= end, or neither") records = [] if url is None: catalogue = self.http.get_json( MEDIA_ENDPOINT, { "search": "Hourly_Generation", "per_page": "100", "orderby": "date", "order": "desc", }, ) records.append(catalogue.record) candidates: list[tuple[str, str]] = [] for item in array_value(catalogue.value): row = object_value(item) candidate = row.get("source_url") if isinstance(candidate, str) and re.search( r"/Hourly_Generation(?:-\d+)?\.csv$", candidate ): candidates.append((str(row.get("date", "")), candidate)) if not candidates: raise RetrievalError( "Eskom catalogue has no hourly renewable CSV; supply a verified official URL" ) url = max(candidates)[1] parsed = urlsplit(url) if ( parsed.scheme != "https" or parsed.hostname not in ESKOM_HOSTS or not parsed.path.startswith("/dataportal/wp-content/uploads/") or not re.search(r"/Hourly_Generation(?:-\d+)?\.csv$", parsed.path) or parsed.query or parsed.fragment ): raise ValueError("Use an official Eskom Hourly_Generation CSV URL") response = self.http.get_text(url) records.append(response.record) frame = parse_generation_csv(response.value, maximum_rows=self.maximum_rows) if start is not None and end is not None: local_dates = frame.timestamp.dt.tz_convert("Africa/Johannesburg").dt.date frame = frame.loc[local_dates.between(start, end)].reset_index(drop=True) frame.attrs.update(export_url=url, requested_start=str(start), requested_end=str(end)) return GenerationResult(frame, tuple(records))