Source code for energy_data_retrieval.sources.aemo.client

"""Public AEMO NEMWeb dispatch-unit SCADA, preserving instantaneous MW.

``timestamp`` is the published SETTLEMENTDATE dispatch label converted from
fixed AEST (UTC+10) to UTC. SCADAVALUE is a snapshot at the start of that
dispatch interval, not a five-minute average or settlement energy.
"""

from __future__ import annotations

import csv
import io
import math
import re
from collections.abc import Iterator, Sequence
from dataclasses import dataclass
from datetime import date, timedelta, timezone
from html.parser import HTMLParser
from pathlib import Path
from tempfile import TemporaryDirectory
from urllib.parse import urljoin, urlsplit
from zipfile import BadZipFile, ZipFile

import pandas as pd

from energy_data_retrieval.transport import HTTPClient, RequestRecord, RetrievalError

AEMO_HOSTS = frozenset({"nemweb.com.au", "www.nemweb.com.au"})
REFERENCE_IDS = ("aemo_dispatch_scada", "aemo_nemweb", "aemo_market_time")
CURRENT_ENDPOINT = "https://nemweb.com.au/Reports/CURRENT/Dispatch_SCADA/"
ARCHIVE_ENDPOINT = "https://nemweb.com.au/Reports/ARCHIVE/Dispatch_SCADA/"
DOCUMENTATION_ENDPOINT = (
    "https://visualisations.aemo.com.au/aemo/nemweb/MMSDataModelReport/"
    "Electricity/MMS%20Data%20Model%20Report_files/MMS_123.htm"
)
NEM_TIMEZONE = timezone(timedelta(hours=10), name="AEST")
_FILE = re.compile(r"PUBLIC_DISPATCHSCADA_\d{8}(?:\d{4}_\d+)?\.zip", re.IGNORECASE)
# Real registered identifiers include TARONG#1 and W/HOE#1.
_DUID = re.compile(r"[A-Za-z0-9_/#-]{1,20}")
_REQUIRED = {"SETTLEMENTDATE", "DUID", "SCADAVALUE"}


def _positive_integer(value: int, name: str) -> None:
    if type(value) is not int or value < 1:
        raise ValueError(f"{name} must be a positive integer")


def _duids(values: Sequence[str] | None) -> tuple[str, ...] | None:
    if values is None:
        return None
    if isinstance(values, str) or not values or len(values) > 10000:
        raise ValueError("duids must be a nonempty sequence of at most 10000 DUID strings")
    if any(not isinstance(value, str) or not _DUID.fullmatch(value) for value in values):
        raise ValueError("Invalid AEMO DUID")
    if len(set(values)) != len(values):
        raise ValueError("duids must be unique")
    return tuple(values)


def _file_url(url: str) -> str:
    parsed = urlsplit(url)
    if (
        parsed.scheme != "https"
        or parsed.hostname not in AEMO_HOSTS
        or parsed.port not in (None, 443)
        or parsed.username is not None
        or parsed.password is not None
        or parsed.query
        or parsed.fragment
    ):
        raise ValueError("Use a public query-free HTTPS AEMO NEMWeb archive URL")
    parts = parsed.path.split("/")
    if (
        len(parts) != 5
        or parts[1].lower() != "reports"
        or parts[2].lower() not in {"current", "archive"}
        or parts[3].lower() != "dispatch_scada"
        or not _FILE.fullmatch(parts[4])
    ):
        raise ValueError("URL must identify a PUBLIC_DISPATCHSCADA ZIP in its source directory")
    return url


class _Links(HTMLParser):
    def __init__(self) -> None:
        super().__init__(convert_charrefs=True)
        self.links: list[str] = []

    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
        if tag.lower() == "a":
            self.links.extend(value for name, value in attrs if name == "href" and value)


[docs] @dataclass(frozen=True, slots=True) class AEMOListing: """A current directory snapshot, sorted by source filename, with provenance.""" urls: tuple[str, ...] request: RequestRecord
[docs] @dataclass(frozen=True, slots=True) class AEMOGeneration: """Normalized observations and the exact downloaded archive's provenance.""" frame: pd.DataFrame request: RequestRecord
@dataclass(slots=True) class _ReadBudget: maximum_bytes: int maximum_members: int bytes_read: int = 0 members_read: int = 0 def _csv_members( data: bytes, budget: _ReadBudget, *, name: str = "input.csv", depth: int = 0 ) -> Iterator[tuple[str, str]]: if not data.startswith(b"PK"): try: yield name, data.decode("utf-8-sig") except UnicodeDecodeError: raise RetrievalError("AEMO CSV is not valid UTF-8") from None return if depth >= 2: raise RetrievalError("AEMO ZIP nesting exceeds two archive levels") try: with ZipFile(io.BytesIO(data)) as archive: for info in archive.infolist(): if info.is_dir(): continue budget.members_read += 1 if budget.members_read > budget.maximum_members: raise RetrievalError("AEMO ZIP exceeds its member bound") if not info.filename.lower().endswith((".csv", ".zip")): raise RetrievalError("AEMO ZIP contains an unsupported member type") remaining = budget.maximum_bytes - budget.bytes_read if info.file_size > remaining: raise RetrievalError("AEMO ZIP exceeds its uncompressed byte bound") with archive.open(info) as stream: body = stream.read(remaining + 1) budget.bytes_read += len(body) if budget.bytes_read > budget.maximum_bytes: raise RetrievalError("AEMO ZIP exceeds its uncompressed byte bound") # Nothing is extracted to disk; member paths are only provenance labels. yield from _csv_members( body, budget, name=f"{name}!{info.filename}", depth=depth + 1 ) except RetrievalError: raise except (BadZipFile, RuntimeError, OSError, NotImplementedError): raise RetrievalError("AEMO returned an invalid or unsupported ZIP archive") from None
[docs] def parse_dispatch_scada( content: str | bytes, *, duids: Sequence[str] | None = None, maximum_rows: int = 500000, maximum_uncompressed_bytes: int = 128 * 1024**2, maximum_members: int = 1000, ) -> pd.DataFrame: """Parse MMS CSV, an interval ZIP, or a daily ZIP of interval ZIPs. Native fields are retained under ``source:`` column names. Missing SCADA readings stay missing, and negative readings stay negative. Duplicate (dispatch label, DUID) keys fail instead of silently choosing a revision. The row bound applies before the optional DUID filter. No technology or generator/load classification is inferred from a DUID's spelling. """ selected = _duids(duids) for value, name in ( (maximum_rows, "maximum_rows"), (maximum_uncompressed_bytes, "maximum_uncompressed_bytes"), (maximum_members, "maximum_members"), ): _positive_integer(value, name) data = content.encode("utf-8") if isinstance(content, str) else content if len(data) > maximum_uncompressed_bytes: raise RetrievalError("AEMO input exceeds its byte bound") budget = _ReadBudget(maximum_uncompressed_bytes, maximum_members) rows: list[dict[str, object]] = [] total_rows = 0 seen: set[tuple[str, str]] = set() try: for member, text in _csv_members(data, budget): headers: dict[str, list[str]] = {} for fields in csv.reader(io.StringIO(text), strict=True): if fields[1:3] != ["DISPATCH", "UNIT_SCADA"]: continue if len(fields) < 4: raise RetrievalError("AEMO SCADA row has no version or columns") if fields[0] == "I": names = fields[4:] if not _REQUIRED.issubset(names) or len(names) != len(set(names)): raise RetrievalError("AEMO SCADA header omits required or unique columns") if fields[3] in headers and headers[fields[3]] != names: raise RetrievalError("AEMO SCADA schema changes within one member") headers[fields[3]] = names elif fields[0] == "D": names = headers.get(fields[3]) if names is None or len(fields) != len(names) + 4: raise RetrievalError( "AEMO SCADA row has no matching header or column count" ) total_rows += 1 if total_rows > maximum_rows: raise RetrievalError("AEMO SCADA exceeds its row bound") native = dict(zip(names, fields[4:], strict=True)) duid = native["DUID"] if not _DUID.fullmatch(duid): raise RetrievalError("AEMO SCADA contains an invalid DUID") key = (native["SETTLEMENTDATE"], duid) if key in seen: raise RetrievalError( "AEMO SCADA contains duplicate dispatch label/DUID keys" ) seen.add(key) if selected is not None and duid not in selected: continue raw_value = native["SCADAVALUE"].strip() try: value = float(raw_value) if raw_value else float("nan") except ValueError: raise RetrievalError("AEMO SCADA has a nonnumeric power value") from None if raw_value and not math.isfinite(value): raise RetrievalError("AEMO SCADA has a nonfinite power value") rows.append( { "timestamp": native["SETTLEMENTDATE"], "source": "aemo", "area": "NEM", "unit_id": duid, "technology": None, "value": value, "unit": "MW", "quantity_kind": "instantaneous_power", "source_member": member, **{f"source:{name}": raw for name, raw in native.items()}, } ) except csv.Error: raise RetrievalError("AEMO returned malformed MMS CSV") from None if not rows: raise RetrievalError("AEMO file contains no matching dispatch-unit SCADA observations") frame = pd.DataFrame(rows) if selected is not None and set(frame["unit_id"]) != set(selected): raise RetrievalError("One or more requested DUIDs are absent from the AEMO file") try: timestamps = pd.to_datetime(frame["timestamp"], format="%Y/%m/%d %H:%M:%S", errors="raise") if ( timestamps.isna().any() or (timestamps.dt.minute % 5 != 0).any() or (timestamps.dt.second != 0).any() ): raise ValueError("Invalid dispatch labels") frame["timestamp"] = timestamps.dt.tz_localize(NEM_TIMEZONE).dt.tz_convert("UTC") except (ValueError, TypeError): raise RetrievalError( "AEMO dispatch labels must be valid five-minute NEM timestamps" ) from None frame = frame.sort_values(["timestamp", "unit_id"], ignore_index=True) if frame.duplicated(["timestamp", "unit_id"]).any(): raise RetrievalError("AEMO SCADA contains duplicate dispatch label/DUID keys") frame.attrs.update( source="AEMO NEMWeb DISPATCH_UNIT_SCADA", reference_ids=list(REFERENCE_IDS), native_timezone="AEST (UTC+10), without daylight saving", timestamp_convention="Published SETTLEMENTDATE dispatch label, converted to UTC", measurement_convention="Instantaneous MW at the start of the dispatch interval", interval_average=False, settlement_energy=False, source_rows_before_filter=total_rows, selected_duids=list(selected) if selected else None, gap_filling_applied=False, negative_values_clipped=False, documentation_url=DOCUMENTATION_ENDPOINT, coverage_note=( "Scheduled and semi-scheduled units, plus non-scheduled units with SCADA; " "not complete distributed generation and not restricted to generator DUIDs" ), ) return frame
[docs] @dataclass(slots=True) class AEMOClient: """Read public NEMWeb files using the shared bounded HTTP transport. No API key is required. Directory listings and current files are transient; retain the returned source URL, retrieval date and SHA-256 with your output. """ http: HTTPClient maximum_rows: int = 500000 maximum_uncompressed_bytes: int = 128 * 1024**2 maximum_members: int = 1000 maximum_files: int = 5000 def __post_init__(self) -> None: for name in ( "maximum_rows", "maximum_uncompressed_bytes", "maximum_members", "maximum_files", ): _positive_integer(getattr(self, name), name)
[docs] def list_files(self, *, archive: bool = False) -> AEMOListing: """Discover current interval ZIPs or available daily archive ZIPs. Only links to matching files in the requested official directory are returned. The provider's archive retention is finite; older files may require a different historical source. """ endpoint = ARCHIVE_ENDPOINT if archive else CURRENT_ENDPOINT response = self.http.get_text(endpoint, use_cache=False) parser = _Links() parser.feed(response.value) parser.close() urls: set[str] = set() for link in parser.links: candidate = urljoin(endpoint, link) try: _file_url(candidate) except ValueError: continue if urlsplit(candidate).path.rsplit("/", 1)[0].lower() != ( urlsplit(endpoint).path.rstrip("/").lower() ): continue urls.add(candidate) if len(urls) > self.maximum_files: raise RetrievalError("AEMO directory exceeds its file bound") if not urls: raise RetrievalError("AEMO directory lists no supported dispatch SCADA files") return AEMOListing(tuple(sorted(urls)), response.record)
[docs] def fetch_file(self, url: str, *, duids: Sequence[str] | None = None) -> AEMOGeneration: """Download and parse one validated interval or daily archive URL.""" _file_url(url) _duids(duids) with TemporaryDirectory(prefix="energy-aemo-") as directory: path = Path(directory) / "dispatch.zip" record = self.http.download_file(url, path) frame = parse_dispatch_scada( path.read_bytes(), duids=duids, maximum_rows=self.maximum_rows, maximum_uncompressed_bytes=self.maximum_uncompressed_bytes, maximum_members=self.maximum_members, ) return AEMOGeneration(frame, record)
[docs] def generation_day(self, day: date, *, duids: Sequence[str] | None = None) -> AEMOGeneration: """Read a daily archive (NEM labels 00:05 through the next day's 00:00). Missing/expired archives raise an explicit retrieval error. A day is a native NEM market date, not a UTC calendar date. No completeness is implied when individual unit observations are absent. """ if type(day) is not date or not 1998 <= day.year <= 9998: raise ValueError("day must be a date in the AEMO NEM period") result = self.fetch_file( f"{ARCHIVE_ENDPOINT}PUBLIC_DISPATCHSCADA_{day:%Y%m%d}.zip", duids=duids ) start = pd.Timestamp(day).tz_localize(NEM_TIMEZONE).tz_convert("UTC") end = start + pd.Timedelta(days=1) if not ((result.frame["timestamp"] > start) & (result.frame["timestamp"] <= end)).all(): raise RetrievalError("AEMO archive contains dispatch labels outside the requested day") result.frame.attrs["nem_day"] = day.isoformat() return result