Source code for energy_data_retrieval.sources.adme.client

"""ADME raw SCADA: measured generation and dispatch limits remain distinct."""

from __future__ import annotations

import math
import re
from dataclasses import dataclass
from datetime import date, datetime, timedelta
from html.parser import HTMLParser

import pandas as pd

from energy_data_retrieval.generation import GenerationResult
from energy_data_retrieval.transport import HTTPClient, RequestRecord, RetrievalError

ADME_HOSTS = frozenset({"pronos.adme.com.uy"})
SCADA_ROOT = "https://pronos.adme.com.uy/svg/pronos/cache_series_scada/"
REFERENCE_IDS = ("adme_open_data", "adme_scada_archive")
_FILE = re.compile(r"scada_central_(\d+)_(\d{10})\.sas")


def _limit(value: int, name: str, upper: int) -> None:
    if type(value) is not int or not 1 <= value <= upper:
        raise ValueError(f"{name} must be an integer in 1..{upper}")


[docs] def parse_scada_sas( payload: str, *, central_id: int, power_series: str | None = None, source_timezone: str = "America/Montevideo", maximum_rows: int = 100_000, ) -> pd.DataFrame: """Read ADME's tab-separated native six-line-header SAS time series. The source's Excel/LibreOffice serial date is treated as a local clock label, converted to UTC without interval shifting. A fixed-offset override may be supplied for exports with a different clock convention. Ambiguous or nonexistent local times fail. Values below -10000 are ADME missing sentinels. Only the actual ``GETOTMEDP.AV`` channel is selected automatically; dispatch limits (``CONSIGNAP``) and forecasts cannot become measured generation. """ _limit(central_id, "central_id", 1_000_000) _limit(maximum_rows, "maximum_rows", 1_000_000) lines = payload.splitlines() if len(lines) < 7 or "<html" in payload[:200].lower() or "<?php" in payload[:200].lower(): raise RetrievalError("ADME response is not a native SCADA SAS file") try: count = int(lines[0].split("\t")[0]) fields = lines[1].split("\t")[:6] year, month, day, hour, minute, second = (int(x) for x in fields) first = datetime(year, month, day, hour, minute, second) period_hours = float(lines[2].split("\t")[0]) samples = int(lines[3].split("\t")[0]) int(lines[4].split("\t")[0]) except (ValueError, TypeError): raise RetrievalError("ADME SAS has an invalid native header") from None if not 1 <= count <= 1000 or not 1 <= samples <= maximum_rows: raise RetrievalError("ADME SAS series/sample count is invalid or exceeds maximum_rows") if not math.isfinite(period_hours) or not 0 < period_hours <= 24: raise RetrievalError("ADME SAS has an invalid sampling period") seconds = round(period_hours * 3600) if seconds < 1 or abs(period_hours * 3600 - seconds) > 0.01: raise RetrievalError("ADME SAS sampling period must resolve to whole seconds") labels = lines[5].split("\t") if len(labels) != count + 1 or labels[0].strip(): raise RetrievalError("ADME SAS names do not match the declared series count") names = [name.strip() for name in labels[1:]] if len([x for x in names if x]) != len(set(x for x in names if x)): raise RetrievalError("ADME SAS contains duplicate named series") actual = [name for name in names if name.endswith("GETOTMEDP.AV")] if power_series is None: if len(actual) != 1: raise RetrievalError( "ADME actual power channel is ambiguous; supply its documented power_series" ) power_series = actual[0] if power_series not in actual: raise RetrievalError( "ADME power_series must identify a GETOTMEDP.AV power channel, not a limit/forecast" ) data_lines = [line for line in lines[6:] if line.strip()] if len(data_lines) != samples: raise RetrievalError("ADME SAS sample count does not match its native header") rows: list[list[float]] = [] for line in data_lines: values = line.split("\t") if len(values) != count + 1: raise RetrievalError("ADME SAS row has an unexpected number of columns") try: row = [float(value) for value in values] except ValueError: raise RetrievalError("ADME SAS observations must be numeric") from None if not all(math.isfinite(value) for value in row): raise RetrievalError( "ADME SAS observations must be finite or use the documented missing sentinel" ) rows.append(row) native_names = [name or f"unnamed_series_{i + 1}" for i, name in enumerate(names)] frame = pd.DataFrame(rows, columns=["source_serial_datetime", *native_names]) try: local = pd.to_datetime( frame["source_serial_datetime"], unit="D", origin="1899-12-30" ).dt.round("s") if local.iloc[0] != first: raise ValueError("Header start does not match data") expected = pd.date_range(first, periods=samples, freq=pd.Timedelta(seconds=seconds)) if not local.eq(pd.Series(expected, index=local.index)).all(): raise ValueError("Irregular source clock labels") frame["timestamp"] = local.dt.tz_localize( source_timezone, ambiguous="raise", nonexistent="raise" ).dt.tz_convert("UTC") except Exception as exc: raise RetrievalError( "ADME SAS has invalid, irregular or ambiguous source clock labels" ) from exc for column in native_names: frame[column] = frame[column].mask(frame[column] < -10000) frame["source_datetime"] = local frame["source"] = "adme" frame["area"] = "Uruguay" frame["unit_id"] = str(central_id) frame["value"] = frame[power_series] frame["unit"] = "MW" frame["power_series"] = power_series frame["sampling_seconds"] = seconds frame["aggregation"] = "sampled_scada" frame["quality"] = frame["value"].map( lambda value: "missing" if pd.isna(value) else "operational_scada" ) limits = [name for name in names if name.endswith("CONSIGNAP.AV")] if len(limits) == 1: frame["dispatch_limit_mw"] = frame[limits[0]] frame["dispatch_limit_series"] = limits[0] if frame.duplicated(["unit_id", "timestamp"]).any(): raise RetrievalError("ADME SAS contains duplicate unit/time observations") frame.attrs.update( source_timezone=source_timezone, source_start=first.isoformat(), source_series=names, reference_ids=list(REFERENCE_IDS), timestamp_semantics=( "Native local SCADA sample label converted to UTC; interval boundaries are not inferred" ), missing_policy=( "Source values below -10000 are missing; other negatives and zero are retained" ), dispatch_limits="CONSIGNAP is a control limit, not generation or measured lost energy", ) return frame
class _Links(HTMLParser): def __init__(self) -> None: super().__init__() self.files: set[str] = set() def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: if tag != "a": return href = dict(attrs).get("href") if isinstance(href, str) and _FILE.fullmatch(href): self.files.add(href)
[docs] @dataclass(slots=True) class ADMEClient: """Discover official SCADA archive files, with one six-day chunk per file.""" http: HTTPClient maximum_requests: int = 32 maximum_rows: int = 100_000 def __post_init__(self) -> None: _limit(self.maximum_requests, "maximum_requests", 1000) _limit(self.maximum_rows, "maximum_rows", 1_000_000)
[docs] def generation( self, start: date, end: date, *, central_id: int, power_series: str | None = None, source_timezone: str = "America/Montevideo", ) -> GenerationResult: """Retrieve inclusive local dates for one plant from raw measured SCADA. The archive exposes rolling seven-day files. The latest file starting before each requested six-day chunk is selected, and actual file coverage is checked. Archive gaps fail explicitly; no forecast archive is used. """ _limit(central_id, "central_id", 1_000_000) if ( type(start) is not date or type(end) is not date or not date(2016, 1, 2) <= start <= end < date(9999, 1, 1) ): raise ValueError("ADME requires ordered inclusive dates from 2016-01-02") chunks = math.ceil(((end - start).days + 1) / 6) if chunks + 1 > self.maximum_requests: raise ValueError("ADME date range exceeds maximum_requests including archive discovery") listing = self.http.get_text(SCADA_ROOT) parser = _Links() parser.feed(listing.value) candidates: list[tuple[datetime, str]] = [] for filename in parser.files: match = _FILE.fullmatch(filename) if match is None or int(match[1]) != central_id: continue try: stamp = datetime.strptime(match[2], "%Y%m%d%H") except ValueError: continue candidates.append((stamp, filename)) records: list[RequestRecord] = [listing.record] frames: list[pd.DataFrame] = [] total = 0 for offset in range(chunks): chunk_start = start + timedelta(days=offset * 6) chunk_end = min(end, chunk_start + timedelta(days=5)) origin = datetime.combine(chunk_start, datetime.min.time()) eligible = [ item for item in candidates if origin - timedelta(days=1) <= item[0] <= origin ] if not eligible: raise RetrievalError( "ADME has no SCADA archive file covering the requested plant/date" ) stamp, filename = max(eligible) response = self.http.get_text(SCADA_ROOT + filename) frame = parse_scada_sas( response.value, central_id=central_id, power_series=power_series, source_timezone=source_timezone, maximum_rows=self.maximum_rows, ) if frame.attrs["source_start"] != stamp.isoformat(): raise RetrievalError("ADME file start differs from its discovered filename") exclusive_end = datetime.combine(chunk_end + timedelta(days=1), datetime.min.time()) step = timedelta(seconds=int(frame["sampling_seconds"].iloc[0])) if ( frame["source_datetime"].min() > origin or frame["source_datetime"].max() + step < exclusive_end ): raise RetrievalError("ADME archive file does not cover all requested local dates") selected = frame.loc[ frame["source_datetime"].between(origin, exclusive_end, inclusive="left") ].reset_index(drop=True) total += len(frame) if total > self.maximum_rows: raise RetrievalError("ADME query exceeds maximum_rows") frames.append(selected) records.append(response.record) result = pd.concat(frames, ignore_index=True) result.attrs.update(frames[0].attrs) if result.duplicated(["unit_id", "timestamp"]).any(): raise RetrievalError("ADME archive produced overlapping observations") return GenerationResult(result, tuple(records))