Source code for energy_data_retrieval.sources.cen.curtailment

"""Explicit CEN curtailment/generation/price CSV contracts; no guessed endpoints.

Source products: CEN ERNC reduction reports, SEN generation and preliminary real
marginal costs, as identified in Acosta et al., DOI 10.1016/j.est.2026.125203.
Downloads require a user-supplied observed public CSV URL. Excel/Qlik exports
must first be exported to long-form CSV with source labels and units preserved.
"""

from __future__ import annotations

import hashlib
import io
from dataclasses import asdict, dataclass
from datetime import UTC, datetime
from pathlib import Path
from typing import Literal
from urllib.parse import urlsplit

import numpy as np
import pandas as pd

from energy_data_retrieval.transport import HTTPClient

CEN_CURTAILMENT_HOSTS = frozenset({"www.coordinador.cl", "coordinador.cl", "sic.coordinador.cl"})
MAX_CSV_BYTES = 100_000_000
MAX_CSV_ROWS = 2_000_000


[docs] @dataclass(frozen=True) class CENHourlySchema: """Map an observed long-form export; no column names or time zones are guessed. ``timezone=None`` requires an explicit offset on every timestamp. Otherwise naive civil-clock labels use the declared zone and ambiguous/nonexistent DST labels fail. Only one-hour intervals are supported. End-labelled exports are shifted one real hour before local time-block classification. """ quantity: Literal["curtailment", "generation", "marginal_cost"] timestamp_column: str entity_column: str value_column: str unit: str timezone: str | None = None timestamp_label: Literal["start", "end"] = "start" technology_column: str | None = None technology_map: tuple[tuple[str, str], ...] = () region_column: str | None = None delimiter: str = "," decimal: str = "." def __post_init__(self) -> None: if self.quantity not in {"curtailment", "generation", "marginal_cost"}: raise ValueError("Unsupported CEN quantity") units = {"USD/MWh"} if self.quantity == "marginal_cost" else {"MWh", "GWh", "MW"} if self.unit not in units or self.timestamp_label not in {"start", "end"}: raise ValueError("Invalid unit or timestamp convention") if self.quantity != "marginal_cost" and not self.technology_column: raise ValueError("Energy exports require an explicit technology column and map") mapping = dict(self.technology_map) if self.technology_column and ( not mapping or len(mapping) != len(self.technology_map) or not set(mapping.values()).issubset({"pv", "wind"}) ): raise ValueError("Technology map must uniquely map source labels to pv or wind")
[docs] @dataclass(frozen=True) class CENHourlyExport: data: pd.DataFrame provenance: dict[str, object]
def _source_url(url: str) -> None: parsed = urlsplit(url) if ( parsed.scheme != "https" or parsed.hostname not in CEN_CURTAILMENT_HOSTS or parsed.username or parsed.password or parsed.query or parsed.fragment or parsed.port not in (None, 443) ): raise ValueError("Provide a public, query-free HTTPS CEN source URL")
[docs] def normalize_cen_hourly(frame: pd.DataFrame, schema: CENHourlySchema) -> pd.DataFrame: """Reject missing values, duplicate entity-hours and unresolved DST timestamps. All source rows must match the declared PV/wind mapping; mixed-technology files must be explicitly filtered upstream. No missing entity/hour is filled. ``value`` is MWh for energy and USD/MWh for prices. MW means hourly mean MW. """ required = {schema.timestamp_column, schema.entity_column, schema.value_column} required.update(c for c in (schema.technology_column, schema.region_column) if c) if frame.empty or not required.issubset(frame.columns): raise ValueError(f"Nonempty CSV requires columns: {sorted(required)}") if frame[list(required)].isna().any().any(): raise ValueError("Missing source values must be reconciled, not zero-filled") stamps = [] for raw in frame[schema.timestamp_column]: stamp = pd.Timestamp(str(raw)) if pd.isna(stamp): raise ValueError("Invalid timestamp") if stamp.tzinfo is None: if schema.timezone is None: raise ValueError("Naive timestamps require a declared source timezone") stamp = stamp.tz_localize(schema.timezone, ambiguous="raise", nonexistent="raise") stamps.append(stamp.tz_convert("UTC")) index = pd.DatetimeIndex(stamps) if ((index.minute != 0) | (index.second != 0) | (index.microsecond != 0)).any(): raise ValueError("Only hourly source intervals are supported") if schema.timestamp_label == "end": index -= pd.Timedelta(hours=1) values = pd.to_numeric(frame[schema.value_column], errors="raise").astype(float) if not np.isfinite(values).all() or (schema.quantity != "marginal_cost" and (values < 0).any()): raise ValueError("Energy must be nonnegative; all observations must be finite") result = pd.DataFrame( { "timestamp": index, "entity_id": frame[schema.entity_column].astype(str).to_numpy(), "value": values.to_numpy() * (1000 if schema.unit == "GWh" else 1), } ) if result["entity_id"].str.strip().eq("").any(): raise ValueError("Empty entity identity") if schema.technology_column: technologies = frame[schema.technology_column].astype(str).map(dict(schema.technology_map)) if technologies.isna().any(): raise ValueError("Unmapped technology label") result["technology"] = technologies.to_numpy() if schema.region_column: result["region"] = frame[schema.region_column].astype(str).to_numpy() if result.duplicated(["timestamp", "entity_id"]).any(): raise ValueError("Duplicate entity-hour observations require reconciliation") result = result.sort_values(["timestamp", "entity_id"]).reset_index(drop=True) result.attrs.update( quantity=schema.quantity, unit="USD/MWh" if schema.quantity == "marginal_cost" else "MWh", timestamp_convention="UTC interval start", interval_hours=1, coverage="observed rows only; roster and missing hours are not inferred", ) return result
def _parse_csv(payload: bytes, schema: CENHourlySchema, source_url: str) -> CENHourlyExport: _source_url(source_url) if len(payload) > MAX_CSV_BYTES: raise ValueError("CEN CSV exceeds bounded import size") frame = pd.read_csv( io.BytesIO(payload), sep=schema.delimiter, decimal=schema.decimal, dtype={schema.entity_column: str}, nrows=MAX_CSV_ROWS + 1, ) if len(frame) > MAX_CSV_ROWS: raise ValueError("CEN CSV exceeds bounded row count") data = normalize_cen_hourly(frame, schema) return CENHourlyExport( data, { "source_url": source_url, "input_sha256": hashlib.sha256(payload).hexdigest(), "imported_at": datetime.now(UTC).isoformat(), "schema": asdict(schema), "rows": len(data), "basis": "user-supplied CEN export; not independently authenticated", }, )
[docs] def import_cen_hourly_csv( path: Path, schema: CENHourlySchema, *, source_url: str ) -> CENHourlyExport: """Read a preserved CEN export with input hash and explicit source contract.""" if path.stat().st_size > MAX_CSV_BYTES: raise ValueError("CEN CSV exceeds bounded import size") return _parse_csv(path.read_bytes(), schema, source_url)
[docs] def fetch_cen_hourly_csv(http: HTTPClient, url: str, schema: CENHourlySchema) -> CENHourlyExport: """Fetch an observed public CSV URL through the repository's bounded transport.""" _source_url(url) response = http.get_text(url) result = _parse_csv(response.value.encode("utf-8"), schema, url) provenance = { **result.provenance, "request": asdict(response.record), "basis": "CEN CSV download", } return CENHourlyExport(result.data, provenance)