"""Explicit CEN curtailment/generation/price CSV contracts; no guessed endpoints.
Source products: CEN ERNC reduction reports, SEN generation and preliminary real
marginal costs, as identified in Acosta et al., DOI 10.1016/j.est.2026.125203.
Downloads require a user-supplied observed public CSV URL. Excel/Qlik exports
must first be exported to long-form CSV with source labels and units preserved.
"""
from __future__ import annotations
import hashlib
import io
from dataclasses import asdict, dataclass
from datetime import UTC, datetime
from pathlib import Path
from typing import Literal
from urllib.parse import urlsplit
import numpy as np
import pandas as pd
from energy_data_retrieval.transport import HTTPClient
CEN_CURTAILMENT_HOSTS = frozenset({"www.coordinador.cl", "coordinador.cl", "sic.coordinador.cl"})
MAX_CSV_BYTES = 100_000_000
MAX_CSV_ROWS = 2_000_000
[docs]
@dataclass(frozen=True)
class CENHourlySchema:
"""Map an observed long-form export; no column names or time zones are guessed.
``timezone=None`` requires an explicit offset on every timestamp. Otherwise
naive civil-clock labels use the declared zone and ambiguous/nonexistent
DST labels fail. Only one-hour intervals are supported. End-labelled exports
are shifted one real hour before local time-block classification.
"""
quantity: Literal["curtailment", "generation", "marginal_cost"]
timestamp_column: str
entity_column: str
value_column: str
unit: str
timezone: str | None = None
timestamp_label: Literal["start", "end"] = "start"
technology_column: str | None = None
technology_map: tuple[tuple[str, str], ...] = ()
region_column: str | None = None
delimiter: str = ","
decimal: str = "."
def __post_init__(self) -> None:
if self.quantity not in {"curtailment", "generation", "marginal_cost"}:
raise ValueError("Unsupported CEN quantity")
units = {"USD/MWh"} if self.quantity == "marginal_cost" else {"MWh", "GWh", "MW"}
if self.unit not in units or self.timestamp_label not in {"start", "end"}:
raise ValueError("Invalid unit or timestamp convention")
if self.quantity != "marginal_cost" and not self.technology_column:
raise ValueError("Energy exports require an explicit technology column and map")
mapping = dict(self.technology_map)
if self.technology_column and (
not mapping
or len(mapping) != len(self.technology_map)
or not set(mapping.values()).issubset({"pv", "wind"})
):
raise ValueError("Technology map must uniquely map source labels to pv or wind")
[docs]
@dataclass(frozen=True)
class CENHourlyExport:
data: pd.DataFrame
provenance: dict[str, object]
def _source_url(url: str) -> None:
parsed = urlsplit(url)
if (
parsed.scheme != "https"
or parsed.hostname not in CEN_CURTAILMENT_HOSTS
or parsed.username
or parsed.password
or parsed.query
or parsed.fragment
or parsed.port not in (None, 443)
):
raise ValueError("Provide a public, query-free HTTPS CEN source URL")
[docs]
def normalize_cen_hourly(frame: pd.DataFrame, schema: CENHourlySchema) -> pd.DataFrame:
"""Reject missing values, duplicate entity-hours and unresolved DST timestamps.
All source rows must match the declared PV/wind mapping; mixed-technology
files must be explicitly filtered upstream. No missing entity/hour is filled.
``value`` is MWh for energy and USD/MWh for prices. MW means hourly mean MW.
"""
required = {schema.timestamp_column, schema.entity_column, schema.value_column}
required.update(c for c in (schema.technology_column, schema.region_column) if c)
if frame.empty or not required.issubset(frame.columns):
raise ValueError(f"Nonempty CSV requires columns: {sorted(required)}")
if frame[list(required)].isna().any().any():
raise ValueError("Missing source values must be reconciled, not zero-filled")
stamps = []
for raw in frame[schema.timestamp_column]:
stamp = pd.Timestamp(str(raw))
if pd.isna(stamp):
raise ValueError("Invalid timestamp")
if stamp.tzinfo is None:
if schema.timezone is None:
raise ValueError("Naive timestamps require a declared source timezone")
stamp = stamp.tz_localize(schema.timezone, ambiguous="raise", nonexistent="raise")
stamps.append(stamp.tz_convert("UTC"))
index = pd.DatetimeIndex(stamps)
if ((index.minute != 0) | (index.second != 0) | (index.microsecond != 0)).any():
raise ValueError("Only hourly source intervals are supported")
if schema.timestamp_label == "end":
index -= pd.Timedelta(hours=1)
values = pd.to_numeric(frame[schema.value_column], errors="raise").astype(float)
if not np.isfinite(values).all() or (schema.quantity != "marginal_cost" and (values < 0).any()):
raise ValueError("Energy must be nonnegative; all observations must be finite")
result = pd.DataFrame(
{
"timestamp": index,
"entity_id": frame[schema.entity_column].astype(str).to_numpy(),
"value": values.to_numpy() * (1000 if schema.unit == "GWh" else 1),
}
)
if result["entity_id"].str.strip().eq("").any():
raise ValueError("Empty entity identity")
if schema.technology_column:
technologies = frame[schema.technology_column].astype(str).map(dict(schema.technology_map))
if technologies.isna().any():
raise ValueError("Unmapped technology label")
result["technology"] = technologies.to_numpy()
if schema.region_column:
result["region"] = frame[schema.region_column].astype(str).to_numpy()
if result.duplicated(["timestamp", "entity_id"]).any():
raise ValueError("Duplicate entity-hour observations require reconciliation")
result = result.sort_values(["timestamp", "entity_id"]).reset_index(drop=True)
result.attrs.update(
quantity=schema.quantity,
unit="USD/MWh" if schema.quantity == "marginal_cost" else "MWh",
timestamp_convention="UTC interval start",
interval_hours=1,
coverage="observed rows only; roster and missing hours are not inferred",
)
return result
def _parse_csv(payload: bytes, schema: CENHourlySchema, source_url: str) -> CENHourlyExport:
_source_url(source_url)
if len(payload) > MAX_CSV_BYTES:
raise ValueError("CEN CSV exceeds bounded import size")
frame = pd.read_csv(
io.BytesIO(payload),
sep=schema.delimiter,
decimal=schema.decimal,
dtype={schema.entity_column: str},
nrows=MAX_CSV_ROWS + 1,
)
if len(frame) > MAX_CSV_ROWS:
raise ValueError("CEN CSV exceeds bounded row count")
data = normalize_cen_hourly(frame, schema)
return CENHourlyExport(
data,
{
"source_url": source_url,
"input_sha256": hashlib.sha256(payload).hexdigest(),
"imported_at": datetime.now(UTC).isoformat(),
"schema": asdict(schema),
"rows": len(data),
"basis": "user-supplied CEN export; not independently authenticated",
},
)
[docs]
def import_cen_hourly_csv(
path: Path, schema: CENHourlySchema, *, source_url: str
) -> CENHourlyExport:
"""Read a preserved CEN export with input hash and explicit source contract."""
if path.stat().st_size > MAX_CSV_BYTES:
raise ValueError("CEN CSV exceeds bounded import size")
return _parse_csv(path.read_bytes(), schema, source_url)
[docs]
def fetch_cen_hourly_csv(http: HTTPClient, url: str, schema: CENHourlySchema) -> CENHourlyExport:
"""Fetch an observed public CSV URL through the repository's bounded transport."""
_source_url(url)
response = http.get_text(url)
result = _parse_csv(response.value.encode("utf-8"), schema, url)
provenance = {
**result.provenance,
"request": asdict(response.record),
"basis": "CEN CSV download",
}
return CENHourlyExport(result.data, provenance)