"""Reproduce published Chile curtailment tables and analyze explicit CEN exports.
Run ``python -m energy_analysis.chile_curtailment --output artifacts/chile-curtailment``.
Offline mode uses published aggregates from DOI 10.1016/j.est.2026.125203.
An optional hourly configuration uses separately acquired CEN data, never a
fabricated reconstruction of the authors' unavailable hourly dataset.
"""
from __future__ import annotations
import argparse
import hashlib
import json
from dataclasses import asdict
from importlib.resources import files
from pathlib import Path
import pandas as pd
from energy_data_retrieval.sources.cen.curtailment import (
CENHourlyExport,
CENHourlySchema,
import_cen_hourly_csv,
)
from energy_methods.curtailment_storage import (
classify_curtailment,
curtailment_ratios,
equivalent_storage,
nodal_price_screen,
storage_sensitivity,
)
[docs]
def reproduce_published_tables() -> dict[str, pd.DataFrame]:
"""Recompute Tables 2/3/7/8 with explicit rounding residuals and tolerances.
Uses 365 days even for 2024, matching the paper's convention. This is an
aggregate reproduction, not an independent raw-data replication.
"""
paper = json.loads(
files("energy_analysis").joinpath("data/chile_curtailment_2027.json").read_text()
)
annual, durations, audit = [], [], []
def check(name: str, calculated: float | None, reported: float, tolerance: float) -> None:
if calculated is None:
raise ValueError(f"Undefined paper comparison: {name}")
audit.append(
{
"metric": name,
"calculated": calculated,
"reported": reported,
"residual": calculated - reported,
"absolute_tolerance": tolerance,
"within_tolerance": abs(calculated - reported) <= tolerance,
"expected_agreement": name != "2024_total_pct",
"matches_expected_agreement": (
(abs(calculated - reported) <= tolerance) == (name != "2024_total_pct")
),
}
)
for item in paper["annual"]:
year = item["year"]
curtailed = item["pv_curtailment_gwh"] + item["wind_curtailment_gwh"]
generated = item["pv_generation_gwh"] + item["wind_generation_gwh"]
check(f"{year}_total_gwh", curtailed, item["reported_total_gwh"], 0.11)
ratios = curtailment_ratios(curtailed, generated)
check(
f"{year}_total_pct",
ratios["curtailed_over_generated_pct"],
item["reported_total_pct"],
0.01,
)
for technology in ("pv", "wind"):
tech = curtailment_ratios(
item[f"{technology}_curtailment_gwh"], item[f"{technology}_generation_gwh"]
)
check(
f"{year}_{technology}_pct",
tech["curtailed_over_generated_pct"],
item[f"reported_{technology}_pct"],
0.01,
)
baseline = equivalent_storage(curtailed * 1000, installed_mw=item["installed_bess_mw"])
annual.append(
{
"year": year,
"pv_curtailment_gwh": item["pv_curtailment_gwh"],
"wind_curtailment_gwh": item["wind_curtailment_gwh"],
"total_curtailment_gwh": curtailed,
"generation_gwh": generated,
**ratios,
"eesc_4h_mw": baseline.scenario_a_mw,
"installed_bess_mw": item["installed_bess_mw"],
"uncovered_4h_mw": baseline.uncovered_a_mw,
}
)
for duration, reported in zip((2, 3, 4, 5), item["reported_eesc_mw"], strict=True):
value = equivalent_storage(curtailed * 1000, duration_hours=duration).scenario_a_mw
durations.append({"year": year, "duration_hours": duration, "eesc_mw": value})
check(f"{year}_eesc_{duration}h_mw", value, reported, 1.0)
block = paper["block_b_2025"]
scenarios = equivalent_storage(
annual[-1]["total_curtailment_gwh"] * 1000,
block_mwh=block["total_gwh"] * 1000,
installed_mw=1714,
)
check("2025_scenario_a_mw", scenarios.scenario_a_mw, block["reported_scenario_a_mw"], 0.1)
check("2025_scenario_b_mw", scenarios.scenario_b_mw, block["reported_scenario_b_mw"], 0.1)
check(
"2025_reduction_pct",
scenarios.reduction_relative_to_a_pct,
block["reported_reduction_pct"],
0.05,
)
return {
"annual": pd.DataFrame(annual),
"duration": pd.DataFrame(durations),
"scenarios": pd.DataFrame([asdict(scenarios)]),
"audit": pd.DataFrame(audit),
"sensitivity": storage_sensitivity(annual[-1]["total_curtailment_gwh"] * 1000),
}
[docs]
def analyze_hourly_exports(
curtailment: CENHourlyExport,
*,
expected_entities: dict[str, list[str]],
prices: CENHourlyExport | None = None,
corridors: tuple[tuple[str, str], ...] = (),
timezone: str = "America/Santiago",
) -> dict[str, pd.DataFrame]:
"""Observed-hour diagnostics with a caller-verified stable PV/wind plant roster.
Every observed hour must include all expected plants; otherwise fail rather
than understate curtailment. Whole missing hours remain absent and are counted
in coverage. Changing historical rosters require separate time slices.
Price screens retain only paired corridor hours and expose excluded counts.
"""
frame = curtailment.data
if frame.attrs.get("quantity") != "curtailment":
raise ValueError("Expected a normalized curtailment export")
if set(expected_entities) != {"pv", "wind"} or any(not v for v in expected_entities.values()):
raise ValueError("Provide nonempty, verified pv and wind entity rosters")
roster = {
entity: technology
for technology, entities in expected_entities.items()
for entity in entities
}
if len(roster) != sum(map(len, expected_entities.values())):
raise ValueError("Roster entities must be unique across technologies")
if not frame["entity_id"].map(roster).eq(frame["technology"]).all():
raise ValueError("Source entity/technology does not match the verified roster")
if not frame.groupby("timestamp")["entity_id"].nunique().eq(len(roster)).all():
raise ValueError("Incomplete plant roster in one or more observed hours")
totals = frame.groupby(["timestamp", "technology"])["value"].sum().unstack("technology")
hourly = classify_curtailment(totals["pv"], totals["wind"], timezone=timezone)
seasonal = hourly.groupby("season").agg(observed_hours=("quadrant", "size"))
high = hourly["quadrant"].eq("IV").groupby(hourly["season"]).sum()
seasonal["high_stress_hours"] = high
seasonal["share_of_all_high_stress_pct"] = (
high * 100 / high.sum() if high.sum() else float("nan")
)
span_hours = int((hourly.index[-1] - hourly.index[0]).total_seconds() / 3600) + 1
coverage = pd.DataFrame(
[
{
"observed_hours": len(hourly),
"span_hours": span_hours,
"whole_missing_hours": span_hours - len(hourly),
"roster_size": len(roster),
"pv_p50_mwh": hourly.attrs["pv_p50_mwh"],
"wind_p50_mwh": hourly.attrs["wind_p50_mwh"],
}
]
)
results = {"hourly": hourly, "seasonal": seasonal, "coverage": coverage}
if prices is None and corridors:
raise ValueError("Corridors require a price export")
if prices is not None:
if prices.data.attrs.get("quantity") != "marginal_cost" or not corridors:
raise ValueError("Price analysis requires marginal_cost data and explicit corridors")
wide = prices.data.pivot(
index="timestamp", columns="entity_id", values="value"
).sort_index()
screens, price_coverage = [], []
for first, second in corridors:
if first == second or not {first, second}.issubset(wide.columns):
raise ValueError("Corridor requires two distinct observed nodes")
pair = wide[[first, second]].dropna()
screen = nodal_price_screen(pair[first], pair[second])
screen["first_node"], screen["second_node"] = first, second
screens.append(screen)
price_coverage.append(
{
"first_node": first,
"second_node": second,
"paired_hours": len(pair),
"unpaired_hours": int(wide[[first, second]].notna().any(axis=1).sum())
- len(pair),
"flagged_hours": int(screen["price_separation_flag"].sum()),
}
)
results["price_screens"] = pd.concat(screens)
results["price_coverage"] = pd.DataFrame(price_coverage)
return results
def _read_config_export(config: dict, base: Path) -> CENHourlyExport:
schema = dict(config["schema"])
if "technology_map" in schema:
schema["technology_map"] = tuple(tuple(pair) for pair in schema["technology_map"])
return import_cen_hourly_csv(
base / config["path"], CENHourlySchema(**schema), source_url=config["source_url"]
)
[docs]
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument(
"--hourly-config", type=Path, help="Explicit local CSV/schema/roster configuration"
)
args = parser.parse_args()
results = reproduce_published_tables()
if not results["audit"]["matches_expected_agreement"].all():
raise ValueError("Unexpected change in published aggregate agreement")
payload = files("energy_analysis").joinpath("data/chile_curtailment_2027.json").read_bytes()
manifest: dict[str, object] = {
"basis": "published aggregates; no raw CEN download in offline mode",
"reference_doi": "10.1016/j.est.2026.125203",
"fixture_sha256": hashlib.sha256(payload).hexdigest(),
"assumptions": json.loads(payload)["assumptions"],
"limitations": [
"EESC is effective-power screening, not feasible BESS sizing",
"No independent replication of hourly seasonal or statistical findings",
],
}
if args.hourly_config:
config = json.loads(args.hourly_config.read_text())
base = args.hourly_config.parent
curtailment = _read_config_export(config["curtailment"], base)
prices = _read_config_export(config["prices"], base) if "prices" in config else None
results.update(
analyze_hourly_exports(
curtailment,
expected_entities=config["expected_entities"],
prices=prices,
corridors=tuple(tuple(pair) for pair in config.get("corridors", [])),
timezone=config.get("timezone", "America/Santiago"),
)
)
manifest["hourly_inputs"] = [curtailment.provenance] + (
[prices.provenance] if prices else []
)
args.output.mkdir(parents=True, exist_ok=True)
outputs = {}
for name, frame in results.items():
frame.to_csv(
args.output / f"{name}.csv", index=name in {"hourly", "seasonal", "price_screens"}
)
frame.to_parquet(args.output / f"{name}.parquet")
for suffix in ("csv", "parquet"):
path = args.output / f"{name}.{suffix}"
outputs[path.name] = hashlib.sha256(path.read_bytes()).hexdigest()
manifest["output_sha256"] = outputs
(args.output / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n")
print(results["annual"].to_string(index=False))
print(results["scenarios"].to_string(index=False))
matched = int(results["audit"]["within_tolerance"].sum())
print(
f"{matched}/{len(results['audit'])} comparisons agree within tolerance; "
f"outputs: {args.output}"
)
print(
"Known source discrepancy: 2024 total is 19.0009% from table inputs, "
"versus 18.99% reported."
)
if __name__ == "__main__":
main()