Source code for energy_analysis.chile_curtailment

"""Reproduce published Chile curtailment tables and analyze explicit CEN exports.

Run ``python -m energy_analysis.chile_curtailment --output artifacts/chile-curtailment``.
Offline mode uses published aggregates from DOI 10.1016/j.est.2026.125203.
An optional hourly configuration uses separately acquired CEN data, never a
fabricated reconstruction of the authors' unavailable hourly dataset.
"""

from __future__ import annotations

import argparse
import hashlib
import json
from dataclasses import asdict
from importlib.resources import files
from pathlib import Path

import pandas as pd

from energy_data_retrieval.sources.cen.curtailment import (
    CENHourlyExport,
    CENHourlySchema,
    import_cen_hourly_csv,
)
from energy_methods.curtailment_storage import (
    classify_curtailment,
    curtailment_ratios,
    equivalent_storage,
    nodal_price_screen,
    storage_sensitivity,
)


[docs] def reproduce_published_tables() -> dict[str, pd.DataFrame]: """Recompute Tables 2/3/7/8 with explicit rounding residuals and tolerances. Uses 365 days even for 2024, matching the paper's convention. This is an aggregate reproduction, not an independent raw-data replication. """ paper = json.loads( files("energy_analysis").joinpath("data/chile_curtailment_2027.json").read_text() ) annual, durations, audit = [], [], [] def check(name: str, calculated: float | None, reported: float, tolerance: float) -> None: if calculated is None: raise ValueError(f"Undefined paper comparison: {name}") audit.append( { "metric": name, "calculated": calculated, "reported": reported, "residual": calculated - reported, "absolute_tolerance": tolerance, "within_tolerance": abs(calculated - reported) <= tolerance, "expected_agreement": name != "2024_total_pct", "matches_expected_agreement": ( (abs(calculated - reported) <= tolerance) == (name != "2024_total_pct") ), } ) for item in paper["annual"]: year = item["year"] curtailed = item["pv_curtailment_gwh"] + item["wind_curtailment_gwh"] generated = item["pv_generation_gwh"] + item["wind_generation_gwh"] check(f"{year}_total_gwh", curtailed, item["reported_total_gwh"], 0.11) ratios = curtailment_ratios(curtailed, generated) check( f"{year}_total_pct", ratios["curtailed_over_generated_pct"], item["reported_total_pct"], 0.01, ) for technology in ("pv", "wind"): tech = curtailment_ratios( item[f"{technology}_curtailment_gwh"], item[f"{technology}_generation_gwh"] ) check( f"{year}_{technology}_pct", tech["curtailed_over_generated_pct"], item[f"reported_{technology}_pct"], 0.01, ) baseline = equivalent_storage(curtailed * 1000, installed_mw=item["installed_bess_mw"]) annual.append( { "year": year, "pv_curtailment_gwh": item["pv_curtailment_gwh"], "wind_curtailment_gwh": item["wind_curtailment_gwh"], "total_curtailment_gwh": curtailed, "generation_gwh": generated, **ratios, "eesc_4h_mw": baseline.scenario_a_mw, "installed_bess_mw": item["installed_bess_mw"], "uncovered_4h_mw": baseline.uncovered_a_mw, } ) for duration, reported in zip((2, 3, 4, 5), item["reported_eesc_mw"], strict=True): value = equivalent_storage(curtailed * 1000, duration_hours=duration).scenario_a_mw durations.append({"year": year, "duration_hours": duration, "eesc_mw": value}) check(f"{year}_eesc_{duration}h_mw", value, reported, 1.0) block = paper["block_b_2025"] scenarios = equivalent_storage( annual[-1]["total_curtailment_gwh"] * 1000, block_mwh=block["total_gwh"] * 1000, installed_mw=1714, ) check("2025_scenario_a_mw", scenarios.scenario_a_mw, block["reported_scenario_a_mw"], 0.1) check("2025_scenario_b_mw", scenarios.scenario_b_mw, block["reported_scenario_b_mw"], 0.1) check( "2025_reduction_pct", scenarios.reduction_relative_to_a_pct, block["reported_reduction_pct"], 0.05, ) return { "annual": pd.DataFrame(annual), "duration": pd.DataFrame(durations), "scenarios": pd.DataFrame([asdict(scenarios)]), "audit": pd.DataFrame(audit), "sensitivity": storage_sensitivity(annual[-1]["total_curtailment_gwh"] * 1000), }
[docs] def analyze_hourly_exports( curtailment: CENHourlyExport, *, expected_entities: dict[str, list[str]], prices: CENHourlyExport | None = None, corridors: tuple[tuple[str, str], ...] = (), timezone: str = "America/Santiago", ) -> dict[str, pd.DataFrame]: """Observed-hour diagnostics with a caller-verified stable PV/wind plant roster. Every observed hour must include all expected plants; otherwise fail rather than understate curtailment. Whole missing hours remain absent and are counted in coverage. Changing historical rosters require separate time slices. Price screens retain only paired corridor hours and expose excluded counts. """ frame = curtailment.data if frame.attrs.get("quantity") != "curtailment": raise ValueError("Expected a normalized curtailment export") if set(expected_entities) != {"pv", "wind"} or any(not v for v in expected_entities.values()): raise ValueError("Provide nonempty, verified pv and wind entity rosters") roster = { entity: technology for technology, entities in expected_entities.items() for entity in entities } if len(roster) != sum(map(len, expected_entities.values())): raise ValueError("Roster entities must be unique across technologies") if not frame["entity_id"].map(roster).eq(frame["technology"]).all(): raise ValueError("Source entity/technology does not match the verified roster") if not frame.groupby("timestamp")["entity_id"].nunique().eq(len(roster)).all(): raise ValueError("Incomplete plant roster in one or more observed hours") totals = frame.groupby(["timestamp", "technology"])["value"].sum().unstack("technology") hourly = classify_curtailment(totals["pv"], totals["wind"], timezone=timezone) seasonal = hourly.groupby("season").agg(observed_hours=("quadrant", "size")) high = hourly["quadrant"].eq("IV").groupby(hourly["season"]).sum() seasonal["high_stress_hours"] = high seasonal["share_of_all_high_stress_pct"] = ( high * 100 / high.sum() if high.sum() else float("nan") ) span_hours = int((hourly.index[-1] - hourly.index[0]).total_seconds() / 3600) + 1 coverage = pd.DataFrame( [ { "observed_hours": len(hourly), "span_hours": span_hours, "whole_missing_hours": span_hours - len(hourly), "roster_size": len(roster), "pv_p50_mwh": hourly.attrs["pv_p50_mwh"], "wind_p50_mwh": hourly.attrs["wind_p50_mwh"], } ] ) results = {"hourly": hourly, "seasonal": seasonal, "coverage": coverage} if prices is None and corridors: raise ValueError("Corridors require a price export") if prices is not None: if prices.data.attrs.get("quantity") != "marginal_cost" or not corridors: raise ValueError("Price analysis requires marginal_cost data and explicit corridors") wide = prices.data.pivot( index="timestamp", columns="entity_id", values="value" ).sort_index() screens, price_coverage = [], [] for first, second in corridors: if first == second or not {first, second}.issubset(wide.columns): raise ValueError("Corridor requires two distinct observed nodes") pair = wide[[first, second]].dropna() screen = nodal_price_screen(pair[first], pair[second]) screen["first_node"], screen["second_node"] = first, second screens.append(screen) price_coverage.append( { "first_node": first, "second_node": second, "paired_hours": len(pair), "unpaired_hours": int(wide[[first, second]].notna().any(axis=1).sum()) - len(pair), "flagged_hours": int(screen["price_separation_flag"].sum()), } ) results["price_screens"] = pd.concat(screens) results["price_coverage"] = pd.DataFrame(price_coverage) return results
def _read_config_export(config: dict, base: Path) -> CENHourlyExport: schema = dict(config["schema"]) if "technology_map" in schema: schema["technology_map"] = tuple(tuple(pair) for pair in schema["technology_map"]) return import_cen_hourly_csv( base / config["path"], CENHourlySchema(**schema), source_url=config["source_url"] )
[docs] def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--output", type=Path, required=True) parser.add_argument( "--hourly-config", type=Path, help="Explicit local CSV/schema/roster configuration" ) args = parser.parse_args() results = reproduce_published_tables() if not results["audit"]["matches_expected_agreement"].all(): raise ValueError("Unexpected change in published aggregate agreement") payload = files("energy_analysis").joinpath("data/chile_curtailment_2027.json").read_bytes() manifest: dict[str, object] = { "basis": "published aggregates; no raw CEN download in offline mode", "reference_doi": "10.1016/j.est.2026.125203", "fixture_sha256": hashlib.sha256(payload).hexdigest(), "assumptions": json.loads(payload)["assumptions"], "limitations": [ "EESC is effective-power screening, not feasible BESS sizing", "No independent replication of hourly seasonal or statistical findings", ], } if args.hourly_config: config = json.loads(args.hourly_config.read_text()) base = args.hourly_config.parent curtailment = _read_config_export(config["curtailment"], base) prices = _read_config_export(config["prices"], base) if "prices" in config else None results.update( analyze_hourly_exports( curtailment, expected_entities=config["expected_entities"], prices=prices, corridors=tuple(tuple(pair) for pair in config.get("corridors", [])), timezone=config.get("timezone", "America/Santiago"), ) ) manifest["hourly_inputs"] = [curtailment.provenance] + ( [prices.provenance] if prices else [] ) args.output.mkdir(parents=True, exist_ok=True) outputs = {} for name, frame in results.items(): frame.to_csv( args.output / f"{name}.csv", index=name in {"hourly", "seasonal", "price_screens"} ) frame.to_parquet(args.output / f"{name}.parquet") for suffix in ("csv", "parquet"): path = args.output / f"{name}.{suffix}" outputs[path.name] = hashlib.sha256(path.read_bytes()).hexdigest() manifest["output_sha256"] = outputs (args.output / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") print(results["annual"].to_string(index=False)) print(results["scenarios"].to_string(index=False)) matched = int(results["audit"]["within_tolerance"].sum()) print( f"{matched}/{len(results['audit'])} comparisons agree within tolerance; " f"outputs: {args.output}" ) print( "Known source discrepancy: 2024 total is 19.0009% from table inputs, " "versus 18.99% reported." )
if __name__ == "__main__": main()