Source code for energy_analysis.chile_fleet.prepare
"""Join normalized site-level retrieval products without inventing observations."""
from __future__ import annotations
import shutil
import tempfile
from pathlib import Path
from .io import (
OPTIONAL_WEATHER,
SiteDataSource,
fingerprint,
load_sites,
validate_output_location,
write_json,
)
[docs]
def prepare_inputs(generation: Path, weather: Path, sites: Path, output: Path) -> dict[str, object]:
"""Create one complete joined Parquet file per registry site.
UTC keys must already describe interval starts with one-hour MWh and weather
in W/m2, degrees Celsius and m/s. Units are a contract; they are not guessed.
The join is outer: a missing generation observation remains NaN when its
weather exists, allowing reference-energy coverage to expose the gap.
"""
if output.exists():
raise FileExistsError(f"Refusing to overwrite existing prepared data: {output}")
validate_output_location(output, generation, weather)
output.parent.mkdir(parents=True, exist_ok=True)
registry = load_sites(sites)
generation_source = SiteDataSource(generation)
weather_source = SiteDataSource(weather, require_actual=False)
temporary = Path(tempfile.mkdtemp(prefix=f".{output.name}-", dir=output.parent))
try:
files: list[dict[str, object]] = []
for index, site_id in enumerate(registry["site_id"].astype(str)):
left = generation_source.site(site_id)
right = weather_source.site(site_id)
left = left.drop(columns=list(OPTIONAL_WEATHER), errors="ignore")
right = right[["site_id", "timestamp", *OPTIONAL_WEATHER]]
joined = left.merge(
right, on=["site_id", "timestamp"], how="outer", validate="one_to_one"
)
joined = joined.sort_values("timestamp", kind="stable")
filename = f"site-{index:06d}.parquet"
joined.to_parquet(temporary / filename, index=False, compression="zstd")
files.append({"site_id": site_id, "file": filename, "rows": len(joined)})
manifest: dict[str, object] = {
"schema": "energy-data-insights-prepared-hourly-v1",
"complete": True,
"units": {
"actual_mwh": "MWh",
"ghi_wm2": "W/m2",
"temperature_c": "degC",
"wind_speed_ms": "m/s",
"gust_ms": "m/s",
},
"time": "UTC interval start; one elapsed hour; no interpolation or forward fill",
"registry_scope": (
"Only registry site IDs are processed; all registry IDs are retained, "
"including empty sites."
),
"sources": {
"generation": fingerprint(generation),
"weather": fingerprint(weather),
"sites": fingerprint(sites),
},
"site_files": files,
}
write_json(temporary / "manifest.json", manifest)
temporary.rename(output)
return manifest
except BaseException:
shutil.rmtree(temporary, ignore_errors=True)
raise