Source code for energy_data_retrieval.climate_grid
"""Bounded decoding of monthly provider NetCDF archives (no climate algorithms)."""
from __future__ import annotations
import gzip
import tempfile
from collections.abc import Generator
from contextlib import contextmanager
from pathlib import Path
import numpy as np
import pandas as pd
import xarray as xr
from energy_data_retrieval.transport import RetrievalError
[docs]
@contextmanager
def open_netcdf(
path: Path, *, maximum_uncompressed_bytes: int = 2 * 1024**3
) -> Generator[xr.Dataset, None, None]:
"""Decode CF fill values/scales; expand gzip on disk with a hard byte limit."""
if maximum_uncompressed_bytes < 1:
raise ValueError("maximum_uncompressed_bytes must be positive")
with tempfile.TemporaryDirectory(prefix="energy-climate-") as temporary:
source = path
if path.suffix == ".gz":
source = Path(temporary) / "data.nc"
size = 0
with gzip.open(path, "rb") as reader, source.open("wb") as writer:
while chunk := reader.read(1024**2):
size += len(chunk)
if size > maximum_uncompressed_bytes:
raise RetrievalError("Expanded NetCDF exceeds the configured byte limit")
writer.write(chunk)
elif path.stat().st_size > maximum_uncompressed_bytes:
raise RetrievalError("NetCDF exceeds the configured byte limit")
with xr.open_dataset(source, decode_cf=True, decode_timedelta=False) as dataset:
yield dataset
[docs]
def normalize_monthly(
dataset: xr.Dataset,
variables: dict[str, tuple[str, str]],
*,
start_year: int,
end_year: int,
bbox: tuple[float, float, float, float] | None = None,
maximum_values: int = 20_000_000,
) -> xr.Dataset:
"""Select before loading. Output time is a calendar-month label, not UTC telemetry.
``variables`` maps native names to (output name, canonical unit). Native units
are checked by the source adapter. Spatial bounds are inclusive cell centres;
west > east selects across the antimeridian. No resampling or gap filling.
"""
if not 1 <= start_year <= end_year <= 9998 or maximum_values < 1:
raise ValueError("Use ordered years and a positive maximum_values")
rename = {
name: target
for name, target in (("latitude", "lat"), ("longitude", "lon"))
if name in dataset.dims
}
ds = dataset.rename(rename)
for axis, limit in (("lat", 90), ("lon", 360)):
if axis not in ds.coords or ds[axis].dims != (axis,):
raise ValueError("Expected one-dimensional latitude/longitude coordinates")
values = ds[axis].values
if not np.isfinite(values).all() or np.any(np.abs(values) > limit):
raise ValueError(f"Invalid {axis} coordinates")
ds = ds.assign_coords(lon=(ds.lon + 180) % 360 - 180).sortby(["lat", "lon"])
if not ds.indexes["lat"].is_unique or not ds.indexes["lon"].is_unique:
raise ValueError("Duplicate spatial coordinates")
times = pd.DatetimeIndex(ds.time.values)
months = times.to_period("M")
if times.hasnans or months.has_duplicates or not months.is_monotonic_increasing:
raise ValueError("Expected unique increasing calendar months")
ds = ds.assign_coords(time=months.to_timestamp()).sel(
time=slice(f"{start_year:04d}-01-01", f"{end_year:04d}-12-31")
)
if bbox is not None:
west, south, east, north = bbox
if not (
np.isfinite(bbox).all()
and -180 <= west <= 180
and -180 <= east <= 180
and -90 <= south <= north <= 90
):
raise ValueError("bbox must be WEST SOUTH EAST NORTH in geographic degrees")
longitude = (
(ds.lon >= west) & (ds.lon <= east)
if west <= east
else (ds.lon >= west) | (ds.lon <= east)
)
ds = ds.sel(lat=slice(south, north)).isel(lon=np.flatnonzero(longitude.values))
if not ds.sizes["lat"] or not ds.sizes["lon"]:
raise ValueError("No grid cell centres fall inside bbox")
selected = ds[list(variables)]
if any(set(selected[name].dims) != {"time", "lat", "lon"} for name in variables):
raise ValueError("Monthly variables must have time, lat and lon dimensions only")
if sum(v.size for v in selected.data_vars.values()) > maximum_values:
raise ValueError("Selected data exceeds maximum_values; select a smaller bbox or archive")
result = selected.rename({key: value[0] for key, value in variables.items()}).load()
for native, (name, unit) in variables.items():
result[name].attrs.update(units=unit, native_variable=native)
result.time.attrs.update(long_name="calendar month", interval="monthly")
return result