"""Read native ERA5 in bounded time/spatial tiles from a pinned public snapshot.
Optional dependencies are imported only when native acquisition is requested.
The public 2026 study snapshot is explicit, never silently advanced to ``main``.
"""
from __future__ import annotations
import importlib
from collections.abc import Callable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from datetime import UTC, date, datetime, timedelta
from typing import Protocol, cast
import numpy as np
import pandas as pd
from numpy.typing import NDArray
from energy_data_retrieval.sources.open_meteo import Site
from energy_data_retrieval.transport import RetrievalError
PINNED_SNAPSHOT = "TZ2KX6A24CJ5DG17T7MG"
VARIABLES = ("t2m", "u10", "v10", "ssrd", "fg10")
class _Array(Protocol):
@property
def values(self) -> NDArray: ...
@property
def attrs(self) -> Mapping[str, object]: ...
@property
def encoding(self) -> Mapping[str, object]: ...
def isel(self, **indexers: slice) -> _Array: ...
class _Dataset(Protocol):
@property
def latitude(self) -> _Array: ...
@property
def longitude(self) -> _Array: ...
@property
def valid_time(self) -> _Array: ...
def __getitem__(self, variable: str) -> _Array: ...
def close(self) -> None: ...
[docs]
@dataclass(frozen=True, slots=True)
class NativeWeatherChunk:
site: Site
frame: pd.DataFrame
snapshot_id: str
cell_latitude: float
cell_longitude: float
[docs]
@dataclass(frozen=True, slots=True)
class NativeERA5Client:
snapshot_id: str = PINNED_SNAPSHOT
chunk_hours: int = 8736
def __post_init__(self) -> None:
if not self.snapshot_id or not 1 <= self.chunk_hours <= 8736:
raise ValueError("A pinned snapshot and 1-8736-hour chunks are required")
def _open(self) -> _Dataset:
try:
icechunk = importlib.import_module("icechunk")
xarray = importlib.import_module("xarray")
except ModuleNotFoundError:
raise ImportError(
"Install energy-data-retrieval[era5-native] for native ERA5 retrieval"
) from None
storage = icechunk.s3_storage(
bucket="earthmover-icechunk-era5",
prefix="icechunkV2",
region="us-east-1",
anonymous=True,
)
repository = icechunk.Repository.open(storage)
session = repository.readonly_session(snapshot_id=self.snapshot_id)
return cast(
_Dataset,
xarray.open_zarr(
session.store,
group="single/temporal",
consolidated=False,
chunks=None,
),
)
[docs]
def iter_weather(
self,
sites: Sequence[Site],
start: date,
end: date,
*,
skip_partition: Callable[[Site, int, int, float, float], bool] | None = None,
) -> Iterator[NativeWeatherChunk]:
"""Read each intersecting native tile once per time block, yielding site frames.
At most five 12 by 12 spatial tiles of ``chunk_hours + 1`` are resident.
Sites sharing a native cell reuse values. Source hours beyond snapshot
coverage fail explicitly, instead of substituting a different weather product.
"""
if start > end or len({site.site_id for site in sites}) != len(sites):
raise ValueError("Dates must be ordered and site IDs unique")
dataset = self._open()
try:
_validate_metadata(dataset)
latitude = np.asarray(dataset.latitude.values, dtype=np.float64)
longitude = np.asarray(dataset.longitude.values, dtype=np.float64)
times = np.asarray(dataset.valid_time.values).astype("datetime64[s]").astype(np.int64)
begin = int(datetime.combine(start, datetime.min.time(), UTC).timestamp())
stop = int(
datetime.combine(end + timedelta(days=1), datetime.min.time(), UTC).timestamp()
)
first, last = int(np.searchsorted(times, begin)), int(np.searchsorted(times, stop))
if (
first >= len(times)
or last >= len(times)
or times[first] != begin
or times[last] != stop
):
raise RetrievalError("Requested interval endpoints are outside this ERA5 snapshot")
tiles: dict[tuple[int, int], list[tuple[Site, int, int]]] = {}
for site in sites:
yi = int(np.abs(latitude - site.latitude).argmin())
xi = int(np.abs((longitude - site.longitude + 180) % 360 - 180).argmin())
tiles.setdefault((yi // 12, xi // 12), []).append((site, yi, xi))
for (tile_y, tile_x), members in sorted(tiles.items()):
for lo in range(first, last, self.chunk_hours):
hi = min(lo + self.chunk_hours, last) + 1
active = [
(site, yi, xi)
for site, yi, xi in members
if skip_partition is None
or not skip_partition(
site,
int(times[lo]),
int(times[hi - 2]),
float(latitude[yi]),
float((longitude[xi] + 180) % 360 - 180),
)
]
if not active:
continue
arrays = {
name: np.asarray(
dataset[name]
.isel(
valid_time=slice(lo, hi),
latitude=slice(tile_y * 12, (tile_y + 1) * 12),
longitude=slice(tile_x * 12, (tile_x + 1) * 12),
)
.values,
dtype=np.float64,
)
for name in VARIABLES
}
for site, yi, xi in active:
cell_arrays = {
name: array[:, yi % 12, xi % 12] for name, array in arrays.items()
}
frame = normalize_native_arrays(site.site_id, times[lo:hi], **cell_arrays)
yield NativeWeatherChunk(
site,
frame,
self.snapshot_id,
float(latitude[yi]),
float((longitude[xi] + 180) % 360 - 180),
)
finally:
dataset.close()
def _validate_metadata(dataset: _Dataset) -> None:
expected_units = {
"t2m": {"K"},
"u10": {"m s**-1", "m s-1", "m/s"},
"v10": {"m s**-1", "m s-1", "m/s"},
"ssrd": {"J m**-2", "J m-2", "J/m2"},
"fg10": {"m s**-1", "m s-1", "m/s"},
}
for name in VARIABLES:
units = dataset[name].attrs.get("units")
if not isinstance(units, str) or units not in expected_units[name]:
raise RetrievalError(f"Unexpected ERA5 native units for {name}")
if tuple(cast(Sequence[int], dataset[name].encoding.get("chunks", ()))) != (8736, 12, 12):
raise RetrievalError("ERA5 source chunk layout changed; inspect before acquiring")
[docs]
def normalize_native_arrays(
site_id: str,
time_seconds: NDArray,
*,
t2m: NDArray,
u10: NDArray,
v10: NDArray,
ssrd: NDArray,
fg10: NDArray,
) -> pd.DataFrame:
"""Convert original ERA5 units and interval labels without interpolation."""
times = np.asarray(time_seconds, dtype=np.int64)
arrays = [np.asarray(value, dtype=np.float64) for value in (t2m, u10, v10, ssrd, fg10)]
if times.ndim != 1 or len(times) < 2 or not np.all(np.diff(times) == 3600):
raise ValueError("Native source must contain consecutive hourly endpoint labels")
if any(array.shape != times.shape or np.isinf(array).any() for array in arrays):
raise ValueError("Native weather arrays must match the time axis and contain no infinity")
temperature, u, v, radiation, gust = arrays
speed = np.hypot(u, v)
frame = pd.DataFrame(
{
"site_id": site_id,
"timestamp": pd.to_datetime(times[:-1], unit="s", utc=True),
"duration_hours": 1.0,
"ghi_wm2": radiation[1:] / 3600.0,
"temperature_c": (temperature[:-1] + temperature[1:]) / 2 - 273.15,
"wind_speed_ms": (speed[:-1] + speed[1:]) / 2,
"gust_ms": gust[1:],
}
)
frame.attrs.update(
source="ECMWF ERA5 via Earthmover Icechunk",
timestamp_convention="UTC interval start",
ghi_alignment="source preceding-hour accumulated J/m2 divided by 3600",
temperature_wind_alignment="mean of instantaneous start/end samples",
gust_alignment="source fg10 maximum since previous postprocessing at end label",
spatial_selection="nearest native 0.25 degree cell",
station_observations=False,
)
return frame