Source code for energy_data_retrieval.sources.open_meteo.client

"""Model-explicit hourly weather with audited end-label to interval-start alignment."""

from __future__ import annotations

import math
from collections.abc import Iterable, Iterator
from dataclasses import dataclass, field
from datetime import UTC, date, datetime, timedelta
from enum import StrEnum

import numpy as np
import pandas as pd

from energy_data_retrieval.storage import bounded_map
from energy_data_retrieval.transport import (
    HTTPClient,
    JSONValue,
    RequestRecord,
    RetrievalError,
    array_value,
    object_value,
)

OPEN_METEO_HOSTS = frozenset(
    {
        "archive-api.open-meteo.com",
        "historical-forecast-api.open-meteo.com",
        "customer-archive-api.open-meteo.com",
        "customer-historical-forecast-api.open-meteo.com",
    }
)
_VARIABLES = ("shortwave_radiation", "temperature_2m", "wind_speed_10m", "wind_gusts_10m")
_DEGRADATION_VARIABLES = ("relative_humidity_2m", "terrestrial_radiation")


[docs] class WeatherProduct(StrEnum): ERA5 = "era5" IFS = "ifs"
[docs] @dataclass(frozen=True, slots=True) class Site: site_id: str latitude: float longitude: float def __post_init__(self) -> None: if ( not self.site_id or not math.isfinite(self.latitude) or not math.isfinite(self.longitude) ): raise ValueError("A nonempty site ID and finite coordinates are required") if not -90 <= self.latitude <= 90 or not -180 <= self.longitude <= 180: raise ValueError("WGS84 latitude/longitude out of range")
[docs] @dataclass(frozen=True, slots=True) class WeatherChunk: site: Site start: date end: date product: WeatherProduct frame: pd.DataFrame request: RequestRecord cell_latitude: float cell_longitude: float
[docs] @dataclass(slots=True) class OpenMeteoClient: http: HTTPClient product: WeatherProduct = WeatherProduct.ERA5 api_key: str | None = field(default=None, repr=False) chunk_days: int = 31 include_degradation: bool = False def __post_init__(self) -> None: if not 1 <= self.chunk_days <= 366: raise ValueError("chunk_days must be between 1 and 366")
[docs] def weather(self, site: Site, start: date, end: date) -> WeatherChunk: """Read a finite UTC date chunk with one extra day for end samples. No implicit Best Match product is used. Both products are modelled data, not station measurements. Native nearest-cell selection disables elevation downscaling. Public and customer endpoints never share credential caches. """ if start > end or (end - start).days + 1 > self.chunk_days: raise ValueError("Request must be a nonempty chunk within chunk_days") host = ( "archive-api.open-meteo.com" if self.product == WeatherProduct.ERA5 else "historical-forecast-api.open-meteo.com" ) if self.api_key: host = "customer-" + host endpoint = f"https://{host}/v1/" + ( "archive" if self.product == WeatherProduct.ERA5 else "forecast" ) parameters = { "latitude": str(site.latitude), "longitude": str(site.longitude), "start_date": start.isoformat(), "end_date": (end + timedelta(days=1)).isoformat(), "hourly": ",".join( _VARIABLES + (_DEGRADATION_VARIABLES if self.include_degradation else ()) ), "models": "era5" if self.product == WeatherProduct.ERA5 else "ecmwf_ifs", "timezone": "UTC", "timeformat": "unixtime", "wind_speed_unit": "ms", "temperature_unit": "celsius", "cell_selection": "nearest", "elevation": "nan", } response = self.http.get_json( endpoint, parameters, secrets={"apikey": self.api_key} if self.api_key else None, ) payload = object_value(response.value) frame = normalize_hourly( payload, site.site_id, start, end, product=self.product, include_degradation=self.include_degradation, ) latitude, longitude = payload.get("latitude"), payload.get("longitude") if not isinstance(latitude, (int, float)) or not isinstance(longitude, (int, float)): raise RetrievalError("Weather response omits selected grid coordinates") return WeatherChunk( site, start, end, self.product, frame, response.record, float(latitude), float(longitude), )
[docs] def iter_weather( self, sites: Iterable[Site], start: date, end: date, *, workers: int = 2, ) -> Iterator[WeatherChunk]: """Yield bounded location/date partitions without collecting a fleet history.""" if start > end: raise ValueError("start must be on or before end") def jobs() -> Iterator[tuple[Site, date, date]]: seen: set[str] = set() for site in sites: if site.site_id in seen: raise ValueError("Site IDs must be unique") seen.add(site.site_id) current = start while current <= end: stop = min(end, current + timedelta(days=self.chunk_days - 1)) yield site, current, stop current = stop + timedelta(days=1) def fetch(job: tuple[Site, date, date]) -> WeatherChunk: return self.weather(*job) yield from bounded_map(fetch, jobs(), workers=workers)
[docs] def normalize_hourly( payload: dict[str, JSONValue], site_id: str, start: date, end: date, *, product: WeatherProduct, include_degradation: bool = False, ) -> pd.DataFrame: """Align GHI, temperature and wind to identical one-hour UTC intervals. Radiation is the preceding-hour mean at interval end. Temperature and wind are means of the two instantaneous endpoints and remain null if either is missing. Forecast gust is the preceding-hour maximum; the archive defines ERA5 gust at the indicated hour, so its end sample is explicitly retained as instantaneous and must not be called a measured or hourly maximum. """ if payload.get("utc_offset_seconds") != 0: raise RetrievalError("Weather source did not return the requested UTC clock") hourly = object_value(payload.get("hourly")) units = object_value(payload.get("hourly_units")) expected_units = { "shortwave_radiation": {"W/m²", "W/m2"}, "temperature_2m": {"°C", "celsius"}, "wind_speed_10m": {"m/s"}, "wind_gusts_10m": {"m/s"}, } if include_degradation: expected_units.update(relative_humidity_2m={"%"}, terrestrial_radiation={"W/m²", "W/m2"}) for variable, allowed in expected_units.items(): unit = units.get(variable) if not isinstance(unit, str) or unit not in allowed: raise RetrievalError(f"Unexpected source unit for {variable}") raw_time = array_value(hourly.get("time")) if len(raw_time) < 2 or not all( isinstance(t, int) and not isinstance(t, bool) for t in raw_time ): raise RetrievalError("Weather time must contain at least two Unix-second integer labels") time_axis = np.asarray(raw_time, dtype=np.int64) if not np.all(np.diff(time_axis) == 3600): raise RetrievalError("Weather source has a duplicate, gap or nonhourly timestamp") values: dict[str, np.ndarray] = {} for variable in expected_units: raw = array_value(hourly.get(variable)) if len(raw) != len(time_axis): raise RetrievalError("Weather value/time lengths differ") if not all( value is None or (isinstance(value, (int, float)) and not isinstance(value, bool)) for value in raw ): raise RetrievalError("Weather source contains a nonnumeric observation") vector = np.asarray(raw, dtype=np.float64) if np.isinf(vector).any(): raise RetrievalError("Weather source contains infinite values") values[variable] = vector timestamps = pd.to_datetime(time_axis[:-1], unit="s", utc=True) frame = pd.DataFrame( { "site_id": site_id, "timestamp": timestamps, "duration_hours": 1.0, "ghi_wm2": values["shortwave_radiation"][1:], "temperature_c": (values["temperature_2m"][:-1] + values["temperature_2m"][1:]) / 2, "wind_speed_ms": (values["wind_speed_10m"][:-1] + values["wind_speed_10m"][1:]) / 2, "gust_ms": values["wind_gusts_10m"][1:], } ) if include_degradation: frame["relative_humidity_percent"] = ( values["relative_humidity_2m"][:-1] + values["relative_humidity_2m"][1:] ) / 2 frame["toa_wm2"] = values["terrestrial_radiation"][1:] begin = pd.Timestamp(datetime.combine(start, datetime.min.time(), UTC)) stop = pd.Timestamp(datetime.combine(end + timedelta(days=1), datetime.min.time(), UTC)) frame = frame.loc[(frame["timestamp"] >= begin) & (frame["timestamp"] < stop)].reset_index( drop=True ) expected_hours = (end - start).days * 24 + 24 if len(frame) != expected_hours: raise RetrievalError( "Weather response lacks requested interval endpoints; no values were filled" ) frame.attrs.update( source="Open-Meteo", product=product.value, timestamp_convention="UTC interval start", ghi_alignment="preceding-hour mean at interval end", temperature_wind_alignment="mean of instantaneous start/end samples", gust_alignment=( "preceding-hour maximum at end" if product == WeatherProduct.IFS else "instantaneous end sample" ), relative_humidity_alignment=( "mean of instantaneous start/end samples" if include_degradation else "not requested" ), toa_alignment=( "Open-Meteo calculated preceding-hour mean" if include_degradation else "not requested" ), station_observations=False, spatial_selection="nearest native grid cell; no elevation downscaling", ) return frame