"""NASA CERES and NOAA POES/MetOp environmental inputs.
These connectors retain provider variables, quality fields and units. Particle
flux at a POES/MetOp detector is not a solar-cell dose at another spacecraft.
CERES TOA flux needs a geometric/radiance model before use at a solar panel.
References
----------
https://ceres.larc.nasa.gov/data/
https://www.ncei.noaa.gov/products/poes-metop-space-environment-monitor
https://cmr.earthdata.nasa.gov/search/site/docs/search/api.html
"""
from __future__ import annotations
import hashlib
import json
import re
from collections.abc import Mapping, Sequence
from dataclasses import asdict
from datetime import UTC, date, datetime
from pathlib import Path
from tempfile import NamedTemporaryFile
from typing import Any, ClassVar
from urllib.parse import urlsplit, urlunsplit
import numpy as np
import pandas as pd
from .http_base import DownloadReceipt, HttpConnector
def _utc(value: str) -> pd.Timestamp:
result = pd.Timestamp(value)
if pd.isna(result) or result.tzinfo is None:
raise ValueError("Use an explicit timezone in start and end timestamps")
return result.tz_convert("UTC")
def _xarray():
try:
import xarray as xr
except ImportError as exc:
raise ImportError(
'Install "orbital-pv[netcdf]" to read environmental netCDF files'
) from exc
return xr
def _file_hash(path: Path) -> str:
with path.open("rb") as stream:
return hashlib.file_digest(stream, "sha256").hexdigest()
class EnvironmentalFileConnector(HttpConnector):
"""Bounded, atomic downloads with checksum receipts and verified cache reuse."""
source = "environmental provider"
allowed_hosts: tuple[str, ...] = ()
def download(
self,
url: str,
destination: str | Path,
*,
expected_sha256: str | None = None,
bearer_token: str | None = None,
max_bytes: int = 512 * 1024 * 1024,
) -> DownloadReceipt:
"""Download an explicit provider URL without embedding credentials in receipts.
Existing files are reused only after their digest and receipt are checked.
Remove or choose another destination to replace an unverified local file.
"""
parsed = urlsplit(url)
if (
parsed.scheme != "https"
or parsed.hostname not in self.allowed_hosts
or parsed.username is not None
or parsed.password is not None
):
raise ValueError(f"Expected an HTTPS data URL on {self.allowed_hosts}")
if expected_sha256 is not None and not re.fullmatch(r"[0-9a-fA-F]{64}", expected_sha256):
raise ValueError("expected_sha256 must be a 64-character hexadecimal digest")
if max_bytes <= 0:
raise ValueError("max_bytes must be positive")
safe_url = urlunsplit((parsed.scheme, parsed.netloc, parsed.path, "", ""))
request_fingerprint = hashlib.sha256(url.encode("utf-8")).hexdigest()
path = Path(destination)
sidecar = path.with_suffix(path.suffix + ".receipt.json")
if path.exists():
if not sidecar.exists():
raise ValueError("Refusing to reuse a file without a provenance receipt")
saved = json.loads(sidecar.read_text(encoding="utf-8"))
digest = _file_hash(path)
if (
saved.get("url") != safe_url
or saved.get("request_url_sha256") != request_fingerprint
or saved.get("sha256") != digest
or saved.get("bytes") != path.stat().st_size
or (expected_sha256 is not None and digest != expected_sha256.lower())
):
raise ValueError(
"Cached environmental file failed provenance/checksum verification"
)
return DownloadReceipt(
**{key: value for key, value in saved.items() if key != "request_url_sha256"}
)
path.parent.mkdir(parents=True, exist_ok=True)
headers = {"Authorization": f"Bearer {bearer_token}"} if bearer_token else {}
temporary: Path | None = None
try:
with self._client.stream("GET", url, headers=headers) as response:
response.raise_for_status()
# A login page with status 200 is not an environmental data file.
if "text/html" in response.headers.get("content-type", ""):
raise ValueError("Provider returned HTML instead of data; check data access")
with NamedTemporaryFile(
dir=path.parent, prefix=path.name + ".", delete=False
) as out:
temporary = Path(out.name)
digest_obj = hashlib.sha256()
size = 0
for chunk in response.iter_bytes():
size += len(chunk)
if size > max_bytes:
raise ValueError("Download exceeds max_bytes")
digest_obj.update(chunk)
out.write(chunk)
digest = digest_obj.hexdigest()
if size == 0:
raise ValueError("Provider returned an empty file")
if expected_sha256 is not None and digest != expected_sha256.lower():
raise ValueError("Downloaded file does not match expected_sha256")
receipt = DownloadReceipt(
self.source, safe_url, datetime.now(UTC).isoformat(), digest, size, str(path)
)
temporary.replace(path)
temporary = None
sidecar.write_text(
json.dumps({**asdict(receipt), "request_url_sha256": request_fingerprint}, indent=2)
+ "\n",
encoding="utf-8",
)
return receipt
finally:
if temporary is not None:
temporary.unlink(missing_ok=True)
[docs]
class CeresConnector(EnvironmentalFileConnector):
"""Discover exact CERES collections through NASA CMR and download selected files.
Supply the collection short name explicitly: different editions and time
aggregations are scientifically different products. Earthdata credentials
may be needed by the file host; catalog discovery is public.
"""
source = "NASA CERES"
allowed_hosts = (
"asdc.larc.nasa.gov",
"opendap.larc.nasa.gov",
"ceres-tool.larc.nasa.gov",
"data.asdc.earthdata.nasa.gov",
)
SEARCH_URL = "https://cmr.earthdata.nasa.gov/search/granules.json"
[docs]
def collections(
self,
*,
short_name_pattern: str = "CER_SYN1deg*",
max_results: int = 100,
) -> list[dict[str, Any]]:
"""Discover current CMR short names and separate edition/version IDs.
NASA's cloud migration changed catalog identifiers. A product title
containing ``_Edition4B`` need not have that suffix in its CMR short name.
"""
if not short_name_pattern.startswith("CER_") or not 1 <= max_results <= 2000:
raise ValueError("Use a CER_ collection pattern and max_results in 1..2000")
response = self._client.get(
"https://cmr.earthdata.nasa.gov/search/collections.json",
params={
"short_name": short_name_pattern,
"options[short_name][pattern]": "true",
"page_size": max_results,
},
)
response.raise_for_status()
entries = response.json().get("feed", {}).get("entry")
if not isinstance(entries, list):
raise ValueError("CMR collection response lacks feed.entry")
total = response.headers.get("CMR-Hits")
if (total is not None and int(total) > max_results) or (
total is None and len(entries) == max_results
):
raise ValueError("Narrow short_name_pattern to avoid truncating the collection catalog")
return [
{
"collection_id": row.get("id"),
"short_name": row.get("short_name"),
"version": row.get("version_id"),
"title": row.get("title"),
}
for row in entries
]
[docs]
def search(
self,
*,
short_name: str,
version: str,
start: str,
end: str,
bounding_box: tuple[float, float, float, float] | None = None,
page_size: int = 100,
max_pages: int = 20,
) -> list[dict[str, Any]]:
"""Return granule metadata and direct-data URLs, failing on truncation."""
first, last = _utc(start), _utc(end)
if first >= last:
raise ValueError("start must precede end")
if not short_name.startswith("CER_") or not version.strip():
raise ValueError("Specify a CER_ collection short_name and explicit version")
if not 1 <= page_size <= 2000 or max_pages < 1:
raise ValueError("page_size must be 1..2000 and max_pages positive")
params: dict[str, Any] = {
"short_name": short_name,
"version": version,
"temporal": f"{first.isoformat()},{last.isoformat()}",
"page_size": page_size,
}
if bounding_box is not None:
west, south, east, north = bounding_box
if not (-180 <= west < east <= 180 and -90 <= south < north <= 90):
raise ValueError(
"bounding_box must be (west, south, east, north), without wrapping"
)
params["bounding_box"] = ",".join(str(v) for v in bounding_box)
result: list[dict[str, Any]] = []
for page in range(1, max_pages + 1):
params["page_num"] = page
response = self._client.get(self.SEARCH_URL, params=params)
response.raise_for_status()
payload = response.json()
entries = payload.get("feed", {}).get("entry")
if not isinstance(entries, list):
raise ValueError("CMR response lacks feed.entry")
for entry in entries:
data_urls = [
link["href"]
for link in entry.get("links", [])
if link.get("rel", "").endswith("/data#")
and link.get("href", "").startswith("https://")
and not link.get("inherited", False)
]
result.append(
{
"id": entry.get("id"),
"title": entry.get("title"),
"time_start": entry.get("time_start"),
"time_end": entry.get("time_end"),
"short_name": short_name,
"version": version,
"data_urls": data_urls,
}
)
total = response.headers.get("CMR-Hits")
if len(entries) < page_size or (total is not None and len(result) >= int(total)):
return result
raise ValueError(
"CMR search reached max_pages; narrow the query to avoid silent truncation"
)
[docs]
class PoesMetopConnector(EnvironmentalFileConnector):
"""Download documented NCEI daily SEM-2 Level-1b files.
Instrument availability varies by date. Missing provider files raise the
original HTTP error; no fabricated observations or nearby-day substitution.
"""
source = "NOAA NCEI POES/MetOp SEM-2"
allowed_hosts = ("www.ncei.noaa.gov", "data.ngdc.noaa.gov", "www.ngdc.noaa.gov")
BASE_URL = "https://www.ncei.noaa.gov/data/poes-metop-space-environment-monitor/access/l1b"
SPACECRAFT: ClassVar[dict[str, str]] = {
"noaa15": "n15",
"noaa16": "n16",
"noaa17": "n17",
"noaa18": "n18",
"noaa19": "n19",
"metop01": "m01",
"metop02": "m02",
"metop03": "m03",
}
@classmethod
def daily_url(cls, spacecraft: str, day: str | date, *, version: str = "v01r00") -> str:
if spacecraft not in cls.SPACECRAFT:
raise ValueError(f"spacecraft must be one of {tuple(cls.SPACECRAFT)}")
if not re.fullmatch(r"v\d{2}r\d{2}", version):
raise ValueError("version must have the form v01r00")
day_value = date.fromisoformat(day) if isinstance(day, str) else day
if isinstance(day_value, datetime) or not isinstance(day_value, date):
raise ValueError("day must be a date or YYYY-MM-DD string")
code = cls.SPACECRAFT[spacecraft]
return (
f"{cls.BASE_URL}/{version}/{day_value.year}/{spacecraft}/"
f"poes_{code}_{day_value:%Y%m%d}_proc.nc"
)
def fetch_day(
self,
spacecraft: str,
day: str | date,
destination: str | Path,
*,
version: str = "v01r00",
expected_sha256: str | None = None,
) -> DownloadReceipt:
return self.download(
self.daily_url(spacecraft, day, version=version),
destination,
expected_sha256=expected_sha256,
)
[docs]
def inspect_environment_netcdf(path: str | Path) -> dict[str, dict[str, Any]]:
"""Inspect provider variable names/dimensions/units before selecting channels."""
xr = _xarray()
with xr.open_dataset(path, decode_times=False) as dataset:
return {
str(name): {
"dimensions": list(variable.dims),
"shape": list(variable.shape),
"units": variable.attrs.get("units"),
"long_name": variable.attrs.get("long_name"),
}
for name, variable in dataset.variables.items()
}
[docs]
def read_ceres_toa(
path: str | Path,
*,
shortwave_variable: str,
longwave_variable: str,
time_variable: str = "time",
latitude_variable: str = "lat",
longitude_variable: str = "lon",
selectors: Mapping[str, Any] | None = None,
) -> pd.DataFrame:
"""Read an explicit CF-netCDF CERES subset into a long-form TOA flux table.
``selectors`` are xarray ``isel`` indices applied before loading. Raw HDF4
editions are not converted implicitly. Use provider-exported netCDF files.
Invalid negative flux is marked missing, never clipped to valid zero flux.
"""
xr = _xarray()
with xr.open_dataset(path, decode_cf=True) as source:
names = (
time_variable,
latitude_variable,
longitude_variable,
shortwave_variable,
longwave_variable,
)
if any(name not in source.variables for name in names):
raise ValueError(
"Requested CERES variables are missing; inspect_environment_netcdf first"
)
subset = source[list(names)].isel(dict(selectors or {}))
if not np.issubdtype(subset[time_variable].dtype, np.datetime64):
raise ValueError("CERES time must be decoded from CF epoch metadata")
for name in (shortwave_variable, longwave_variable):
units = re.sub(r"[\s^*]", "", str(subset[name].attrs.get("units", ""))).lower()
if units not in {"wm-2", "w/m2", "wm\u22122"}:
raise ValueError(
f"Expected W m-2 for {name}, found {subset[name].attrs.get('units')}"
)
frame = subset.to_dataframe().reset_index(drop=False)
frame = frame.loc[:, list(dict.fromkeys(names))].rename(
columns={
time_variable: "timestamp_utc",
latitude_variable: "latitude_deg",
longitude_variable: "longitude_deg",
shortwave_variable: "toa_shortwave_w_m2",
longwave_variable: "toa_longwave_w_m2",
}
)
frame["timestamp_utc"] = pd.to_datetime(frame["timestamp_utc"], utc=True, errors="raise")
if frame["timestamp_utc"].isna().any():
raise ValueError("CERES time coordinate contains missing timestamps")
if (
not np.isfinite(frame[["latitude_deg", "longitude_deg"]]).all().all()
or (frame["latitude_deg"].abs() > 90).any()
or ((frame["longitude_deg"] < -180) | (frame["longitude_deg"] > 360)).any()
):
raise ValueError("CERES latitude/longitude coordinates must be geographic degrees")
invalid_count = 0
for name in ("toa_shortwave_w_m2", "toa_longwave_w_m2"):
invalid = (frame[name] < 0) | ~np.isfinite(frame[name])
invalid_count += int(invalid.sum())
frame.loc[invalid, name] = np.nan
frame.attrs.update(
{
"source": "NASA CERES",
"source_file_sha256": _file_hash(Path(path)),
"measurement_boundary": "top-of-atmosphere hemispheric flux",
"source_variables": {"shortwave": shortwave_variable, "longwave": longwave_variable},
"invalid_flux_values": invalid_count,
"interpretation": "Requires Earth-view geometry; not irradiance at the spacecraft",
}
)
return frame
[docs]
def read_poes_metop(
path: str | Path,
*,
channels: Sequence[str],
time_variable: str = "time",
quality_variables: Sequence[str] = (),
) -> pd.DataFrame:
"""Read chosen SEM-2 channels with their exact units and quality columns.
Select detector-specific variables after inspecting the file. This function
does not reinterpret counts as flux, integral channels as differential flux,
or local flux as accumulated solar-cell damage. Supports CF time and NOAA's
native daily format: independent same-length 1D variables and milliseconds
since 1970 in the documented ``time.long_name`` (``units='time'``). The latter
is recognized only with NOAA POES/MetOp provider metadata, not guessed for
arbitrary numeric coordinates. Provider negative sentinel values are retained.
"""
if not channels or len(set(channels)) != len(channels):
raise ValueError("channels must be a nonempty sequence of unique provider variable names")
xr = _xarray()
with xr.open_dataset(path, decode_cf=True) as source:
names = list(dict.fromkeys((time_variable, *channels, *quality_variables)))
if any(name not in source.variables for name in names):
raise ValueError("Requested SEM-2 variables are missing; inspect the file first")
native_noaa = (
source.attrs.get("Project") == "POES/MetOp"
and source.attrs.get("naming_authority") == "gov.noaa.ngdc"
and source[time_variable].attrs.get("long_name") == "milliseconds since 1970"
and source[time_variable].attrs.get("units") == "time"
)
for name in names:
if (
source[name].ndim != 1
or source[name].size != source[time_variable].size
or (not native_noaa and source[name].dims != source[time_variable].dims)
):
raise ValueError(
"Selected channels must share the one-dimensional record/time axis"
)
frame = pd.DataFrame({name: source[name].values for name in names})
units = {name: source[name].attrs.get("units", "unspecified") for name in channels}
descriptions = {name: source[name].attrs.get("long_name", name) for name in channels}
global_attributes = {str(k): str(v) for k, v in source.attrs.items()}
# CF metadata must have decoded time; numeric time without an epoch is ambiguous.
if native_noaa:
frame[time_variable] = pd.to_datetime(
frame[time_variable], unit="ms", utc=True, errors="raise"
).dt.tz_localize(None)
elif not np.issubdtype(frame[time_variable].dtype, np.datetime64):
raise ValueError("SEM time was not decoded from CF epoch metadata")
frame = frame.rename(columns={time_variable: "timestamp_utc"})
frame["timestamp_utc"] = pd.to_datetime(frame["timestamp_utc"], utc=True)
if frame["timestamp_utc"].isna().any():
raise ValueError("SEM time contains missing timestamps")
frame.attrs.update(
{
"source": "NOAA NCEI POES/MetOp SEM-2",
"source_file_sha256": _file_hash(Path(path)),
"units": units,
"channel_descriptions": descriptions,
"provider_attributes": global_attributes,
"quality_variables": list(quality_variables),
"time_encoding": "NOAA native milliseconds since 1970" if native_noaa else "CF epoch",
"measurement_boundary": "local particle detector",
"interpretation": "Channel response and contamination corrections require instrument-specific work",
}
)
return frame