"""AEMET seasonal bias adjustment and explicit mono-Si degradation scenarios.
Meteorological calibration is distinct from module degradation calibration.
No fitted irradiance/UV or measured field degradation is claimed.
"""
from __future__ import annotations
import hashlib
import json
from pathlib import Path
import numpy as np
import pandas as pd
import rasterio
from rasterio.transform import from_origin
from rasterio.warp import transform
from energy_methods.degradation import DegradationParameters, degradation_rates
from energy_methods.degradation_weather import faiman_from_ghi
from energy_methods.weather_calibration import (
VARIABLES,
correct_hourly,
fit_monthly_bias,
spatial_weights,
)
RATE_NAMES = ("hydrolysis", "photodegradation", "thermomechanical", "total")
[docs]
def weather_node(latitude: float, longitude: float) -> str:
"""Match acquisition's explicitly requested 0.25-degree nodes."""
return f"{np.round(latitude * 4) / 4:.2f}_{np.round(longitude * 4) / 4:.2f}"
[docs]
def read_weather(folder: Path, slug: str) -> pd.DataFrame:
"""Read captured, checksum-verified aligned hourly weather."""
meta = json.loads((folder / "weather" / f"{slug}.json").read_text())
frames = []
for item in meta["files"]:
path = folder / "weather" / item["file"]
if hashlib.sha256(path.read_bytes()).hexdigest() != item["sha256"]:
raise ValueError("ERA5 weather checksum mismatch")
frames.append(pd.read_csv(path, parse_dates=["timestamp"]))
frame = pd.concat(frames, ignore_index=True).set_index("timestamp")
frame.index = pd.to_datetime(frame.index, utc=True)
expected = pd.date_range("2022-01-01", "2024-01-01", inclusive="left", freq="h", tz="UTC")
if not frame.index.equals(expected):
raise ValueError("Two complete hourly years required")
required = [*VARIABLES, "ghi_wm2"]
if not np.isfinite(frame[required].to_numpy(dtype=float)).all():
raise ValueError("Missing hourly drivers; no gap filling")
if ((frame.relative_humidity_percent < 0) | (frame.relative_humidity_percent > 100)).any():
raise ValueError("Invalid relative humidity")
if (frame[["ghi_wm2", "wind_speed_ms"]] < 0).any().any():
raise ValueError("Negative irradiance or wind")
return frame
[docs]
def daily_pairs(observations: pd.DataFrame, weather: pd.DataFrame) -> pd.DataFrame:
"""Pair native daily date labels with declared UTC-day model aggregates.
AEMET tmed is matched to model (daily Tmin+Tmax)/2, not hourly mean T.
Daily sampling/period differences remain a limitation; validation is also
reported on monthly averages, rather than claiming hourly validation.
"""
model = weather[list(VARIABLES)].resample("D").mean()
daily_t = weather.temperature_c.resample("D")
model["temperature_c"] = (daily_t.min() + daily_t.max()) / 2
model.index = pd.DatetimeIndex(model.index).strftime("%Y-%m-%d")
records = []
aliases = {
"tmed": "temperature_c",
"hrMedia": "relative_humidity_percent",
"velmedia": "wind_speed_ms",
}
for source, variable in aliases.items():
if source not in observations:
continue
values = pd.to_numeric(
observations[source].astype(str).str.replace(",", "."), errors="coerce"
)
observed = pd.Series(values.to_numpy(), index=observations.fecha.astype(str))
if not observed.index.is_unique:
raise ValueError("Duplicate daily observations")
joined = pd.DataFrame({"observed": observed, "modelled": model[variable]})
joined["variable"] = variable
joined.index.name = "date"
records.append(joined.reset_index())
return pd.concat(records, ignore_index=True)
[docs]
def validate(
pairs: pd.DataFrame,
monthly: dict[str, np.ndarray],
*,
year: int = 2023,
hourly_weather: pd.DataFrame | None = None,
) -> list[dict]:
"""Holdout monthly RMSE and paired-day count, never used for fitting."""
dates = pd.to_datetime(pairs.date)
data = pairs.loc[dates.dt.year == year].copy()
data["month"] = pd.to_datetime(data.date).dt.month
metrics = []
for variable, correction in monthly.items():
sample = data.loc[data.variable == variable].dropna(subset=["observed", "modelled"]).copy()
sample["corrected"] = (
sample.modelled.to_numpy() * correction[sample.month.to_numpy() - 1]
if variable == "wind_speed_ms"
else sample.modelled.to_numpy() + correction[sample.month.to_numpy() - 1]
)
if variable == "relative_humidity_percent":
sample["corrected"] = sample.corrected.clip(0, 100)
if hourly_weather is not None:
hourly_month = pd.DatetimeIndex(hourly_weather.index).month.to_numpy() - 1
corrected_rh = (
(hourly_weather.relative_humidity_percent + correction[hourly_month])
.clip(0, 100)
.resample("D")
.mean()
)
corrected_rh.index = pd.DatetimeIndex(corrected_rh.index).strftime("%Y-%m-%d")
sample["corrected"] = sample.date.map(corrected_rh)
counts = sample.groupby("month").size()
means = sample.groupby("month")[["observed", "modelled", "corrected"]].mean()
means = means.loc[counts >= 20]
if len(means) != 12:
continue
raw = means.modelled - means.observed
adjusted = means.corrected - means.observed
metrics.append(
{
"variable": variable,
"validation_year": year,
"paired_days": len(sample),
"months": len(means),
"raw_monthly_rmse": float(np.sqrt(np.mean(raw**2))),
"corrected_monthly_rmse": float(np.sqrt(np.mean(adjusted**2))),
"raw_mean_bias": float(raw.mean()),
"corrected_mean_bias": float(adjusted.mean()),
}
)
return metrics
[docs]
def annual_drivers(weather: pd.DataFrame, *, uv_fraction: float = 0.05) -> dict[str, float]:
"""Full 2023; annual means and mean daily module extrema; explicit UV proxy."""
expected = pd.date_range("2023-01-01", "2024-01-01", freq="h", inclusive="left", tz="UTC")
if not weather.index.equals(expected) or not 0 < uv_fraction < 1:
raise ValueError("Complete 2023 and a physical explicit UV fraction required")
if not np.isfinite(weather[[*VARIABLES, "ghi_wm2"]].to_numpy(float)).all():
raise ValueError("Finite full-year weather required")
t = pd.Series(
faiman_from_ghi(
weather.temperature_c.to_numpy(),
weather.ghi_wm2.to_numpy(),
weather.wind_speed_ms.to_numpy(),
),
index=weather.index,
)
# Preserve mathematical extrema bounds against floating-point reduction noise.
return {
"module_temperature_mean_c": float(t.mean()),
"module_temperature_upper_c": float(max(t.mean(), t.resample("D").max().mean())),
"module_temperature_lower_c": float(min(t.mean(), t.resample("D").min().mean())),
"relative_humidity_percent": float(weather.relative_humidity_percent.mean()),
"uv_dose_kwh_m2": float(weather.ghi_wm2.sum() * uv_fraction / 1000),
"cycles_per_year": 365.0,
}
[docs]
def rate_scenario(
weather: pd.DataFrame, parameters: DegradationParameters, *, uv_fraction: float = 0.05
) -> dict[str, float]:
"""Published mono-Si scenario; not new field calibration."""
rates = degradation_rates(
**annual_drivers(weather, uv_fraction=uv_fraction), parameters=parameters
)
return {name: float(getattr(rates, name)) for name in RATE_NAMES}
[docs]
def run_study(folder: Path, parameter_path: Path) -> dict:
"""Fit station biases, test temporal/spatial transfer, and calculate raster cells."""
provenance = json.loads((folder / "aemet_provenance.json").read_text())
station_path = folder / "stations.csv"
if hashlib.sha256(station_path.read_bytes()).hexdigest() != provenance["station_sha256"]:
raise ValueError("Station checksum mismatch")
stations = pd.read_csv(station_path)
frames = []
for item in provenance["daily_files"]:
path = folder / item["file"]
if hashlib.sha256(path.read_bytes()).hexdigest() != item["sha256"]:
raise ValueError("AEMET daily partition checksum mismatch")
frames.append(pd.read_csv(path, dtype=str))
observations = pd.concat(frames, ignore_index=True)
grid = pd.read_csv(folder / "grid.csv")
sx, sy = transform("EPSG:4326", "EPSG:32628", stations.longitude, stations.latitude)
stations["easting"], stations["northing"] = sx, sy
params = DegradationParameters(**json.loads(parameter_path.read_text()))
cache = {}
pairs_by_station = {}
coefficients = []
failed = []
for station in stations.to_dict("records"):
sid = station["station_id"]
slug = weather_node(station["latitude"], station["longitude"])
if slug not in cache:
cache[slug] = read_weather(folder, slug)
pairs = daily_pairs(observations.loc[observations.indicativo == sid], cache[slug])
pairs_by_station[sid] = pairs
fitted = fit_monthly_bias(pairs, year=2022)
if fitted.empty:
failed.append(sid)
continue
fitted["station_id"] = sid
coefficients.append(fitted)
if not coefficients:
raise ValueError("No station has twelve supported training months")
coeff = pd.concat(coefficients, ignore_index=True)
coeff.to_csv(folder / "calibration_parameters.csv", index=False)
station_by_id = stations.set_index("station_id")
def transfer(xy: np.ndarray, *, exclude: str | None = None) -> dict[str, np.ndarray]:
result = {}
for variable in VARIABLES:
rows = coeff.loc[(coeff.variable == variable) & (coeff.station_id != exclude)]
pivot = rows.pivot(index="station_id", columns="month", values="value")
if pivot.empty:
continue
coordinates = station_by_id.loc[pivot.index, ["easting", "northing"]].to_numpy(float)
weights = spatial_weights(coordinates, xy)
values = weights @ pivot.to_numpy(float)
values[weights.sum(axis=1) == 0] = np.nan
result[variable] = values
return result
validations = []
for station in stations.to_dict("records"):
sid = station["station_id"]
subset = coeff.loc[coeff.station_id == sid]
own = {
str(v): g.sort_values("month").value.to_numpy(float)
for v, g in subset.groupby("variable")
}
slug = weather_node(station["latitude"], station["longitude"])
for metric in validate(pairs_by_station[sid], own, hourly_weather=cache[slug]):
validations.append({"station_id": sid, "validation": "temporal_holdout", **metric})
transferred = transfer(np.array([[station["easting"], station["northing"]]]), exclude=sid)
other = {v: a[0] for v, a in transferred.items() if np.isfinite(a[0]).all()}
for metric in validate(pairs_by_station[sid], other, hourly_weather=cache[slug]):
validations.append(
{"station_id": sid, "validation": "spatial_and_temporal_holdout", **metric}
)
validation = pd.DataFrame(validations)
validation.to_csv(folder / "validation.csv", index=False)
fields = transfer(grid[["easting", "northing"]].to_numpy(float))
results = []
for i, point in enumerate(grid.to_dict("records")):
monthly = {v: a[i] for v, a in fields.items()}
supported = len(monthly) == 3 and all(np.isfinite(a).all() for a in monthly.values())
slug = weather_node(point["latitude"], point["longitude"])
if slug not in cache:
cache[slug] = read_weather(folder, slug)
weather = cache[slug].loc[pd.DatetimeIndex(cache[slug].index).year == 2023].copy()
record = {**point, "weather_node": slug, "calibration_supported": supported}
record.update({f"raw_driver_{k}": v for k, v in annual_drivers(weather).items()})
raw = rate_scenario(weather, params)
record.update({f"raw_{k}": v for k, v in raw.items()})
if supported:
corrected = correct_hourly(weather, monthly)
record.update(
{f"corrected_driver_{k}": v for k, v in annual_drivers(corrected).items()}
)
rates = rate_scenario(corrected, params)
record.update({f"corrected_{k}": v for k, v in rates.items()})
record.update({f"delta_{k}": rates[k] - raw[k] for k in RATE_NAMES})
record["rh_clipped_hours"] = corrected.attrs["rh_clipped_hours"]
# Scenario spread, explicitly NOT a statistical confidence interval.
for fraction in (0.04, 0.06):
record[f"total_uv_{fraction:.2f}"] = rate_scenario(
corrected, params, uv_fraction=fraction
)["total"]
results.append(record)
result = pd.DataFrame(results)
result.to_csv(folder / "rates.csv", index=False)
for prefix in ("raw", "corrected", "delta"):
array = np.full((4, 50, 106), np.nan, dtype="float32")
for band, name in enumerate(RATE_NAMES):
if f"{prefix}_{name}" in result:
array[band, result.row.to_numpy(int), result.col.to_numpy(int)] = result[
f"{prefix}_{name}"
].to_numpy(float)
with rasterio.open(
folder / f"{prefix}_rates.tif",
"w",
driver="GTiff",
height=50,
width=106,
count=4,
dtype="float32",
crs="EPSG:32628",
transform=from_origin(160000, 3290000, 5000, 5000),
nodata=np.nan,
compress="deflate",
) as dst:
dst.write(array)
for band, name in enumerate(RATE_NAMES, 1):
dst.set_band_description(band, f"{prefix}_{name}_percent_per_year")
dst.update_tags(
evidence="Published mono-Si scenario; weather bias adjusted",
accuracy="5 km display; native ERA5 0.25 degrees; not field loss",
)
summary = {
"training_year": 2022,
"validation_and_map_year": 2023,
"station_inventory": len(stations),
"stations_with_coefficients": coeff.station_id.nunique(),
"station_support_by_variable": coeff.groupby("variable").station_id.nunique().to_dict(),
"unsupported_stations": failed,
"grid_cells": len(result),
"supported_grid_cells": int(result.calibration_supported.sum()),
"degradation_parameters": params.to_dict(),
"weather_calibration": "Monthly additive temperature/RH and wind multiplier, 2022 only",
"spatial_transfer": "IDW nearest four stations within 75 km; no elevation downscaling",
"radiation": "Uncorrected ERA5 GHI; UV=5% GHI proxy; 4-6% sensitivity is not confidence",
"module": "GHI-based Faiman, U0=26.9, U1=6.2, 2 m height, open-rack scenario",
"stress": "Annual mean T/RH, mean daily module extrema, 365 cycles/year; A_N=1",
"limits": [
"Daily date/period and station sampling differences remain",
"Only two years; seasonal means corrected, hourly/extreme skill not established",
"Spatial transfer may fail at mountains/coasts; inspect spatial holdout validation",
"Weather correction is not a new module/material calibration",
"No PID, LID, LeTID, salt corrosion, hail or field-loss measurement inferred",
],
"ranges_percent_per_year": {
name: [
float(result[f"corrected_{name}"].min()),
float(result[f"corrected_{name}"].max()),
]
for name in RATE_NAMES
},
"validation": validation.groupby(["validation", "variable"])[
["raw_monthly_rmse", "corrected_monthly_rmse"]
]
.mean()
.reset_index()
.to_dict("records"),
"reference_ids": [
"aemet_opendata_specification",
"kaaya_2019_degradation",
"ascencio_vasquez_2019_degradation_mapping",
],
}
(folder / "summary.json").write_text(json.dumps(summary, indent=2) + "\n")
return summary