"""Stream native climate pixels through KGPV without expanding entire COGs.
This is an exploratory mixed-period application: CanaryClim 1979-2013 and
Global Solar Atlas 1994-2018. It deliberately uses the cellwise classifier,
not map_kgpv's interface for climatologies sharing one baseline. Neither the
source periods nor the coarser solar information are relabelled to match.
"""
from __future__ import annotations
import hashlib
import json
import math
from contextlib import ExitStack
from pathlib import Path
import numpy as np
import pandas as pd
import rasterio
from rasterio.enums import Resampling
from rasterio.vrt import WarpedVRT
from rasterio.windows import Window, from_bounds
from energy_methods.kgpv import ZONE_CODES, classify_kgpv
REFERENCE_IDS = (
"ascencio_vasquez_2021_thesis",
"ascencio_vasquez_2019_kgpv",
"canaryclim_2025_data",
"patino_2023_canaryclim",
"global_solar_atlas_2019",
"rasterio_documentation",
)
# Disjoint geographic windows cover the seven main island groups, including
# Lobos, La Graciosa and the other Chinijo islets. Coordinates are view bounds,
# not administrative polygons: source masks determine land/coastal coverage.
ISLANDS = {
"el_hierro": ("El Hierro", (-18.18, 27.62, -17.87, 27.88)),
"la_palma": ("La Palma", (-17.96, 28.43, -17.70, 28.88)),
"la_gomera": ("La Gomera", (-17.37, 27.99, -17.07, 28.24)),
"tenerife": ("Tenerife", (-16.94, 27.99, -16.10, 28.65)),
"gran_canaria": ("Gran Canaria", (-15.85, 27.72, -15.34, 28.19)),
"fuerteventura": ("Fuerteventura and Lobos", (-14.54, 28.03, -13.79, 28.80)),
"lanzarote": ("Lanzarote and Chinijo", (-13.90, 28.82, -13.32, 29.43)),
}
SAMPLE_SITES = {
"El Hierro": (-17.98, 27.75),
"La Palma": (-17.85, 28.68),
"La Gomera": (-17.24, 28.11),
"Tenerife south": (-16.59, 28.08),
"Tenerife north": (-16.52, 28.39),
"Teide": (-16.6438, 28.2724),
"Anaga": (-16.23, 28.55),
"Gran Canaria": (-15.60, 28.00),
"Fuerteventura": (-14.01, 28.43),
"Lanzarote": (-13.60, 29.04),
"La Graciosa": (-13.51, 29.25),
}
[docs]
def file_sha256(path: Path) -> str:
with path.open("rb") as stream:
return hashlib.file_digest(stream, "sha256").hexdigest()
[docs]
def climate_path(directory: Path, variable: str, month: int) -> Path:
return directory / f"CHELSACanaryClim_{variable}_{month:02d}_1979-2013_V.1.0.tif"
def _validate_climate(source, reference, variable: str, month: int) -> None:
tags = source.tags()
unit = "K" if variable == "tas" else "kg m-2 month-1"
if (
source.crs != rasterio.crs.CRS.from_epsg(4326)
or source.count != 1
or source.shape != reference.shape
or source.transform != reference.transform
or tags.get("variable") != variable
or tags.get("variable_unit") != unit
or tags.get("version") != "1.0"
or tags.get("datetime") != f"15-{month:02d}-1996"
or source.nodata is None
or not np.isfinite([source.scales[0], source.offsets[0]]).all()
or source.scales[0] <= 0
):
raise ValueError("Unexpected CanaryClim metadata, month, units or grid")
[docs]
def decode_climate(raw: np.ndarray, source, variable: str) -> np.ndarray:
"""Mask the encoded sentinel before applying the file's scale and offset."""
values = raw.astype(np.float64)
values[raw == source.nodata] = np.nan
values = values * source.scales[0] + source.offsets[0]
if variable == "tas":
values -= 273.15
elif variable != "pr":
raise ValueError("Expected tas or pr")
return values
def _window(reference, bounds: tuple[float, float, float, float]) -> Window:
window = from_bounds(*bounds, transform=reference.transform)
left = max(0, math.floor(window.col_off))
top = max(0, math.floor(window.row_off))
right = min(reference.width, math.ceil(window.col_off + window.width))
bottom = min(reference.height, math.ceil(window.row_off + window.height))
if left >= right or top >= bottom:
raise ValueError("Island window does not intersect the source grid")
return Window.from_slices((top, bottom), (left, right))
def _classify_window(sources, solar, window: Window, first: np.ndarray):
reference = sources["tas"][0]
land = first != reference.nodata
count = int(land.sum())
classes = np.zeros(first.shape, dtype=np.uint8)
if not count:
return classes, land
fields = {}
for variable in ("tas", "pr"):
monthly = []
for month, source in enumerate(sources[variable]):
raw = first if variable == "tas" and month == 0 else source.read(1, window=window)
monthly.append(decode_climate(raw[land], source, variable))
fields[variable] = np.stack(monthly, axis=-1)
ghi = solar.read(1, window=window, masked=True).filled(np.nan)[land]
rows = np.nonzero(land)[0] + window.row_off
latitude = reference.transform.f + (rows + 0.5) * reference.transform.e
result = classify_kgpv(fields["tas"], fields["pr"], ghi, latitude)
classes[land] = result.class_id
return classes, land
[docs]
def prepare_maps(
raw_directory: Path,
output_directory: Path,
*,
islands: tuple[str, ...] = tuple(ISLANDS),
tile_size: int = 512,
) -> dict:
"""Write two-band native GeoTIFFs, with bounded tile working memory.
Band 1: KGPV class 1..12 (0 unclassified). Band 2: January temperature's
native valid footprint (1 land/input coverage, 0 outside). Missing climate
or solar inputs stay unclassified; no coastal extrapolation is performed.
Radiation is nearest-cell sampled before classification, retaining its
9-arcsecond source support. All output climate pixels keep native centres.
"""
if not islands or len(set(islands)) != len(islands) or set(islands) - ISLANDS.keys():
raise ValueError("Select unique known island groups")
if type(tile_size) is not int or not 128 <= tile_size <= 1024:
raise ValueError("tile_size must be an integer in 128..1024")
output_directory.mkdir(parents=True, exist_ok=True)
records = {}
with ExitStack() as stack, rasterio.Env(GDAL_CACHEMAX=128_000_000):
sources = {
variable: [
stack.enter_context(rasterio.open(climate_path(raw_directory, variable, month)))
for month in range(1, 13)
]
for variable in ("tas", "pr")
}
reference = sources["tas"][0]
for variable, monthly in sources.items():
for month, source in enumerate(monthly, start=1):
_validate_climate(source, reference, variable, month)
if (
reference.transform.a <= 0
or reference.transform.e >= 0
or reference.transform.b != 0
or reference.transform.d != 0
):
raise ValueError("Expected a north-up CanaryClim grid")
ghi_source = stack.enter_context(rasterio.open(raw_directory / "GHI.tif"))
if (
ghi_source.crs != reference.crs
or ghi_source.count != 1
or not np.isclose(ghi_source.transform.a, 0.0025)
or not np.isclose(ghi_source.transform.e, -0.0025)
or ghi_source.nodata is None
):
raise ValueError("Expected the pinned 9-arcsecond GSA GHI raster")
solar = stack.enter_context(
WarpedVRT(
ghi_source,
crs=reference.crs,
transform=reference.transform,
width=reference.width,
height=reference.height,
resampling=Resampling.nearest,
nodata=np.nan,
dtype="float32",
warp_mem_limit=64,
)
)
for key in islands:
label, bounds = ISLANDS[key]
view = _window(reference, bounds)
path = output_directory / f"kgpv_canary_{key}.tif"
counts = np.zeros(13, dtype=np.int64)
land_count = 0
profile = dict(
driver="GTiff",
width=int(view.width),
height=int(view.height),
count=2,
dtype="uint8",
crs=reference.crs,
transform=reference.window_transform(view),
nodata=0,
tiled=True,
blockxsize=512,
blockysize=512,
compress="deflate",
zlevel=9,
BIGTIFF="IF_SAFER",
SPARSE_OK=True,
)
with rasterio.open(path, "w", **profile) as target:
target.set_band_description(1, "KGPV class: 0 unclassified, 1..12 see zone_codes")
target.set_band_description(2, "CanaryClim valid temperature footprint: 0/1")
target.update_tags(
title=f"{label}: exploratory native-grid KGPV",
zone_codes=json.dumps(dict(enumerate(ZONE_CODES))),
climate_period="1979-2013",
solar_period="1994-2018",
baseline_policy="Mixed historical periods; not a common-baseline map",
solar_resolution_degrees="0.0025",
solar_sampling="nearest source cell; no new subpixel radiation detail",
climate_resolution_note=(
"Native COG spacing about 5 m; published model benchmark 100 m. "
"Pixel spacing is not validated climate or KGPV accuracy."
),
source=(
"CHELSACanaryClim v1.0; Global Solar Atlas 2.0, World Bank/ESMAP/Solargis"
),
references=";".join(REFERENCE_IDS),
license="Derived climate: CC0; solar: CC BY 4.0 with GSA additional terms",
)
for row in range(0, int(view.height), tile_size):
for col in range(0, int(view.width), tile_size):
local = Window.from_slices(
(row, min(row + tile_size, int(view.height))),
(col, min(col + tile_size, int(view.width))),
)
window = Window.from_slices(
(int(view.row_off) + row, int(view.row_off) + row + int(local.height)),
(int(view.col_off) + col, int(view.col_off) + col + int(local.width)),
)
first = reference.read(1, window=window)
classes, land = _classify_window(sources, solar, window, first)
if not land.any():
continue
target.write(classes, 1, window=local)
target.write(land.astype(np.uint8), 2, window=local)
land_count += int(land.sum())
counts += np.bincount(classes[land], minlength=13)
target.build_overviews([4, 16, 64], Resampling.nearest)
target.update_tags(ns="rio_overview", resampling="nearest")
records[key] = {
"name": label,
"file": path.name,
"shape": [int(view.height), int(view.width)],
"transform": list(reference.window_transform(view)),
"bounds": list(bounds),
"source_land_pixels": land_count,
"classified_pixels": int(counts[1:].sum()),
"missing_land_pixels": int(counts[0]),
"class_counts": dict(zip(ZONE_CODES[1:], counts[1:].tolist(), strict=True)),
"sha256": file_sha256(path),
"size_bytes": path.stat().st_size,
}
print(
f"{label}: {counts[1:].sum():,} classified / {land_count:,} land pixels; "
f"{path.stat().st_size:,} bytes",
flush=True,
)
samples = []
for name, (lon, lat) in SAMPLE_SITES.items():
row, col = reference.index(lon, lat)
window = Window.from_slices((row, row + 1), (col, col + 1))
record = {
"name": name,
"requested_lon": lon,
"requested_lat": lat,
"lon": reference.xy(row, col)[0],
"lat": reference.xy(row, col)[1],
}
for variable, prefix in (("tas", "t"), ("pr", "p")):
for month, source in enumerate(sources[variable], start=1):
value = decode_climate(source.read(1, window=window), source, variable)[0, 0]
record[f"{prefix}{month:02d}"] = value
record["annual_ghi"] = float(
solar.read(1, window=window, masked=True).filled(np.nan)[0, 0]
)
result = classify_kgpv(
[record[f"t{m:02d}"] for m in range(1, 13)],
[record[f"p{m:02d}"] for m in range(1, 13)],
record["annual_ghi"],
record["lat"],
)
record["kgpv_class"] = int(result.class_id)
record["kgpv_zone"] = str(result.zone)
samples.append(record)
pd.DataFrame(samples).to_csv(
output_directory / "kgpv_canary_samples.csv",
index=False,
float_format="%.10g",
na_rep="",
lineterminator="\n",
)
return records