"""Crop pinned CHELSA v2.1 climatology COGs without downloading global rasters."""
from __future__ import annotations
import hashlib
import io
import json
import math
import re
import time
from collections import OrderedDict
from datetime import UTC, datetime
from pathlib import Path
import httpx
import numpy as np
import rasterio
from rasterio.windows import Window, from_bounds
from energy_data_retrieval.transport import RetrievalError
CHELSA_BASE_URL = "https://os.unil.cloud.switch.ch/chelsa02/chelsa/global/climatologies"
REFERENCE_IDS = ("chelsa_2021_climatologies", "karger_2017_chelsa")
[docs]
def sha256(path: Path) -> str:
with path.open("rb") as stream:
return hashlib.file_digest(stream, "sha256").hexdigest()
[docs]
class RangeReader(io.RawIOBase):
"""Seekable HTTP ranges with exact response validation and bounded memory/I/O.
The source-specific caller fixes the URL, size and strong ETag. Every read
carries If-Match; an ETag is a version validator, not a SHA-256 checksum.
No redirects or full-body fallbacks are accepted. The caller owns HTTP.
"""
def __init__(
self,
http: httpx.Client,
url: str,
size: int,
etag: str,
*,
block_size: int = 1_048_576,
maximum_bytes: int = 64_000_000,
) -> None:
self.http, self.url, self.length, self.etag = http, url, size, etag
self.block_size, self.maximum_bytes = block_size, maximum_bytes
self.position = 0
self.transferred_bytes = 0
self.cache: OrderedDict[int, bytes] = OrderedDict()
[docs]
def readable(self) -> bool:
return True
[docs]
def seekable(self) -> bool:
return True
[docs]
def tell(self) -> int:
return self.position
[docs]
def seek(self, offset: int, whence: int = 0) -> int:
if whence not in (0, 1, 2):
raise ValueError("Invalid seek origin")
position = offset + (0 if whence == 0 else self.position if whence == 1 else self.length)
if position < 0:
raise ValueError("Cannot seek before the source")
self.position = position
return position
def _block(self, number: int) -> bytes:
if number in self.cache:
self.cache.move_to_end(number)
return self.cache[number]
start = number * self.block_size
stop = min(start + self.block_size, self.length) - 1
length = stop - start + 1
for attempt in range(3):
if self.transferred_bytes + length > self.maximum_bytes:
raise RetrievalError("COG request exceeded its regional transfer budget")
try:
with self.http.stream(
"GET",
self.url,
follow_redirects=False,
headers={
"Range": f"bytes={start}-{stop}",
"If-Match": self.etag,
"Accept-Encoding": "identity",
},
) as response:
if response.status_code in (429, 500, 502, 503, 504):
raise httpx.ReadError("Retryable COG response")
if (
response.status_code != 206
or response.headers.get("content-range")
!= f"bytes {start}-{stop}/{self.length}"
or response.headers.get("etag") != self.etag
):
raise RetrievalError("COG range, version or response status changed")
data = bytearray()
for chunk in response.iter_bytes(64 * 1024):
self.transferred_bytes += len(chunk)
data.extend(chunk)
if len(data) > length or self.transferred_bytes > self.maximum_bytes:
raise RetrievalError("COG range exceeded its declared byte limit")
if len(data) != length:
raise RetrievalError("Incomplete COG range response")
self.cache[number] = bytes(data)
if len(self.cache) > 8:
self.cache.popitem(last=False)
return self.cache[number]
except httpx.TransportError:
if attempt == 2:
raise RetrievalError("COG regional retrieval failed") from None
time.sleep(1)
raise AssertionError("Unreachable retry state")
[docs]
def read(self, size: int = -1) -> bytes:
if self.closed:
raise ValueError("Read from a closed COG")
if size < 0:
size = self.length - self.position
if size > 32_000_000:
raise RetrievalError("Refusing an unbounded raster read")
end = min(self.position + size, self.length)
parts = []
while self.position < end:
number = self.position // self.block_size
block = self._block(number)
offset = self.position % self.block_size
piece = block[offset : offset + end - self.position]
parts.append(piece)
self.position += len(piece)
return b"".join(parts)
[docs]
class CHELSAClient:
"""Explicit acquisition of native tas/pr windows for baseline 1981-2010."""
def __init__(self, http: httpx.Client) -> None:
self.http = http
[docs]
def download_window(
self,
directory: Path,
variable: str,
month: int,
bounds: tuple[float, float, float, float],
*,
expected_etag: str | None = None,
expected_pixels_sha256: str | None = None,
) -> tuple[Path, dict]:
if variable not in {"tas", "pr"} or type(month) is not int or not 1 <= month <= 12:
raise ValueError("Select tas/pr and an integer month in 1..12")
if (
len(bounds) != 4
or not np.isfinite(bounds).all()
or not -180 <= bounds[0] < bounds[2] <= 180
or not -90 <= bounds[1] < bounds[3] <= 90
):
raise ValueError("Expected finite west,south,east,north bounds")
if (bounds[2] - bounds[0]) * (bounds[3] - bounds[1]) * 120**2 > 10_000_000:
raise ValueError("Requested region exceeds ten million native cells")
name = f"CHELSA_{variable}_{month:02d}_1981-2010_V.2.1.tif"
url = f"{CHELSA_BASE_URL}/{variable}/1981-2010/{name}"
path = Path(directory) / name
manifest = path.with_suffix(".tif.subset.json")
if path.exists() or manifest.exists():
if not path.exists() or not manifest.exists():
raise ValueError("Incomplete CHELSA subset checkpoint")
record = json.loads(manifest.read_text())
if (
record["url"] != url
or record["requested_bounds"] != list(bounds)
or record["sha256"] != sha256(path)
or (expected_etag is not None and record["etag"] != expected_etag)
or (
expected_pixels_sha256 is not None
and record["pixels_sha256"] != expected_pixels_sha256
)
):
raise ValueError("CHELSA subset checkpoint or source pin changed")
return path, record
head = self.http.head(url, follow_redirects=False)
if head.status_code != 200:
raise RetrievalError("CHELSA metadata request failed")
etag = head.headers.get("etag", "")
if not re.fullmatch(r'"[0-9a-f]{1,64}(?:-\d+)?"', etag):
raise RetrievalError("CHELSA source needs a strong version ETag")
if expected_etag is not None and etag != expected_etag:
raise ValueError("CHELSA source version differs from the pinned ETag")
size = int(head.headers["content-length"])
if not 0 < size <= 1_000_000_000:
raise RetrievalError("Unexpected CHELSA source size")
readers = []
def opener(filename, mode="rb"):
if filename != name or mode not in ("r", "rb"):
raise FileNotFoundError(filename)
reader = RangeReader(self.http, url, size, etag)
readers.append(reader)
return reader
path.parent.mkdir(parents=True, exist_ok=True)
temporary = path.with_suffix(".tif.partial")
try:
with (
rasterio.Env(GDAL_DISABLE_READDIR_ON_OPEN="EMPTY_DIR"),
rasterio.open(name, opener=opener) as source,
):
tags = source.tags()
unit = "K" if variable == "tas" else "kg m-2 month-1"
if (
source.crs != rasterio.crs.CRS.from_epsg(4326)
or source.count != 1
or tags.get("version") != "2.1"
or tags.get("datetime") != "1981-2010"
or tags.get("variable") != variable
or tags.get("variable_unit") != unit
or source.transform.b != 0
or source.transform.d != 0
or not np.isclose(source.transform.a, 1 / 120)
or not np.isclose(source.transform.e, -1 / 120)
or source.nodata != 65535
or source.dtypes != ("uint16",)
or not np.isfinite([source.scales[0], source.offsets[0]]).all()
or source.scales[0] <= 0
):
raise ValueError("Unexpected CHELSA units, baseline, encoding or grid")
floating = from_bounds(*bounds, transform=source.transform)
row, col = math.floor(floating.row_off), math.floor(floating.col_off)
bottom = math.ceil(floating.row_off + floating.height)
right = math.ceil(floating.col_off + floating.width)
if row < 0 or col < 0 or bottom > source.height or right > source.width:
raise ValueError("Region extends outside the CHELSA source")
window = Window.from_slices((row, bottom), (col, right))
values = source.read(1, window=window)
pixel_hash = hashlib.sha256(values.astype("<u2").tobytes()).hexdigest()
if expected_pixels_sha256 is not None and pixel_hash != expected_pixels_sha256:
raise ValueError("CHELSA subset pixels differ from the reviewed data")
transform = source.window_transform(window)
profile = dict(
driver="GTiff",
width=values.shape[1],
height=values.shape[0],
count=1,
dtype="uint16",
crs=source.crs,
transform=transform,
nodata=source.nodata,
compress="deflate",
tiled=True,
)
with rasterio.open(temporary, "w", **profile) as target:
target.write(values, 1)
target.scales, target.offsets = source.scales, source.offsets
target.update_tags(**tags)
record = {
"url": url,
"etag": etag,
"source_size_bytes": size,
"source_shape": list(source.shape),
"source_transform": list(source.transform),
"requested_bounds": list(bounds),
"shape": list(values.shape),
"transform": list(transform),
"scale": source.scales[0],
"offset": source.offsets[0],
"units": unit,
"baseline": [1981, 2010],
"pixels_sha256": pixel_hash,
"retrieved_at_utc": datetime.now(UTC).isoformat(),
"transfer_bytes": sum(r.transferred_bytes for r in readers),
"license": "CC0 1.0",
"reference_ids": list(REFERENCE_IDS),
}
temporary.replace(path)
record.update(sha256=sha256(path), size_bytes=path.stat().st_size)
manifest.write_text(json.dumps(record, indent=2) + "\n")
finally:
temporary.unlink(missing_ok=True)
for reader in readers:
reader.close()
return path, record