Source code for space_pv_connectors.ground_raster

"""Native-grid subsets of public ground-imaging Cloud Optimized GeoTIFFs.

Rasterio is optional and is imported only when reading or writing an image.
Pixels retain their source values, resolution and nodata mask. This module
does not infer farm boundaries, resample bands or classify cloudy pixels.
"""

from __future__ import annotations

import importlib
import json
import math
import os
from copy import deepcopy
from dataclasses import dataclass
from datetime import UTC, datetime
from hashlib import file_digest
from pathlib import Path
from tempfile import NamedTemporaryFile
from typing import Any
from urllib.parse import urlsplit

import numpy as np

from .http_base import DownloadReceipt


def _rasterio() -> Any:
    try:
        return importlib.import_module("rasterio")
    except ImportError as exc:
        raise ImportError(
            "Ground image reading requires rasterio; install it with "
            "python -m pip install 'orbital-pv[ground]'."
        ) from exc


def _validate_bbox(bbox: tuple[float, float, float, float]) -> tuple[float, float, float, float]:
    try:
        if len(bbox) != 4 or any(isinstance(value, bool) for value in bbox):
            raise ValueError
        west, south, east, north = (float(value) for value in bbox)
    except (TypeError, ValueError) as exc:
        raise ValueError("bbox must contain (west, south, east, north) in WGS84 degrees") from exc
    if not all(math.isfinite(value) for value in (west, south, east, north)):
        raise ValueError("bbox values must be finite")
    if not (-180 <= west < east <= 180 and -90 <= south < north <= 90):
        raise ValueError(
            "bbox must satisfy -180 <= west < east <= 180 and -90 <= south < north <= 90; "
            "split antimeridian-crossing areas into separate requests"
        )
    return west, south, east, north


def _band_calibration(metadata: dict[str, Any], count: int) -> tuple[list[float], list[float]]:
    bands = metadata.get("raster:bands")
    if not isinstance(bands, list) or len(bands) != count:
        raise ValueError("One explicit raster:bands scale and offset is required per image band")
    scales: list[float] = []
    offsets: list[float] = []
    for band in bands:
        try:
            if not isinstance(band, dict):
                raise ValueError
            scale, offset = float(band["scale"]), float(band["offset"])
        except (KeyError, TypeError, ValueError) as exc:
            raise ValueError("Reflectance requires explicit raster:bands scale and offset") from exc
        if not math.isfinite(scale) or not math.isfinite(offset) or scale <= 0:
            raise ValueError("Reflectance scale must be finite and positive; offset must be finite")
        scales.append(scale)
        offsets.append(offset)
    return scales, offsets


def _same_nodata(values: tuple[float | None, ...]) -> bool:
    first = values[0]
    return all(
        value == first
        or (value is not None and first is not None and math.isnan(value) and math.isnan(first))
        for value in values
    )


[docs] @dataclass(frozen=True, slots=True) class GroundImage: """A native-resolution rectangular crop, retaining raw pixel semantics. ``data`` has shape ``(bands, rows, columns)``; True mask entries mean missing data. ``transform`` holds affine ``(a, b, c, d, e, f)`` coefficients and ``bounds`` is ``(left, bottom, right, top)`` in ``crs``. Pixel-aligned bounds can extend slightly beyond the requested ``bbox_wgs84``. ``visual`` is a rendered RGB product, not calibrated reflectance. ``scl`` is a categorical scene-classification product, not a power or irradiance measurement. No cloud masking is applied implicitly. """ data: np.ma.MaskedArray crs: str transform: tuple[float, float, float, float, float, float] bounds: tuple[float, float, float, float] scene_id: str asset_key: str source_url: str asset_metadata: dict[str, Any] bbox_wgs84: tuple[float, float, float, float] item_metadata: dict[str, Any] nodata_values: tuple[float | None, ...] retrieved_at: str
[docs] def reflectance(self) -> np.ma.MaskedArray: """Return ``raw * scale + offset`` using explicit STAC band metadata. Only assets declaring the ``reflectance`` role are accepted. Missing data remain masked before calibration; negative reflectance values are retained. The raw ``data`` is never modified. For example, Collection 1 commonly declares scale 0.0001 and offset -0.1, so dividing every product by 10000 without its offset would be wrong. """ if "reflectance" not in self.asset_metadata.get("roles", []): raise ValueError("This asset does not declare a reflectance role; use its raw data") scales, offsets = _band_calibration(self.asset_metadata, self.data.shape[0]) scale = np.asarray(scales, dtype=np.float64)[:, None, None] offset = np.asarray(offsets, dtype=np.float64)[:, None, None] return np.ma.array( self.data.astype(np.float64) * scale + offset, mask=np.ma.getmaskarray(self.data), copy=False, )
[docs] def save_geotiff(self, destination: str | Path, *, overwrite: bool = False) -> DownloadReceipt: """Save raw pixels, georeferencing, calibration and provenance. The ``.meta.json`` sidecar includes the complete source STAC item, asset metadata, requested/actual bounds and the output SHA-256. The checksum identifies this derived crop, not the original full tile. A GeoTIFF internal mask is used when all band masks agree. Distinct band masks are retained through a shared nodata value when possible; unsupported mask combinations raise rather than discard information. """ rio = _rasterio() transform_module = importlib.import_module("rasterio.transform") path = Path(destination) sidecar = path.with_suffix(path.suffix + ".meta.json") stale_mask = Path(str(path) + ".msk") if not overwrite and any(candidate.exists() for candidate in (path, sidecar, stale_mask)): raise FileExistsError( f"Output already exists: {path}; set overwrite=True to replace it" ) if len(self.nodata_values) != self.data.shape[0] or not _same_nodata(self.nodata_values): raise ValueError( "A GeoTIFF needs the same nodata value across bands; save assets separately" ) nodata = self.nodata_values[0] masks = np.ma.getmaskarray(self.data) common_mask = bool(np.all(masks == masks[0])) if not common_mask: if nodata is None: raise ValueError( "Cannot preserve distinct band masks without a shared nodata value" ) value_mask = ( np.isnan(self.data.data) if math.isnan(nodata) else self.data.data == nodata ) if not np.array_equal(masks, value_mask): raise ValueError( "Cannot encode these distinct band masks losslessly in one GeoTIFF" ) count, height, width = self.data.shape profile = { "driver": "GTiff", "width": width, "height": height, "count": count, "dtype": self.data.dtype, "crs": self.crs, "transform": transform_module.Affine(*self.transform), "nodata": nodata, "compress": "deflate", } path.parent.mkdir(parents=True, exist_ok=True) with NamedTemporaryFile(dir=path.parent, suffix=".tif", delete=False) as temporary: temporary_path = Path(temporary.name) try: with ( rio.Env(GDAL_TIFF_INTERNAL_MASK=True), rio.open(temporary_path, "w", **profile) as dst, ): dst.write(self.data.data) if common_mask: dst.write_mask(~masks[0]) if "reflectance" in self.asset_metadata.get("roles", []): scales, offsets = _band_calibration(self.asset_metadata, count) dst.scales = tuple(scales) dst.offsets = tuple(offsets) dst.update_tags( source_url=self.source_url, scene_id=self.scene_id, asset_key=self.asset_key, pixel_values="raw; scale and offset have not been applied", ) with temporary_path.open("rb") as stream: digest = file_digest(stream, "sha256").hexdigest() size = temporary_path.stat().st_size provenance = { "source": "Earth Search ground image crop", "url": self.source_url, "retrieved_at": self.retrieved_at, "saved_at": datetime.now(UTC).isoformat(), "sha256": digest, "bytes": size, "path": str(path), "scene_id": self.scene_id, "asset_key": self.asset_key, "bbox_wgs84": list(self.bbox_wgs84), "bounds": list(self.bounds), "crs": self.crs, "transform": list(self.transform), "shape": list(self.data.shape), "dtype": str(self.data.dtype), "nodata_values": [ "NaN" if value is not None and math.isnan(value) else value for value in self.nodata_values ], "scale_offset_applied": False, "cloud_mask_applied": False, "resampling": "none; native pixel-aligned window", "asset": self.asset_metadata, "stac_item": self.item_metadata, } # Serialize before replacing an existing output: malformed user- # supplied metadata must not destroy a previously valid crop. metadata_text = json.dumps(provenance, indent=2, allow_nan=False) + "\n" os.replace(temporary_path, path) if stale_mask.exists(): stale_mask.unlink() sidecar.write_text(metadata_text, encoding="utf-8") finally: temporary_path.unlink(missing_ok=True) return DownloadReceipt( source="Earth Search ground image crop", url=self.source_url, retrieved_at=self.retrieved_at, sha256=digest, bytes=size, path=str(path), )
[docs] def load_ground_image(path: str | Path, *, max_pixels: int = 4_000_000) -> GroundImage: """Load a saved native crop after verifying its pixels and provenance. Read the GeoTIFF and ``.meta.json`` written by :meth:`GroundImage.save_geotiff`. The SHA-256, byte count, source identity, native grid, nodata and calibration must agree. The metadata's original ``path`` is informational, so snapshots can move between computers. Unverified external mask/PAM files are rejected. ``max_pixels`` counts all band samples and is enforced before reading the pixels. Raw values, per-band masks and the complete source STAC item are retained; this function never makes network requests or applies calibration. """ if isinstance(max_pixels, bool) or not isinstance(max_pixels, int) or max_pixels <= 0: raise ValueError("max_pixels must be a positive integer") image_path = Path(path) sidecar = image_path.with_suffix(image_path.suffix + ".meta.json") for suffix in (".msk", ".aux.xml"): if Path(str(image_path) + suffix).exists(): raise ValueError("Saved crops cannot use unverified external mask or PAM sidecars") try: metadata = json.loads(sidecar.read_text(encoding="utf-8")) except (ValueError, UnicodeDecodeError) as exc: raise ValueError("Ground crop provenance is not valid JSON") from exc if not isinstance(metadata, dict): raise ValueError("Ground crop provenance must be a JSON object") with image_path.open("rb") as stream: checksum = file_digest(stream, "sha256").hexdigest() if metadata.get("sha256") != checksum: raise ValueError("Ground crop SHA-256 does not match its provenance") if ( isinstance(metadata.get("bytes"), bool) or metadata.get("bytes") != image_path.stat().st_size ): raise ValueError("Ground crop byte count does not match its provenance") if ( metadata.get("source") != "Earth Search ground image crop" or metadata.get("scale_offset_applied") is not False or metadata.get("cloud_mask_applied") is not False or metadata.get("resampling") != "none; native pixel-aligned window" ): raise ValueError("Expected provenance for an unscaled, unresampled native ground crop") try: scene_id, asset_key, href = (metadata[key] for key in ("scene_id", "asset_key", "url")) if not all(isinstance(value, str) and value for value in (scene_id, asset_key, href)): raise ValueError parsed = urlsplit(href) if parsed.scheme != "https" or not parsed.hostname or parsed.username or parsed.password: raise ValueError item, asset = metadata["stac_item"], metadata["asset"] if ( not isinstance(item, dict) or not isinstance(item.get("properties"), dict) or item.get("id") != scene_id or not isinstance(item.get("assets"), dict) or not isinstance(asset, dict) or item["assets"].get(asset_key) != asset or asset.get("href") != href ): raise ValueError bbox = _validate_bbox(tuple(metadata["bbox_wgs84"])) expected_shape = metadata["shape"] if ( not isinstance(expected_shape, list) or len(expected_shape) != 3 or any(isinstance(n, bool) or not isinstance(n, int) or n <= 0 for n in expected_shape) ): raise ValueError expected_transform = tuple(float(value) for value in metadata["transform"]) expected_bounds = tuple(float(value) for value in metadata["bounds"]) if ( len(expected_transform) != 6 or len(expected_bounds) != 4 or not all(math.isfinite(value) for value in (*expected_transform, *expected_bounds)) or not isinstance(metadata["crs"], str) or not isinstance(metadata["dtype"], str) or not isinstance(metadata["retrieved_at"], str) or not metadata["retrieved_at"] ): raise ValueError recorded_nodata = metadata["nodata_values"] if not isinstance(recorded_nodata, list) or len(recorded_nodata) != expected_shape[0]: raise ValueError expected_nodata = [] for value in recorded_nodata: if value == "NaN": expected_nodata.append(float("nan")) elif value is None: expected_nodata.append(None) elif not isinstance(value, bool) and isinstance(value, (int, float)): if not math.isfinite(value): raise ValueError expected_nodata.append(float(value)) else: raise ValueError except (KeyError, TypeError, ValueError, OverflowError) as exc: raise ValueError( "Ground crop provenance has invalid source identity or grid metadata" ) from exc rio = _rasterio() with ( rio.Env(GDAL_PAM_ENABLED=False, GDAL_DISABLE_READDIR_ON_OPEN="EMPTY_DIR"), rio.open(image_path) as src, ): if src.driver != "GTiff" or src.crs is None: raise ValueError("Saved ground crop must be a georeferenced GeoTIFF") sample_count = int(src.count) * src.height * src.width if sample_count > max_pixels: raise ValueError( f"Crop requires {sample_count:,} band samples, above max_pixels={max_pixels:,}" ) affine = src.transform transform = (affine.a, affine.b, affine.c, affine.d, affine.e, affine.f) left, bottom, right, top = src.bounds bounds = (float(left), float(bottom), float(right), float(top)) if ( affine.b != 0 or affine.d != 0 or affine.a <= 0 or affine.e >= 0 or [src.count, src.height, src.width] != expected_shape or src.crs != rio.crs.CRS.from_user_input(metadata["crs"]) or any(dtype != metadata["dtype"] for dtype in src.dtypes) or not np.allclose(transform, expected_transform, rtol=0, atol=1e-8) or not np.allclose(bounds, expected_bounds, rtol=0, atol=1e-8) ): raise ValueError("Ground crop grid does not match its provenance") for actual, expected in zip(src.nodatavals, expected_nodata, strict=True): if actual != expected and not ( actual is not None and expected is not None and math.isnan(actual) and math.isnan(expected) ): raise ValueError("Ground crop nodata does not match its provenance") tags = src.tags() if any( tags.get(key) != value for key, value in ( ("scene_id", scene_id), ("asset_key", asset_key), ("source_url", href), ("pixel_values", "raw; scale and offset have not been applied"), ) ): raise ValueError("Ground crop source tags do not match its provenance") if "reflectance" in asset.get("roles", []): scales, offsets = _band_calibration(asset, src.count) else: scales, offsets = [1.0] * src.count, [0.0] * src.count if not np.allclose(src.scales, scales, rtol=0, atol=1e-12) or not np.allclose( src.offsets, offsets, rtol=0, atol=1e-12 ): raise ValueError("Ground crop calibration does not match its source metadata") data = src.read(masked=True) nodata_values = tuple(src.nodatavals) crs = src.crs.to_string() return GroundImage( data=data, crs=crs, transform=transform, bounds=bounds, scene_id=scene_id, asset_key=asset_key, source_url=href, asset_metadata=deepcopy(asset), bbox_wgs84=bbox, item_metadata=deepcopy(item), nodata_values=nodata_values, retrieved_at=metadata["retrieved_at"], )
[docs] def read_ground_image( item: dict[str, Any], *, bbox: tuple[float, float, float, float], asset: str = "visual", max_pixels: int = 4_000_000, ) -> GroundImage: """Read a WGS84 area from one HTTPS GeoTIFF asset at native resolution. ``bbox`` is ``(west, south, east, north)`` in longitude/latitude degrees. The crop covers the transformed bounding rectangle, rounded outward to source pixels; it is not an exact polygon clip. The selected tile must cover the entire crop. Select another tile or mosaic explicitly when an area crosses a tile edge. ``max_pixels`` limits the total band samples (bands * rows * columns) before pixel data are read. COG range reads retrieve the intersecting compressed blocks, which can be larger than the requested pixel window. Different assets retain their native grids (e.g. RGB 10 m, SCL 20 m). No reprojection, resampling, reflectance scaling or cloud mask occurs. """ bounds_wgs84 = _validate_bbox(bbox) if isinstance(max_pixels, bool) or not isinstance(max_pixels, int) or max_pixels <= 0: raise ValueError("max_pixels must be a positive integer") if not isinstance(item, dict) or not isinstance(item.get("id"), str): raise ValueError("A STAC item with a string id is required") assets = item.get("assets") if not isinstance(assets, dict) or asset not in assets: raise ValueError(f"STAC item has no {asset!r} asset") metadata = assets[asset] if not isinstance(metadata, dict) or not isinstance(metadata.get("href"), str): raise ValueError("The selected asset needs an HTTPS href") href = metadata["href"] parsed = urlsplit(href) if parsed.scheme != "https" or not parsed.netloc or parsed.username or parsed.password: raise ValueError("Ground imagery assets must use public HTTPS URLs without credentials") media_type = metadata.get("type", "") if not isinstance(media_type, str) or "image/tiff" not in media_type.lower(): raise ValueError( "Select a GeoTIFF/COG asset; JPEG previews and metadata are not image grids" ) rio = _rasterio() warp = importlib.import_module("rasterio.warp") windows = importlib.import_module("rasterio.windows") with ( rio.Env( GDAL_DISABLE_READDIR_ON_OPEN="EMPTY_DIR", GDAL_HTTP_TIMEOUT="30", GDAL_HTTP_MAX_RETRY="2", GDAL_HTTP_RETRY_DELAY="1", ), rio.open(href) as src, ): if src.crs is None: raise ValueError("Ground imagery asset has no coordinate reference system") if ( src.transform.b != 0 or src.transform.d != 0 or src.transform.a <= 0 or src.transform.e >= 0 ): raise ValueError("Only north-up GeoTIFF grids are supported") projected = warp.transform_bounds("EPSG:4326", src.crs, *bounds_wgs84, densify_pts=21) if not all(math.isfinite(value) for value in projected): raise ValueError("The requested bbox cannot be projected into this image grid") fractional = windows.from_bounds(*projected, transform=src.transform) # Floating-point transforms can put an exact pixel edge a few ulps # across its integer index. Round only that numerical noise away. col_start = math.floor(round(fractional.col_off, 8)) row_start = math.floor(round(fractional.row_off, 8)) col_stop = math.ceil(round(fractional.col_off + fractional.width, 8)) row_stop = math.ceil(round(fractional.row_off + fractional.height, 8)) if col_stop <= 0 or row_stop <= 0 or col_start >= src.width or row_start >= src.height: raise ValueError("The requested bbox does not intersect the selected image") if col_start < 0 or row_start < 0 or col_stop > src.width or row_stop > src.height: raise ValueError( "The selected image does not cover the entire bbox; select another scene " "or mosaic the intersecting tiles explicitly" ) width, height = col_stop - col_start, row_stop - row_start sample_count = int(src.count) * width * height if width <= 0 or height <= 0: raise ValueError("The requested bbox produces an empty image window") if sample_count > max_pixels: raise ValueError( f"Crop requires {sample_count:,} band samples, above max_pixels={max_pixels:,}; " "use a smaller bbox or explicitly raise the limit" ) window = windows.Window.from_slices((row_start, row_stop), (col_start, col_stop)) data = src.read(window=window, masked=True) affine = src.window_transform(window) left, bottom, right, top = windows.bounds(window, src.transform) image = GroundImage( data=data, crs=src.crs.to_string(), transform=(affine.a, affine.b, affine.c, affine.d, affine.e, affine.f), bounds=(float(left), float(bottom), float(right), float(top)), scene_id=item["id"], asset_key=asset, source_url=href, asset_metadata=deepcopy(metadata), bbox_wgs84=bounds_wgs84, item_metadata=deepcopy(item), nodata_values=tuple(src.nodatavals), retrieved_at=datetime.now(UTC).isoformat(), ) return image