Source code for energy_data_retrieval.sources.usgs_3dep.client

"""USGS National Map discovery and downloads of project-based 1 m terrain."""

from __future__ import annotations

import math
from dataclasses import dataclass
from pathlib import Path
from urllib.parse import urlsplit

import rasterio

from energy_data_retrieval.terrain import Bounds, TerrainSource, validate_bounds
from energy_data_retrieval.transport import HTTPClient, JSONValue, RequestRecord, object_value

USGS_3DEP_HOSTS = frozenset({"tnmaccess.nationalmap.gov", "prd-tnm.s3.amazonaws.com"})
BASE_URL = "https://tnmaccess.nationalmap.gov/api/v1/products"
DATASET = "Digital Elevation Model (DEM) 1 meter"
REFERENCE_IDS = ("usgs_3dep", "usgs_tnm_api")
USGS_3DEP_SOURCE = TerrainSource(
    "usgs_3dep",
    "USGS 3DEP project-based 1 m DEM",
    "lidar-derived bare-earth DTM",
    "NAVD88 in CONUS; verify project metadata elsewhere",
    REFERENCE_IDS,
)


[docs] @dataclass(frozen=True, slots=True) class TerrainProduct: """Provider product metadata retained verbatim; publication is not acquisition.""" source_id: str title: str download_url: str size_bytes: int metadata: dict[str, JSONValue]
[docs] @dataclass(frozen=True, slots=True) class TerrainSearch: """Complete bounded catalogue result and every page's provenance.""" products: tuple[TerrainProduct, ...] requests: tuple[RequestRecord, ...] total: int
def _download_url(url: str) -> None: parsed = urlsplit(url) if ( parsed.scheme != "https" or parsed.hostname != "prd-tnm.s3.amazonaws.com" or parsed.port not in (None, 443) or parsed.username or parsed.password or parsed.query or parsed.fragment or not parsed.path.startswith("/StagedProducts/Elevation/1m/") or not parsed.path.lower().endswith(".tif") or "/../" in parsed.path ): raise ValueError("Expected an official USGS 3DEP 1 m GeoTIFF URL")
[docs] class USGS3DEPClient: """Discover all matching project tiles before choosing one explicitly. There is no global or complete US 1 m coverage claim. Overlapping projects can differ in survey date, extent, datum and nodata. This connector selects the project-based 1 m dataset, not the newer seamless 1 m product. Configure a sufficient bounded download budget for the selected file's advertised size. """ def __init__(self, http: HTTPClient) -> None: self.http = http
[docs] def search( self, bounds_wgs84: Bounds, *, maximum_items: int = 1000, page_size: int = 100, ) -> TerrainSearch: validate_bounds(bounds_wgs84) if type(maximum_items) is not int or maximum_items < 1: raise ValueError("maximum_items must be a positive integer") if type(page_size) is not int or not 1 <= page_size <= 1000: raise ValueError("page_size must be an integer in 1..1000") products: list[TerrainProduct] = [] records: list[RequestRecord] = [] seen: set[str] = set() total: int | None = None while total is None or len(products) < total: response = self.http.get_json( BASE_URL, { "datasets": DATASET, "bbox": ",".join(map(str, bounds_wgs84)), "max": str(min(page_size, maximum_items)), "offset": str(len(products)), "prodFormats": "GeoTIFF", }, use_cache=False, ) document = object_value(response.value) current = document.get("total") if type(current) is not int or current < 0 or document.get("errors"): raise ValueError("Invalid USGS catalogue response") if current > maximum_items: raise ValueError("USGS result exceeds maximum_items; narrow the area") if total is not None and total != current: raise ValueError("USGS catalogue changed during pagination; retry the search") total = current rows = document.get("items") if not isinstance(rows, list) or (not rows and len(products) < total): raise ValueError("Incomplete USGS catalogue page") records.append(response.record) for row in rows: item = object_value(row) source_id, title = item.get("sourceId"), item.get("title") url, size = item.get("downloadURL"), item.get("sizeInBytes") if ( not isinstance(source_id, str) or not source_id or source_id in seen or not isinstance(title, str) or not isinstance(url, str) or type(size) is not int or size < 1 ): raise ValueError("Invalid or repeated USGS product metadata") _download_url(url) seen.add(source_id) products.append(TerrainProduct(source_id, title, url, size, dict(item))) if len(products) > total: raise ValueError("USGS page exceeds advertised total") return TerrainSearch(tuple(products), tuple(records), total)
[docs] def download_product( self, directory: Path, product: TerrainProduct, *, expected_sha256: str | None = None, ) -> tuple[Path, RequestRecord]: _download_url(product.download_url) if type(product.size_bytes) is not int or product.size_bytes < 1: raise ValueError("USGS product requires a positive advertised byte count") if product.size_bytes > self.http.config.maximum_response_bytes: raise ValueError( "Product exceeds HTTP byte budget; explicitly increase it or choose another" ) path = Path(directory) / Path(urlsplit(product.download_url).path).name record = self.http.download_file( product.download_url, path, expected_sha256=expected_sha256, expected_size_bytes=product.size_bytes, ) with rasterio.open(path) as dataset: if ( dataset.count != 1 or dataset.crs is None or not dataset.crs.is_projected or dataset.crs.linear_units not in {"metre", "meter"} or not all(math.isclose(x, 1.0) for x in dataset.res) ): raise ValueError("USGS response is not projected 1 m numeric terrain") return path, record