Source code for energy_data_retrieval.sources.natural_earth.client

"""Bounded, checksummed acquisition of Natural Earth Admin 0 country contours."""

from __future__ import annotations

import hashlib
import json
from dataclasses import asdict, dataclass
from pathlib import Path
from typing import Literal
from zipfile import ZipFile

from energy_data_retrieval.transport import HTTPClient, RequestRecord, RetrievalError

NATURAL_EARTH_HOSTS = frozenset({"naturalearth.s3.amazonaws.com"})
REFERENCE_IDS = ("natural_earth_countries",)


[docs] @dataclass(frozen=True, slots=True) class BoundaryDownload: """Local .shp path, companion files and exact public-source provenance.""" shapefile: Path manifest: Path request: RequestRecord
[docs] class NaturalEarthClient: def __init__(self, http: HTTPClient) -> None: self.http = http
[docs] def download_countries( self, directory: Path, *, scale: Literal["10m", "50m", "110m"] = "10m", expected_sha256: str | None = None, ) -> BoundaryDownload: """Keep .shp/.shx/.dbf/.prj/.cpg companions and a source/hash manifest. Scale means 1:10/50/110 million, not metre accuracy. The official URL tracks releases; pass a previously recorded SHA-256 to require identical bytes. Only exact, flat member names are read; archives are never blindly extracted. Compressed size is bounded by the shared HTTP configuration. """ if scale not in {"10m", "50m", "110m"}: raise ValueError("Natural Earth scale must be 10m, 50m or 110m") stem = f"ne_{scale}_admin_0_countries" directory = Path(directory) archive = directory / f"{stem}.zip" url = f"https://naturalearth.s3.amazonaws.com/{scale}_cultural/{stem}.zip" request = self.http.download_file(url, archive, expected_sha256=expected_sha256) required = {f"{stem}{suffix}" for suffix in (".shp", ".shx", ".dbf", ".prj")} optional = {f"{stem}.cpg", f"{stem}.VERSION.txt"} with ZipFile(archive) as source: names = source.namelist() if len(names) != len(set(names)) or not required.issubset(names): raise RetrievalError("Country archive has duplicate or missing shapefile members") selected = sorted(required | (optional & set(names))) if sum(source.getinfo(name).file_size for name in selected) > 50_000_000: raise RetrievalError("Country shapefile exceeds the 50 MB expanded limit") # Read and verify CRCs before writing any companion files. contents = {name: source.read(name) for name in selected} hashes = {} for name, content in contents.items(): (directory / name).write_bytes(content) hashes[name] = hashlib.sha256(content).hexdigest() manifest = directory / f"{stem}.provenance.json" manifest.write_text( json.dumps( { "source": "Natural Earth Admin 0 - Countries", "scale": scale, "version": contents.get(f"{stem}.VERSION.txt", b"unspecified").decode().strip(), "license": "Public domain", "terms_url": "https://www.naturalearthdata.com/about/terms-of-use/", "boundary_policy": "Natural Earth default de facto boundaries", "request": asdict(request), "files_sha256": hashes, "reference_ids": list(REFERENCE_IDS), }, indent=2, ) + "\n", encoding="utf-8", ) return BoundaryDownload(directory / f"{stem}.shp", manifest, request)