Source code for space_pv_connectors.research_http

"""Provenance and bounded downloads shared by the research API connectors."""

from __future__ import annotations

import hashlib
import json
from collections.abc import Callable
from dataclasses import asdict, dataclass
from pathlib import Path
from typing import Any

import httpx

from .http_base import DownloadReceipt, HttpConnector


[docs] @dataclass(frozen=True, slots=True) class ResearchResponse: """Provider metadata and the receipt for its original response bytes. ``request_url`` includes non-secret query parameters; authentication headers are never serialized. JSON connectors retain provider units, licenses, pagination, versions and uncertainty fields. The arXiv connector documents its Atom-to-metadata normalization and can cache original XML bytes. Neither representation is a validated engineering or PV specification. """ data: Any receipt: DownloadReceipt request_url: str documentation: str
def bounded_int(value: int, name: str, lower: int, upper: int) -> int: if isinstance(value, bool) or not isinstance(value, int) or not lower <= value <= upper: raise ValueError(f"{name} must be an integer between {lower} and {upper}") return value def required_text(value: str, name: str) -> str: if not isinstance(value, str) or not value.strip(): raise ValueError(f"{name} must be a nonempty string") return value def require_fields(data: Any, /, **fields: type) -> None: if not isinstance(data, dict) or any( not isinstance(data.get(key), kind) for key, kind in fields.items() ): raise ValueError(f"Unexpected provider response; required fields: {', '.join(fields)}") def _validate_json(data: Any) -> None: if isinstance(data, dict) and ( data.get("error") or data.get("errors") or data.get("status") == "ERROR" ): raise ValueError("Provider returned an API error in its JSON response") class ResearchConnector(HttpConnector): """Internal HTTP helpers; no automatic crawling, retries or paid requests.""" SOURCE = "Research API" DOCUMENTATION = "" def _json( self, url: str, *, params: dict[str, Any] | None = None, headers: dict[str, str] | None = None, cache_path: str | Path | None = None, refresh: bool = False, validate: Callable[[Any], None] = _validate_json, ) -> ResearchResponse: request_url = str(httpx.URL(url, params=params)) path = Path(cache_path) if cache_path is not None else None sidecar = path.with_name(path.name + ".provenance.json") if path else None cached = path is not None and path.exists() and not refresh if cached: if sidecar is None or not sidecar.exists(): raise ValueError("Cache has no provenance sidecar; use refresh=True") provenance = json.loads(sidecar.read_text(encoding="utf-8")) content = path.read_bytes() if ( provenance.get("request_url") != request_url or provenance.get("source") != self.SOURCE ): raise ValueError("Cache belongs to a different request or provider") if provenance.get("sha256") != hashlib.sha256(content).hexdigest(): raise ValueError("Cache checksum does not match its provenance") receipt = DownloadReceipt( **{key: provenance[key] for key in DownloadReceipt.__dataclass_fields__} ) else: response = self._client.get( url, params=params, headers={"Accept": "application/json", **(headers or {})} ) response.raise_for_status() content = response.content receipt = self.receipt(self.SOURCE, str(response.url), content, path=path) try: data = json.loads(content) except ValueError as exc: raise ValueError(f"{self.SOURCE} did not return valid JSON") from exc _validate_json(data) validate(data) if not cached and path is not None and sidecar is not None: path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(content) sidecar.write_text( json.dumps( { **asdict(receipt), "request_url": request_url, "documentation": self.DOCUMENTATION, }, indent=2, ) + "\n", encoding="utf-8", ) return ResearchResponse(data, receipt, request_url, self.DOCUMENTATION) def _download( self, url: str, destination: str | Path, *, checksum: str | None = None, max_bytes: int = 100_000_000, context: dict[str, Any] | None = None, ) -> DownloadReceipt: """Download public bytes; verify optional ``algorithm:hex`` checksum. Nothing is written until the size and checksum checks pass. Downloads deliberately use no API credentials, including across redirects. """ bounded_int(max_bytes, "max_bytes", 1, 2**40) parsed = httpx.URL(url) if parsed.scheme != "https" or not parsed.host or parsed.userinfo: raise ValueError("Downloads require a public HTTPS URL without credentials") expected = None digest = None if checksum is not None: algorithm, sep, expected = checksum.lower().partition(":") algorithm = algorithm.replace("-", "") if not sep or algorithm not in {"md5", "sha1", "sha256", "sha512"}: raise ValueError("Unsupported provider checksum") digest = hashlib.new(algorithm, usedforsecurity=False) if len(expected) != digest.digest_size * 2 or any( c not in "0123456789abcdef" for c in expected ): raise ValueError("Invalid provider checksum") content = bytearray() with self._client.stream("GET", url) as response: response.raise_for_status() if "text/html" in response.headers.get("content-type", "").lower(): raise ValueError("Download returned an HTML page instead of a data file") for chunk in response.iter_bytes(): if len(content) + len(chunk) > max_bytes: raise ValueError( "Download exceeds max_bytes; inspect metadata before increasing it" ) content.extend(chunk) if digest is not None: digest.update(chunk) final_url = str(response.url) if digest is not None and digest.hexdigest() != expected: raise ValueError("Download checksum does not match provider metadata") path = Path(destination) path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(content) receipt = self.receipt(self.SOURCE, final_url, bytes(content), path=path) path.with_name(path.name + ".provenance.json").write_text( json.dumps( { **asdict(receipt), "request_url": url, "documentation": self.DOCUMENTATION, "provider_checksum": checksum, "context": context or {}, }, indent=2, ) + "\n", encoding="utf-8", ) return receipt __all__ = ["ResearchResponse"]