"""Bounded HTTP reads with source allowlists, retries and public-only caching."""
from __future__ import annotations
import codecs
import hashlib
import json
import logging
import math
import os
import re
import tempfile
import threading
import time
from collections.abc import Callable, Generator, Mapping
from contextlib import contextmanager
from contextvars import ContextVar
from dataclasses import asdict, dataclass
from datetime import UTC, datetime
from email.message import Message
from email.utils import parsedate_to_datetime
from pathlib import Path
from typing import Literal, cast
from urllib.parse import quote, quote_plus, urlsplit
import httpx
type JSONValue = bool | int | float | str | list[JSONValue] | dict[str, JSONValue] | None
_SECRET_NAMES = frozenset(
{
"user_key",
"token",
"api_key",
"apikey",
"access_token",
"authorization",
"password",
"securitytoken",
"servicekey",
"api-key",
}
)
_RETRY_STATUSES = frozenset({408, 429, 500, 502, 503, 504})
_ACTIVE_SECRETS: ContextVar[tuple[str, ...]] = ContextVar("retrieval_secrets", default=())
_QUERY_SECRET = re.compile(
r"([?&](?:user_key|token|api_key|apikey|api-key|access_token|authorization|password|"
r"securitytoken|servicekey)=)[^&\s\"']+",
re.IGNORECASE,
)
_QUERY_SECRET_WITH_PUBLIC_CURSOR = re.compile(
r"([?&](?:user_key|api_key|apikey|api-key|access_token|authorization|password|"
r"securitytoken|servicekey)=)[^&\s\"']+",
re.IGNORECASE,
)
def _redact_text(
text: str, secrets: tuple[str, ...], *, public_pagination_token: bool = False
) -> str:
for secret in secrets:
for spelling in {secret, quote(secret, safe=""), quote_plus(secret, safe="")}:
text = text.replace(spelling, "[REDACTED]")
pattern = _QUERY_SECRET_WITH_PUBLIC_CURSOR if public_pagination_token else _QUERY_SECRET
return pattern.sub(r"\1[REDACTED]", text)
def _redact_value(
value: JSONValue, secrets: tuple[str, ...], *, public_pagination_token: bool = False
) -> JSONValue:
if isinstance(value, str):
return _redact_text(value, secrets, public_pagination_token=public_pagination_token)
if isinstance(value, list):
return [
_redact_value(item, secrets, public_pagination_token=public_pagination_token)
for item in value
]
if isinstance(value, dict):
return {
_redact_text(
key, secrets, public_pagination_token=public_pagination_token
): _redact_value(item, secrets, public_pagination_token=public_pagination_token)
for key, item in value.items()
}
return value
class _SecretLogFilter(logging.Filter):
def filter(self, record: logging.LogRecord) -> bool:
record.msg = _redact_text(record.getMessage(), _ACTIVE_SECRETS.get())
record.args = ()
return True
@contextmanager
def _secret_scope(secrets: tuple[str, ...]) -> Generator[None, None, None]:
token = _ACTIVE_SECRETS.set(secrets)
try:
yield
finally:
_ACTIVE_SECRETS.reset(token)
[docs]
class RetrievalError(RuntimeError):
"""Sanitized retrieval failure; never contains a credential-bearing URL."""
[docs]
@dataclass(frozen=True, slots=True)
class HTTPConfig:
"""Finite resource and retry limits, shared by all sources."""
timeout_seconds: float = 45.0
attempts: int = 4
retry_base_seconds: float = 1.0
maximum_retry_seconds: float = 60.0
minimum_interval_seconds: float = 0.25
maximum_response_bytes: int = 64 * 1024 * 1024
cache_ttl_seconds: float = 3600.0
def __post_init__(self) -> None:
values = (
self.timeout_seconds,
self.retry_base_seconds,
self.maximum_retry_seconds,
self.minimum_interval_seconds,
self.cache_ttl_seconds,
)
if not all(math.isfinite(value) and value >= 0 for value in values):
raise ValueError("HTTP durations must be finite and nonnegative")
if self.timeout_seconds == 0 or not 1 <= self.attempts <= 10:
raise ValueError("Positive timeout and 1-10 attempts are required")
if self.maximum_response_bytes < 1:
raise ValueError("A positive response byte limit is required")
[docs]
@dataclass(frozen=True, slots=True)
class RequestRecord:
"""Non-secret provenance for the exact stored, sanitized response bytes."""
endpoint: str
parameters: dict[str, str]
retrieved_at_utc: str
sha256: str
size_bytes: int
cache_hit: bool = False
[docs]
@dataclass(frozen=True, slots=True)
class JSONResponse:
value: JSONValue
record: RequestRecord
[docs]
@dataclass(frozen=True, slots=True)
class TextResponse:
"""UTF-8 response with a hash of its stored bytes after credential redaction."""
value: str
record: RequestRecord
[docs]
@dataclass(frozen=True, slots=True)
class BytesResponse:
"""Exact public response bytes, media type and provider request identifier."""
value: bytes
record: RequestRecord
content_type: str
request_id: str | None
[docs]
def atomic_bytes(path: Path, data: bytes) -> None:
"""Replace one file atomically; failed writes never publish a partial file."""
path.parent.mkdir(parents=True, exist_ok=True)
descriptor, name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent)
temporary = Path(name)
try:
with os.fdopen(descriptor, "wb") as stream:
stream.write(data)
stream.flush()
os.fsync(stream.fileno())
temporary.replace(path)
finally:
temporary.unlink(missing_ok=True)
[docs]
def object_value(value: JSONValue) -> dict[str, JSONValue]:
if not isinstance(value, dict):
raise RetrievalError("Expected a JSON object")
return value
[docs]
def array_value(value: JSONValue) -> list[JSONValue]:
if not isinstance(value, list):
raise RetrievalError("Expected a JSON array")
return value
def _reject_nonfinite(value: str) -> None:
raise ValueError(f"Non-finite JSON number: {value}")
def _decode(body: bytes, *, legacy_encoding: str | None = None) -> JSONValue:
try:
return cast(JSONValue, json.loads(body, parse_constant=_reject_nonfinite))
except UnicodeDecodeError:
if legacy_encoding is not None:
try:
return cast(
JSONValue,
json.loads(body.decode(legacy_encoding), parse_constant=_reject_nonfinite),
)
except ValueError:
pass
raise RetrievalError("Source returned invalid JSON") from None
except ValueError:
raise RetrievalError("Source returned invalid JSON") from None
def _legacy_json_encoding(content_type: str) -> str | None:
"""Honor only explicitly declared Latin-1/Latin-9 for an opted-in source."""
message = Message()
message["Content-Type"] = content_type
charset = message.get_content_charset()
if charset is None:
return None
try:
encoding = codecs.lookup(charset).name
except LookupError:
return None
return encoding if encoding in {"iso8859-1", "iso8859-15"} else None
[docs]
class HTTPClient:
"""Bounded HTTPS retrieval with a fixed host allowlist and no redirects.
Authentication must be passed separately through ``secrets`` or
``secret_headers``. Authenticated responses never use the public cache.
Supply an httpx transport for offline tests, rather than a preauthenticated
client. This ensures authorization cannot silently bypass cache isolation.
"""
def __init__(
self,
allowed_hosts: frozenset[str],
*,
config: HTTPConfig | None = None,
cache_directory: Path | None = None,
transport: httpx.BaseTransport | None = None,
sleep: Callable[[float], None] = time.sleep,
) -> None:
if not allowed_hosts:
raise ValueError("At least one approved host is required")
self.config = config or HTTPConfig()
self.allowed_hosts = allowed_hosts
self.cache_directory = cache_directory
self._sleep = sleep
self._rate_lock = threading.Lock()
self._next_request = 0.0
for name in (
"httpx",
"httpcore.connection",
"httpcore.http11",
"httpcore.http2",
"httpcore.proxy",
):
logger = logging.getLogger(name)
if not any(isinstance(item, _SecretLogFilter) for item in logger.filters):
logger.addFilter(_SecretLogFilter())
self._client = httpx.Client(
transport=transport,
timeout=self.config.timeout_seconds,
follow_redirects=False,
trust_env=False,
limits=httpx.Limits(max_connections=8, max_keepalive_connections=4),
headers={"User-Agent": "energy-data-insights/0.1", "Accept": "application/json"},
)
def __enter__(self) -> HTTPClient:
return self
def __exit__(self, *_: object) -> None:
self.close()
[docs]
def close(self) -> None:
self._client.close()
def _wait_for_slot(self) -> None:
with self._rate_lock:
now = time.monotonic()
wait = max(0.0, self._next_request - now)
self._next_request = max(now, self._next_request) + self.config.minimum_interval_seconds
if wait:
self._sleep(wait)
def _retry_delay(self, response: httpx.Response | None, attempt: int) -> float:
delay = self.config.retry_base_seconds * (2**attempt)
if response is not None:
value = response.headers.get("retry-after", "")
try:
requested = float(value)
except ValueError:
try:
requested = (parsedate_to_datetime(value) - datetime.now(UTC)).total_seconds()
except (TypeError, ValueError, OverflowError):
requested = 0.0
if math.isfinite(requested):
delay = max(delay, requested)
return min(self.config.maximum_retry_seconds, max(0.0, delay))
[docs]
def request_bytes(
self,
endpoint: str,
parameters: Mapping[str, str | list[str]] | None = None,
*,
method: Literal["GET", "POST"] = "GET",
form: Mapping[str, str | list[str]] | None = None,
files: Mapping[str, tuple[str, bytes, str]] | None = None,
) -> BytesResponse:
"""Retrieve public binary/text data or submit an idempotent calculation.
POST is for read-only provider calculations, never state changes. Public
fields only; no authentication and no cache. List-valued query/form
fields retain repeated keys. Uploads and responses obey the configured
byte limit. Provenance includes method, form values and upload hashes.
Redirects are refused; retries include 529 (PVGIS computation overload).
"""
parsed = urlsplit(endpoint)
if (
parsed.scheme != "https"
or parsed.hostname not in self.allowed_hosts
or parsed.username is not None
or parsed.password is not None
or parsed.query
or parsed.fragment
or parsed.port not in (None, 443)
):
raise ValueError("Endpoint must be a query-free HTTPS URL on an approved host")
if method not in {"GET", "POST"} or (method == "GET" and (form or files)):
raise ValueError("Use GET without a body or POST for a read-only calculation")
query, fields, uploads = dict(parameters or {}), dict(form or {}), dict(files or {})
for group in (query, fields, uploads):
if any(name.lower() in _SECRET_NAMES for name in group):
raise ValueError("Binary retrieval accepts public parameters only")
provenance = {"method": method}
for prefix, group in (("query", query), ("form", fields)):
for name, value in group.items():
values = value if isinstance(value, list) else [value]
if not values or not all(isinstance(item, str) for item in values):
raise ValueError("Public query/form values must be strings or nonempty lists")
provenance[f"{prefix}:{name}"] = (
json.dumps(value) if isinstance(value, list) else value
)
upload_size = 0
for name, (filename, body, media_type) in uploads.items():
if not filename or not media_type or not isinstance(body, bytes):
raise ValueError("Uploads require a filename, bytes and media type")
upload_size += len(body)
provenance[f"file:{name}:sha256"] = hashlib.sha256(body).hexdigest()
provenance[f"file:{name}:size_bytes"] = str(len(body))
if upload_size > self.config.maximum_response_bytes:
raise ValueError("Upload exceeds the configured byte limit")
last_status: int | None = None
request_id: str | None = None
for attempt in range(self.config.attempts):
self._wait_for_slot()
response: httpx.Response | None = None
try:
with self._client.stream(
method,
endpoint,
params=query,
data=fields or None,
files=uploads or None,
headers={"Accept": "*/*"},
) as response:
last_status = response.status_code
request_id = response.headers.get("X-Request-ID")
if last_status == 200:
body = bytearray()
for chunk in response.iter_bytes(chunk_size=64 * 1024):
if len(body) + len(chunk) > self.config.maximum_response_bytes:
raise RetrievalError(
"Source response exceeded the configured byte limit"
)
body.extend(chunk)
raw = bytes(body)
record = RequestRecord(
endpoint,
provenance,
datetime.now(UTC).isoformat(),
hashlib.sha256(raw).hexdigest(),
len(raw),
)
return BytesResponse(
raw, record, response.headers.get("Content-Type", ""), request_id
)
if last_status not in _RETRY_STATUSES | {529}:
break
except httpx.TransportError:
last_status = None
if attempt + 1 < self.config.attempts:
self._sleep(self._retry_delay(response, attempt))
status = f"HTTP {last_status}" if last_status is not None else "transport failure"
identifier = f"; request_id={request_id[:128]}" if request_id else ""
raise RetrievalError(
f"Retrieval failed from {parsed.hostname}: {status}{identifier}"
) from None
[docs]
def download_file(
self,
endpoint: str,
path: Path,
*,
expected_sha256: str | None = None,
expected_size_bytes: int | None = None,
) -> RequestRecord:
"""Stream one public export to disk with verified whole-file restart semantics.
An interrupted transfer is retried from byte zero. HTTP ranges are not
assumed safe for changing archives. Reuse requires both an intact file
and its matching manifest. Credential-bearing and redirected URLs fail.
A supplied source byte count is checked before publishing or reusing a file.
"""
parsed = urlsplit(endpoint)
if (
parsed.scheme != "https"
or parsed.hostname not in self.allowed_hosts
or parsed.username is not None
or parsed.password is not None
or parsed.query
or parsed.fragment
or parsed.port not in (None, 443)
):
raise ValueError("Endpoint must be a query-free HTTPS URL on an approved host")
if expected_sha256 is not None and not re.fullmatch("[a-fA-F0-9]{64}", expected_sha256):
raise ValueError("Expected SHA-256 must be a 64-character hexadecimal digest")
if expected_size_bytes is not None:
if type(expected_size_bytes) is not int or expected_size_bytes < 0:
raise ValueError("Expected byte size must be a nonnegative integer")
if expected_size_bytes > self.config.maximum_response_bytes:
raise ValueError("Expected byte size exceeds the configured byte limit")
expected = expected_sha256.lower() if expected_sha256 else None
manifest = path.with_suffix(path.suffix + ".download.json")
if path.exists() or manifest.exists():
if not path.exists() or not manifest.exists():
raise ValueError("Incomplete download checkpoint; inspect it before retrying")
saved = json.loads(manifest.read_text())
with path.open("rb") as stream:
actual = hashlib.file_digest(stream, "sha256").hexdigest()
if (
saved["endpoint"] != endpoint
or saved["sha256"] != actual
or saved["size_bytes"] != path.stat().st_size
or path.stat().st_size > self.config.maximum_response_bytes
or (expected is not None and actual != expected)
or (expected_size_bytes is not None and path.stat().st_size != expected_size_bytes)
):
raise ValueError("Download checkpoint differs from its source or checksum")
return RequestRecord(
endpoint, {}, saved["retrieved_at_utc"], actual, path.stat().st_size, True
)
path.parent.mkdir(parents=True, exist_ok=True)
last_status: int | None = None
for attempt in range(self.config.attempts):
self._wait_for_slot()
response: httpx.Response | None = None
descriptor, name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent)
temporary = Path(name)
try:
with (
os.fdopen(descriptor, "wb") as target,
self._client.stream(
"GET", endpoint, headers={"Accept": "application/octet-stream"}
) as response,
):
last_status = response.status_code
if last_status == 200:
digest = hashlib.sha256()
size = 0
for chunk in response.iter_bytes(chunk_size=64 * 1024):
size += len(chunk)
if size > self.config.maximum_response_bytes:
raise RetrievalError("Export exceeded the configured byte limit")
if expected_size_bytes is not None and size > expected_size_bytes:
raise RetrievalError(
"Downloaded export exceeds its expected byte size"
)
digest.update(chunk)
target.write(chunk)
if expected_size_bytes is not None and size != expected_size_bytes:
raise RetrievalError("Downloaded export has an unexpected byte size")
if expected is not None and digest.hexdigest() != expected:
raise RetrievalError("Downloaded export has an unexpected SHA-256")
target.flush()
os.fsync(target.fileno())
record = RequestRecord(
endpoint, {}, datetime.now(UTC).isoformat(), digest.hexdigest(), size
)
elif last_status not in _RETRY_STATUSES:
break
if last_status == 200:
temporary.replace(path)
atomic_bytes(manifest, json.dumps(asdict(record), indent=2).encode())
return record
except httpx.TransportError:
last_status = None
finally:
temporary.unlink(missing_ok=True)
if attempt + 1 < self.config.attempts:
self._sleep(self._retry_delay(response, attempt))
status = f"HTTP {last_status}" if last_status is not None else "transport failure"
raise RetrievalError(f"Export retrieval failed from {parsed.hostname}: {status}") from None
[docs]
def get_text(
self,
endpoint: str,
parameters: Mapping[str, str] | None = None,
*,
secrets: Mapping[str, str] | None = None,
secret_headers: Mapping[str, str] | None = None,
use_cache: bool = True,
) -> TextResponse:
"""Read bounded UTF-8 text, retaining tabs, line endings and physical units.
Credentials must use ``secrets`` or ``secret_headers``. Authenticated
responses bypass the public cache; logs, response text and provenance
redact credentials. Public CSV/PSV/TSV bytes remain unchanged. Cache
entries are accepted only after source, size and hash verification.
"""
parsed = urlsplit(endpoint)
if (
parsed.scheme != "https"
or parsed.hostname not in self.allowed_hosts
or parsed.username is not None
or parsed.password is not None
or parsed.query
or parsed.fragment
or parsed.port not in (None, 443)
):
raise ValueError("Endpoint must be a query-free HTTPS URL on an approved host")
public = dict(parameters or {})
if any(name.lower() in _SECRET_NAMES for name in public):
raise ValueError("Text retrieval accepts public parameters only, never credentials")
private = dict(secrets or {})
private_headers = dict(secret_headers or {})
header_names = {name.lower() for name in private_headers}
if len(header_names) != len(private_headers) or not header_names <= _SECRET_NAMES | {
"x-api-key",
"x-auth-token",
}:
raise ValueError("secret_headers requires unique credential header names")
if any(
not value
or not value.isascii()
or any(ord(character) < 32 or ord(character) == 127 for character in value)
for value in private_headers.values()
):
raise ValueError(
"Secret header values must be nonempty ASCII without control characters"
)
if any(not value for value in private.values()) or {name.lower() for name in public} & {
name.lower() for name in private
}:
raise ValueError("Secret parameters must be nonempty and separate")
active_secrets = (*private.values(), *private_headers.values())
cache_key = hashlib.sha256(
json.dumps(["text-utf8", endpoint, public], sort_keys=True).encode()
).hexdigest()
cache = (
self.cache_directory / f"{cache_key}.text.json"
if use_cache and not active_secrets and self.cache_directory is not None
else None
)
if cache is not None and cache.exists():
try:
# JSON escaping can expand one control byte to six bytes.
if cache.stat().st_size <= 6 * self.config.maximum_response_bytes + 65536:
saved = object_value(_decode(cache.read_bytes()))
value = saved["value"]
metadata = object_value(saved["record"])
captured = datetime.fromisoformat(str(metadata["retrieved_at_utc"]))
if isinstance(value, str):
raw = value.encode("utf-8")
if (
0
<= (datetime.now(UTC) - captured).total_seconds()
<= self.config.cache_ttl_seconds
and metadata["endpoint"] == endpoint
and metadata["parameters"] == public
and len(raw) <= self.config.maximum_response_bytes
and metadata["size_bytes"] == len(raw)
and metadata["sha256"] == hashlib.sha256(raw).hexdigest()
):
return TextResponse(
value,
RequestRecord(
endpoint,
public,
str(metadata["retrieved_at_utc"]),
str(metadata["sha256"]),
len(raw),
True,
),
)
except (OSError, KeyError, TypeError, ValueError, RetrievalError):
pass
last_status: int | None = None
for attempt in range(self.config.attempts):
self._wait_for_slot()
response: httpx.Response | None = None
try:
with (
_secret_scope(active_secrets),
self._client.stream(
"GET",
endpoint,
params=public | private,
# PANGAEA rejects specific text Accept combinations.
headers={"Accept": "*/*"} | private_headers,
) as response,
):
last_status = response.status_code
if last_status == 200:
chunks = bytearray()
for chunk in response.iter_bytes(chunk_size=64 * 1024):
if len(chunks) + len(chunk) > self.config.maximum_response_bytes:
raise RetrievalError(
"Source response exceeded the configured byte limit"
)
chunks.extend(chunk)
raw = bytes(chunks)
try:
value = raw.decode("utf-8")
except UnicodeDecodeError:
raise RetrievalError("Source returned invalid UTF-8 text") from None
if active_secrets:
value = _redact_text(value, active_secrets)
raw = value.encode("utf-8")
record = RequestRecord(
endpoint,
public,
datetime.now(UTC).isoformat(),
hashlib.sha256(raw).hexdigest(),
len(raw),
)
if cache is not None:
atomic_bytes(
cache,
json.dumps(
{"record": asdict(record), "value": value},
ensure_ascii=False,
allow_nan=False,
).encode("utf-8"),
)
return TextResponse(value, record)
if last_status not in _RETRY_STATUSES:
break
except httpx.TransportError:
last_status = None
if attempt + 1 < self.config.attempts:
self._sleep(self._retry_delay(response, attempt))
status = f"HTTP {last_status}" if last_status is not None else "transport failure"
raise RetrievalError(f"Retrieval failed from {parsed.hostname}: {status}") from None
[docs]
def get_json(
self,
endpoint: str,
parameters: Mapping[str, str] | None = None,
*,
secrets: Mapping[str, str] | None = None,
secret_headers: Mapping[str, str] | None = None,
allow_legacy_charset: bool = False,
use_cache: bool = True,
public_pagination_token: bool = False,
) -> JSONResponse:
"""Fetch JSON, preserving zero/null values and non-secret request provenance.
``public_pagination_token`` opts into a provider's non-authentication
``token`` cursor, including next links on its first page. Other credential
names and explicit secrets remain redacted; authentication tokens cannot
use this option. The default treats all token parameters as secrets.
Header credentials are never included in request provenance or cache keys.
``allow_legacy_charset`` permits only a declared Latin-1/Latin-9 fallback
when Unicode JSON decoding fails; it never overrides valid Unicode JSON.
"""
parsed = urlsplit(endpoint)
if (
parsed.scheme != "https"
or parsed.hostname not in self.allowed_hosts
or parsed.username is not None
or parsed.password is not None
or parsed.query
or parsed.fragment
or parsed.port not in (None, 443)
):
raise ValueError("Endpoint must be a query-free HTTPS URL on an approved host")
public = dict(parameters or {})
if any(
name.lower() in _SECRET_NAMES and not (public_pagination_token and name == "token")
for name in public
):
raise ValueError("Pass credential parameters through secrets, never parameters")
private = dict(secrets or {})
private_headers = dict(secret_headers or {})
header_names = {name.lower() for name in private_headers}
if len(header_names) != len(private_headers) or not header_names <= _SECRET_NAMES | {
"x-api-key",
"x-auth-token",
}:
raise ValueError("secret_headers requires unique credential header names")
if any(
not value
or not value.isascii()
or any(ord(character) < 32 or ord(character) == 127 for character in value)
for value in private_headers.values()
):
raise ValueError(
"Secret header values must be nonempty ASCII without control characters"
)
active_secrets = (*private.values(), *private_headers.values())
if public_pagination_token:
if any(name.lower() == "token" for name in private):
raise ValueError(
"An authentication token cannot also be a public pagination cursor"
)
cursor = public.get("token")
if cursor is not None and (
not isinstance(cursor, str)
or not 1 <= len(cursor) <= 8192
or any(ord(character) < 32 for character in cursor)
):
raise ValueError("Public pagination token must be a bounded nonempty string")
if any(not value for value in private.values()) or set(public) & set(private):
raise ValueError("Secret parameters must be nonempty and separate")
cache_key = hashlib.sha256(
json.dumps(
[endpoint, public, public_pagination_token, allow_legacy_charset],
sort_keys=True,
separators=(",", ":"),
).encode()
).hexdigest()
cache = (
self.cache_directory / f"{cache_key}.json"
if use_cache and not active_secrets and self.cache_directory is not None
else None
)
if cache is not None and cache.exists():
try:
saved = object_value(_decode(cache.read_bytes()))
record = cast(dict[str, object], object_value(saved["record"]))
captured = datetime.fromisoformat(str(record["retrieved_at_utc"]))
body = json.dumps(saved["value"], separators=(",", ":"), allow_nan=False).encode()
if (
0
<= (datetime.now(UTC) - captured).total_seconds()
<= self.config.cache_ttl_seconds
and hashlib.sha256(body).hexdigest() == record["sha256"]
and len(body) == record["size_bytes"]
):
return JSONResponse(
saved["value"],
RequestRecord(
endpoint,
public,
str(record["retrieved_at_utc"]),
str(record["sha256"]),
len(body),
True,
),
)
except (OSError, KeyError, TypeError, ValueError, RetrievalError):
pass # A malformed cache entry is ignored, never accepted as source truth.
last_status: int | None = None
for attempt in range(self.config.attempts):
self._wait_for_slot()
response: httpx.Response | None = None
try:
with (
_secret_scope(active_secrets),
self._client.stream(
"GET", endpoint, params=public | private, headers=private_headers
) as response,
):
last_status = response.status_code
if response.status_code == 200:
chunks = bytearray()
for chunk in response.iter_bytes():
if len(chunks) + len(chunk) > self.config.maximum_response_bytes:
raise RetrievalError(
"Source response exceeded the configured byte limit"
)
chunks.extend(chunk)
raw = bytes(chunks)
value = _redact_value(
_decode(
raw,
legacy_encoding=(
_legacy_json_encoding(response.headers.get("Content-Type", ""))
if allow_legacy_charset
else None
),
),
active_secrets,
public_pagination_token=public_pagination_token,
)
# Canonical bytes identify the sanitized stored response.
body = json.dumps(value, separators=(",", ":"), allow_nan=False).encode()
record = RequestRecord(
endpoint,
public,
datetime.now(UTC).isoformat(),
hashlib.sha256(body).hexdigest(),
len(body),
)
if cache is not None:
atomic_bytes(
cache,
json.dumps(
{"record": asdict(record), "value": value},
allow_nan=False,
).encode(),
)
return JSONResponse(value, record)
if response.status_code not in _RETRY_STATUSES:
break
except httpx.TransportError:
last_status = None
if attempt + 1 < self.config.attempts:
self._sleep(self._retry_delay(response, attempt))
status = f"HTTP {last_status}" if last_status is not None else "transport failure"
raise RetrievalError(f"Retrieval failed from {parsed.hostname}: {status}") from None