Source code for space_pv_connectors.arxiv

"""Version-pinned arXiv metadata with original-byte provenance and offline cache."""

from __future__ import annotations

import hashlib
import json
import re
import xml.etree.ElementTree as ET
from dataclasses import asdict
from pathlib import Path

import httpx

from .http_base import DownloadReceipt
from .research_http import ResearchConnector, ResearchResponse

ATOM = "{http://www.w3.org/2005/Atom}"


def _paper_id(value: str) -> str:
    if not isinstance(value, str) or not re.fullmatch(r"\d{4}\.\d{4,5}v[1-9]\d*", value):
        raise ValueError("Use an explicit modern arXiv version, e.g. 2511.19468v2")
    return value


[docs] def parse_arxiv_atom(content: bytes, paper_id: str) -> dict[str, object]: """Parse one exact-version Atom entry; never silently substitute latest.""" paper_id = _paper_id(paper_id) if b"<!DOCTYPE" in content.upper() or b"<!ENTITY" in content.upper(): raise ValueError("DTD/entity declarations are not accepted") try: root = ET.fromstring(content) except ET.ParseError as exc: raise ValueError("arXiv did not return valid Atom XML") from exc if root.tag != ATOM + "feed": raise ValueError("Expected an arXiv Atom feed") entries = root.findall(ATOM + "entry") if len(entries) != 1: raise LookupError("Expected exactly one arXiv entry") entry = entries[0] identifier = entry.findtext(ATOM + "id", "") if identifier not in (f"http://arxiv.org/abs/{paper_id}", f"https://arxiv.org/abs/{paper_id}"): raise ValueError("arXiv entry does not match the requested version") fields = { k: " ".join(entry.findtext(ATOM + k, "").split()) for k in ("title", "published", "updated") } authors = [a.findtext(ATOM + "name", "").strip() for a in entry.findall(ATOM + "author")] if not all(fields.values()) or not authors or not all(authors): raise ValueError("Incomplete arXiv metadata") return { "paper_id": paper_id, **fields, "authors": authors, "url": f"https://arxiv.org/abs/{paper_id}", "pdf_url": f"https://arxiv.org/pdf/{paper_id}", "categories": [c.get("term", "") for c in entry.findall(ATOM + "category")], "evidence_kind": "bibliographic_metadata_not_validated_engineering_data", }
[docs] class ArxivConnector(ResearchConnector): """One explicit paper per request; reuse cache and observe arXiv rate limits. The original Atom bytes are cached with SHA-256, URL and retrieval time. This connector does not extract or promote scientific claims from PDFs. Downloading an accessible paper does not grant redistribution rights. """ SOURCE = "arXiv" DOCUMENTATION = "https://info.arxiv.org/help/api/user-manual.html" BASE_URL = "https://export.arxiv.org/api/query"
[docs] def paper( self, paper_id: str, *, cache_path: str | Path | None = None, refresh: bool = False ) -> ResearchResponse: """Fetch pinned metadata, or validate and read its existing local cache.""" paper_id = _paper_id(paper_id) params = {"id_list": paper_id, "max_results": 1} request_url = str(httpx.URL(self.BASE_URL, params=params)) path = Path(cache_path) if cache_path is not None else None sidecar = path.with_name(path.name + ".provenance.json") if path else None cached = path is not None and path.exists() and not refresh if cached: if sidecar is None or not sidecar.exists(): raise ValueError("Cache has no provenance sidecar; use refresh=True") provenance = json.loads(sidecar.read_text(encoding="utf-8")) content = path.read_bytes() if ( provenance.get("request_url") != request_url or provenance.get("source") != self.SOURCE ): raise ValueError("Cache belongs to a different request or provider") if provenance.get("sha256") != hashlib.sha256(content).hexdigest(): raise ValueError("Cache checksum does not match its provenance") receipt = DownloadReceipt( **{k: provenance[k] for k in DownloadReceipt.__dataclass_fields__} ) else: content_buffer = bytearray() with self._client.stream( "GET", self.BASE_URL, params=params, headers={"Accept": "application/atom+xml"} ) as response: response.raise_for_status() for chunk in response.iter_bytes(): if len(content_buffer) + len(chunk) > 2_000_000: raise ValueError("arXiv metadata exceeds 2 MB") content_buffer.extend(chunk) receipt = self.receipt( self.SOURCE, str(response.url), bytes(content_buffer), path=path ) content = bytes(content_buffer) data = parse_arxiv_atom(content, paper_id) if not cached and path is not None and sidecar is not None: path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(content) sidecar.write_text( json.dumps( { **asdict(receipt), "request_url": request_url, "documentation": self.DOCUMENTATION, }, indent=2, ) + "\n", encoding="utf-8", ) return ResearchResponse(data, receipt, request_url, self.DOCUMENTATION)
[docs] def download_paper( self, paper_id: str, destination: str | Path, *, max_bytes: int = 20_000_000 ) -> DownloadReceipt: """Download a selected version with a checksum receipt; no bulk crawl.""" paper_id = _paper_id(paper_id) return self._download( f"https://arxiv.org/pdf/{paper_id}", destination, max_bytes=max_bytes, context={"paper_id": paper_id}, )