Source code for space_pv_connectors.spenvis

"""Read the documented SPENVIS/UNIRAD block-structured CSV interchange format.

This is an importer for user-exported results, not a SPENVIS or IRENE service
client. Model configuration, statistical confidence, shielding, energy-bin
meaning and units are preserved rather than inferred from a filename.

Specification: https://space-env.esa.int/Manuals/Mulassis/ml-v02-00/OutputFiles.html
"""

from __future__ import annotations

import csv
import hashlib
from dataclasses import dataclass
from pathlib import Path
from typing import Any

import pandas as pd


[docs] @dataclass(frozen=True) class SpenvisVariable: name: str units: str dimension: int title: str metadata_names: tuple[str, ...] = ()
[docs] @dataclass(frozen=True) class SpenvisBlock: comments: tuple[str, ...] metadata: dict[str, dict[str, Any]] annotations: tuple[str, ...] variables: tuple[SpenvisVariable, ...] data: pd.DataFrame footer: str following_blocks: int
def _record(line: str, number: int) -> list[str]: try: return [ value.strip() for value in next( csv.reader( [line], delimiter=",", quotechar="'", skipinitialspace=True, strict=True, ) ) ] except (csv.Error, StopIteration) as exc: raise ValueError(f"Malformed SPENVIS record at line {number}") from exc def _real(value: str) -> float: return float(value.replace("D", "E").replace("d", "e"))
[docs] def parse_spenvis(text: str) -> list[SpenvisBlock]: """Parse scalar/vector variables, metadata, annotations and multiple blocks. Vector elements become ``NAME[0]``, ``NAME[1]``, etc. Their energy/coordinate association remains in ``metadata`` and must be selected explicitly. Units are retained verbatim in each variable and in ``data.attrs['units']``. Truncated records, inconsistent column/header counts, duplicate names, and provider error footers raise ``ValueError``. FORTRAN D exponents are supported. """ lines = text.lstrip("\ufeff").splitlines() blocks: list[SpenvisBlock] = [] offset = 0 while offset < len(lines): if not lines[offset].strip(): offset += 1 continue header_line = offset + 1 fields = _record(lines[offset], header_line) if len(fields) != 9 or fields[0] != "*": raise ValueError(f"Expected SPENVIS block header at line {header_line}") try: nh, nc, nm, na, nv, nd, nl, nb = map(int, fields[1:]) except ValueError as exc: raise ValueError(f"Noninteger block header at line {header_line}") from exc if min(nc, nm, na, nb) < 0 or nv < 1 or nd < 1 or nl < -1 or nh != 1 + nc + nm + na + nv: raise ValueError(f"Inconsistent SPENVIS header at line {header_line}") if offset + nh >= len(lines): raise ValueError("Truncated SPENVIS header or missing data footer") offset += 1 comments = [] for _ in range(nc): record = _record(lines[offset], offset + 1) if len(record) != 1: raise ValueError(f"Invalid comment at line {offset + 1}") comments.append(record[0]) offset += 1 metadata: dict[str, dict[str, Any]] = {} for _ in range(nm): record = _record(lines[offset], offset + 1) try: name, kind = record[0], int(record[1]) count = 1 if kind < 0 else kind if ( not name or name in metadata or count < 1 or len(record) not in {2 + count, 3 + count} ): raise ValueError("Invalid metadata width or name") values = record[2 : 2 + count] metadata[name] = { "values": values[0] if kind < 0 else tuple(_real(value) for value in values), "units": record[-1] if len(record) == 3 + count else None, } except (IndexError, ValueError) as exc: raise ValueError(f"Invalid SPENVIS metadata at line {offset + 1}") from exc offset += 1 annotations = tuple(lines[offset : offset + na]) offset += na variables = [] columns: list[str] = [] units: dict[str, str] = {} for _ in range(nv): record = _record(lines[offset], offset + 1) try: name, unit, dimension, title = record[:4] size = int(dimension) if not name or size < 1: raise ValueError("Invalid variable name or dimension") new_columns = [name] if size == 1 else [f"{name}[{index}]" for index in range(size)] if any(column in units for column in new_columns): raise ValueError("Duplicate variable columns") columns.extend(new_columns) units.update(dict.fromkeys(new_columns, unit)) variables.append(SpenvisVariable(name, unit, size, title, tuple(record[4:]))) except ValueError as exc: raise ValueError(f"Invalid SPENVIS variable at line {offset + 1}") from exc offset += 1 if len(columns) != nd: raise ValueError("SPENVIS variable dimensions disagree with declared data columns") rows = [] footer = None while offset < len(lines): record = _record(lines[offset], offset + 1) if lines[offset].lstrip().startswith("'"): if len(record) != 1 or record[0] == "*": raise ValueError("Missing SPENVIS footer before next block") footer = record[0] offset += 1 break if len(record) != nd: raise ValueError(f"SPENVIS data width mismatch at line {offset + 1}") try: rows.append([_real(value) for value in record]) except ValueError as exc: raise ValueError(f"Invalid SPENVIS numeric data at line {offset + 1}") from exc offset += 1 if footer is None or (nl >= 0 and len(rows) != nl): raise ValueError("Truncated SPENVIS block or declared data row count mismatch") if footer.strip("* ").upper().startswith("ERROR"): raise ValueError(f"SPENVIS provider reported: {footer}") frame = pd.DataFrame(rows, columns=columns, dtype=float) frame.attrs.update( { "units": units, "metadata": metadata, "comments": tuple(comments), "source_format": "SPENVIS/UNIRAD CSV", "units_converted": False, } ) blocks.append( SpenvisBlock( tuple(comments), metadata, annotations, tuple(variables), frame, footer, nb ) ) if not blocks: raise ValueError("No SPENVIS blocks found") for index, block in enumerate(blocks): if block.following_blocks > len(blocks) - index - 1: raise ValueError("Truncated SPENVIS section: declared following blocks are missing") return blocks
[docs] def read_spenvis(path: str | Path, *, max_bytes: int = 128 * 1024 * 1024) -> list[SpenvisBlock]: """Read a local export and attach its checksum to each returned data frame.""" source = Path(path) if max_bytes <= 0 or source.stat().st_size > max_bytes: raise ValueError("SPENVIS file exceeds max_bytes") content = source.read_bytes() blocks = parse_spenvis(content.decode("utf-8-sig")) for block in blocks: block.data.attrs["source_file_sha256"] = hashlib.sha256(content).hexdigest() return blocks