"""Reproducible statistics and browser exports from PV Parquet snapshots.
No live requests are made. Missing values remain unknown; group sums describe
reported values, never an estimate of unreported capacity or market coverage.
"""
from __future__ import annotations
import hashlib
import json
from datetime import datetime
from pathlib import Path
from tempfile import NamedTemporaryFile
from typing import Any
import pandas as pd
import pyarrow as pa
import pyarrow.parquet as pq
[docs]
def field_profile(path: Path) -> pd.DataFrame:
"""Count known/missing values for every column, including all-null columns.
Completeness is non-null coverage, not accuracy or applicability. Empty lists
are reported values. False and zero are known values.
"""
table = pq.read_table(path)
records = []
for field, column in zip(table.schema, table.columns, strict=True):
metadata = field.metadata or {}
records.append(
{
"field": field.name,
"type": str(field.type),
"unit": metadata.get(b"unit", b"").decode(),
"known": len(column) - column.null_count,
"missing": column.null_count,
"coverage_pct": (
100 * (len(column) - column.null_count) / len(column) if len(column) else None
),
}
)
return pd.DataFrame(records).set_index("field")
[docs]
def grouped_statistics(path: Path, by: str, value: str) -> pd.DataFrame:
"""Group records and a numeric field, retaining unknown groups and values.
reported_sum includes only known values (all-missing groups stay null).
complete_sum is null unless *every* row in the group reports the field.
Never add AC, DC, unspecified capacity or parent/phase records together.
"""
schema = pq.read_schema(path)
value_type = schema.field(value).type
if not (pa.types.is_integer(value_type) or pa.types.is_floating(value_type)):
raise ValueError(f"{value} must be a numeric field, not {value_type}")
if by == value:
raise ValueError("Grouping and numeric fields must differ")
frame = pd.read_parquet(path, columns=[by, value])
groups = frame.groupby(by, dropna=False, sort=True)[value]
result = frame.groupby(by, dropna=False, sort=True).agg(
records=(value, "size"),
known=(value, "count"),
mean=(value, "mean"),
median=(value, "median"),
)
result["missing"] = result["records"] - result["known"]
result["reported_sum"] = groups.sum(min_count=1)
result["complete_sum"] = result["reported_sum"].where(result["missing"].eq(0))
return result
[docs]
def snapshot_overview(snapshot: Path) -> pd.DataFrame:
"""Read actual Parquet metadata for row/column counts and physical file sizes."""
manifest = json.loads((snapshot / "manifest.json").read_text())
records = []
for entity in manifest["tables"]:
path = snapshot / f"{entity}.parquet"
metadata = pq.read_metadata(path)
records.append(
{
"table": entity,
"rows": metadata.num_rows,
"columns": len(pq.read_schema(path)),
"parquet_bytes": path.stat().st_size,
}
)
return pd.DataFrame(records).set_index("table")
def _json_value(value: Any) -> Any:
if isinstance(value, datetime):
return value.isoformat()
return value
[docs]
def export_viewer_data(snapshot: Path) -> dict[str, Any]:
"""Export lossless browser projections of every table in a catalogue snapshot.
JSON is a derived delivery format, not an independently maintained database.
All rows and populated fields are retained; omitted all-null columns are
declared in the full schema and restored as null in the viewer. Lists stay
lists and UTC timestamps use ISO 8601. Hashes bind each projection to Parquet.
The snapshot must be trusted, generated by build_parquet.
"""
manifest = json.loads((snapshot / "manifest.json").read_text())
schema = json.loads((snapshot / "schema.json").read_text())
index: dict[str, Any] = {
"schema_version": manifest["schema_version"],
"input_sha256": manifest["input_sha256"],
"fields": schema["fields"],
"tables": {},
}
for entity in schema["entity_types"]:
path = snapshot / f"{entity}.parquet"
digest = hashlib.sha256(path.read_bytes()).hexdigest()
if digest != manifest["tables"][entity]["sha256"]:
raise ValueError(f"Parquet checksum mismatch: {entity}")
parquet = pq.ParquetFile(path)
count = parquet.metadata.num_rows
if count != manifest["tables"][entity]["rows"]:
raise ValueError(f"Parquet row count mismatch: {entity}")
# Two bounded passes avoid retaining a full Arrow table, row dictionaries,
# compact arrays and encoded JSON simultaneously for large catalogues.
populated: set[str] = set()
for batch in parquet.iter_batches(batch_size=128, use_threads=False):
populated.update(
field.name
for field, column in zip(batch.schema, batch.columns, strict=True)
if column.null_count < batch.num_rows
)
columns = [name for name in parquet.schema_arrow.names if name in populated]
filename = f"{entity}.viewer.json"
hasher, byte_count = hashlib.sha256(), 0
temporary: Path | None = None
try:
with NamedTemporaryFile(dir=snapshot, mode="wb", delete=False) as output:
temporary = Path(output.name)
def write(text: str, content_hasher=hasher) -> None:
nonlocal byte_count
encoded = text.encode("utf-8")
output.write(encoded)
content_hasher.update(encoded)
byte_count += len(encoded)
prefix = json.dumps(
{"parquet_sha256": digest, "columns": columns},
ensure_ascii=False,
separators=(",", ":"),
allow_nan=False,
)
write(prefix[:-1] + ',"rows":[')
emitted = 0
for batch in parquet.iter_batches(
batch_size=128, columns=columns, use_threads=False
):
for row in batch.to_pylist():
if emitted:
write(",")
write(
json.dumps(
[_json_value(row[name]) for name in columns],
ensure_ascii=False,
separators=(",", ":"),
allow_nan=False,
)
)
emitted += 1
if emitted != count:
raise ValueError(f"Parquet row count mismatch: {entity}")
write("]}\n")
# Preserve existing permissions; new documentation assets need the
# readable mode previously supplied by write_text, not tempfile's 0600.
target = snapshot / filename
temporary.chmod(target.stat().st_mode & 0o777 if target.exists() else 0o644)
temporary.replace(target)
finally:
if temporary is not None:
temporary.unlink(missing_ok=True)
index["tables"][entity] = {
"rows": count,
"parquet": path.name,
"parquet_sha256": digest,
"viewer": filename,
"viewer_sha256": hasher.hexdigest(),
"viewer_bytes": byte_count,
}
(snapshot / "viewer-index.json").write_text(
json.dumps(index, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
)
return index