Source code for romtools.workflows._adaptive.validation

"""Leakage-free K-fold QoI validation and user-defined error measures."""

from dataclasses import dataclass
from typing import Callable, Tuple
import os

import numpy as np

from romtools.workflows.model_builders import QoiModelBuilderWithTrainingData
from romtools.workflows.models import QoiModel
from .execution import _evaluate_sample, _process_state
from .sampling import _positive_integer


def _error_value(value, description):
    scalar = np.asarray(value)
    if scalar.ndim != 0 or scalar.dtype.kind not in "iuf":
        raise ValueError(f"{description} must be a finite nonnegative scalar")
    result = float(scalar)
    if not np.isfinite(result) or result < 0:
        raise ValueError(f"{description} must be a finite nonnegative scalar")
    return result


[docs] def relative_l2_error(qoi_rom: np.ndarray, qoi_fom: np.ndarray) -> float: """Return ||q_rom-q_fom||_2 / max(||q_fom||_2, 1e-12).""" return float(np.linalg.norm(qoi_rom - qoi_fom) / max(np.linalg.norm(qoi_fom), 1e-12))
[docs] @dataclass(frozen=True) class SpaceFillingValidationContext: """Inputs to a validation callable returning one nonnegative scalar error. Parameter/QoI rows and training directories have matching order. Arrays are read-only copies. Builders must fit independent models using only their supplied subsets; a fold must not reuse the production ROM's fitted basis. A custom strategy owns aggregation and can use ``error_functor`` to compare its predictions with independently prepared reference QoIs. """ rom_model: QoiModel rom_model_builder: QoiModelBuilderWithTrainingData parameter_names: Tuple[str, ...] training_parameters: np.ndarray training_qois: np.ndarray training_dirs: Tuple[str, ...] iteration: int validation_directory: str random_seed: int error_functor: Callable
def _build_rom(builder, directory, training_dirs, parameters, qois, seed): os.makedirs(directory, exist_ok=False) with _process_state(seed): return builder.build_from_training_dirs( directory, list(training_dirs), parameters.copy(), qois.copy() )
[docs] @dataclass(frozen=True) class KFoldValidation: """Maximum held-out sample error from independent subset-trained ROMs. No additional FOM evaluations are performed. Each validation uses ``num_folds`` builds and one held-out ROM evaluation per training sample. The error functor comes from the context; its result is not renormalized. """ num_folds: int = 5 def __post_init__(self): _positive_integer(self.num_folds, "num_folds") if self.num_folds < 2: raise ValueError("num_folds must be at least 2") def __call__(self, context: SpaceFillingValidationContext) -> float: count = len(context.training_parameters) if count < self.num_folds: raise ValueError("K-fold validation requires at least num_folds training samples") rng = np.random.default_rng(context.random_seed) folds = np.array_split(rng.permutation(count), self.num_folds) maximum_error = 0.0 for fold_index, held_out in enumerate(folds): training = np.ones(count, dtype=bool) training[held_out] = False indices = np.flatnonzero(training) directory = os.path.join(context.validation_directory, f"fold_{fold_index}") try: rom = _build_rom( context.rom_model_builder, os.path.join(directory, "offline_data"), [context.training_dirs[i] for i in indices], context.training_parameters[indices], context.training_qois[indices], int(rng.integers(0, 2**32)), ) except Exception as error: raise RuntimeError(f"validation fold {fold_index} ROM construction failed: {error}") from error for index in held_out: index = int(index) prediction, _ = _evaluate_sample( rom, context.parameter_names, context.training_parameters[index], os.path.join(directory, f"run_{index}"), index, int(rng.integers(0, 2**32)), ) reference = context.training_qois[index] if prediction.shape != reference.shape: raise ValueError(f"validation fold {fold_index}, sample {index}: QoI dimension mismatch") try: error = _error_value( context.error_functor(prediction.copy(), reference.copy()), "error_functor result", ) except Exception as exception: raise ValueError( f"validation fold {fold_index}, sample {index}: {exception}" ) from exception maximum_error = max(maximum_error, error) return maximum_error
def _readonly_copy(array): result = array.copy() result.setflags(write=False) return result