12. Model Validation

Model validation evaluates trained models against defined criteria. Unlike data validation (inputs), model validation assesses outputs: predictions, performance, and behavior. It answers: "Does this model work well enough to deploy?"

The foundational check is holdout validation. Training data splits into train, validation, and test sets. The test set is never seen during training—it's the unbiased measure of performance.

# model_validation.py
import numpy as np
from sklearn.model_selection import cross_val_score, StratifiedKFold
from sklearn.metrics import (
    accuracy_score, precision_score, recall_score, 
    f1_score, roc_auc_score, confusion_matrix
)

def validate_model(model, X_test, y_test, threshold_accuracy=0.90):
    """Thorough model validation."""
    y_pred = model.predict(X_test)
    y_prob = model.predict_proba(X_test)[:, 1]
    
    metrics = {
        "accuracy": accuracy_score(y_test, y_pred),
        "precision": precision_score(y_test, y_pred, zero_division=0),
        "recall": recall_score(y_test, y_pred, zero_division=0),
        "f1": f1_score(y_test, y_pred, zero_division=0),
        "roc_auc": roc_auc_score(y_test, y_prob)
    }
    
    cm = confusion_matrix(y_test, y_pred)
    
    passed = metrics["accuracy"] >= threshold_accuracy
    
    return {
        "passed": passed,
        "metrics": metrics,
        "confusion_matrix": cm.tolist(),
        "threshold": threshold_accuracy
    }

# Cross-validation for reliability
def cross_validate(model, X, y, cv=5):
    """K-fold cross validation."""
    skf = StratifiedKFold(n_splits=cv, shuffle=True, random_state=42)
    
    scores = cross_val_score(
        model, X, y, 
        cv=skf, 
        scoring="accuracy",
        n_jobs=-1
    )
    
    return {
        "scores": scores.tolist(),
        "mean": scores.mean(),
        "std": scores.std(),
        "min": scores.min(),
        "max": scores.max()
    }

Course A007: MLOps for Local AI (Operator Track)

Chapters 13–24