17. Compression Evaluation

Chapter 17 of 18 · 25 min

KEY INSIGHT

Thorough evaluation of compressed models requires checking not just aggregate metrics but also behavioral consistency with the original model across diverse inputs. Evaluating compression requires more than comparing accuracy numbers. The compressed model must behave consistently with the original across different input types, edge cases, and confidence patterns. ### Behavioral Consistency ```python class BehavioralEvaluator: def __init__(self, original_model, compressed_model, test_data): self.original = original_model self.compressed = compressed_model self.test_data = test_data def evaluate_behavioral_alignment(self): """ Evaluate how well compressed model aligns with original's behavior. """ self.original.eval() self.compressed.eval() results = { 'confidence_calibration': self._check_calibration(), 'prediction_agreement': self._check_prediction_agreement(), 'error_correlation': self._check_error_correlation(), 'adversarial_reliableness': self._check_adversarial_reliableness() } return results def _check_prediction_agreement(self): """ Check what fraction of predictions match between models. """ agreements = [] confidence_agreement = [] with torch.no_grad(): for batch in self.test_data: inputs = batch['input'] orig_outputs = self.original(inputs) comp_outputs = self.compressed(inputs) # Top-1 agreement orig_pred = orig_outputs.argmax(dim=1) comp_pred = comp_outputs.argmax(dim=1) agreements.append((orig_pred == comp_pred).float().mean().item()) # Confidence alignment orig_conf = F.softmax(orig_outputs, dim=1).max(dim=1)[0] comp_conf = F.softmax(comp_outputs, dim=1).max(dim=1)[0] confidence_agreement.append( torch.abs(orig_conf - comp_conf).mean().item() ) return { 'mean_top1_agreement': np.mean(agreements), 'mean_confidence_diff': np.mean(confidence_agreement) } ``` ### Calibration Analysis ```python def _check_calibration(self): """ Evaluate whether confidence scores match actual accuracy. """ from sklearn.calibration import calibration_curve all_probs = [] all_labels = [] all_preds = [] with torch.no_grad(): for batch in self.test_data: inputs = batch['input'] labels = batch['target'] outputs = self.compressed(inputs) probs = F.softmax(outputs, dim=1) all_probs.append(probs.cpu().numpy()) all_labels.append(labels.cpu().numpy()) all_probs = np.concatenate(all_probs) all_labels = np.concatenate(all_labels) all_preds = all_probs.argmax(axis=1) # Expected Calibration Error (ECE) n_bins = 10 bin_boundaries = np.linspace(0, 1, n_bins + 1) ece = 0 total = len(all_labels) for i in range(n_bins): bin_lower = bin_boundaries[i] bin_upper = bin_boundaries[i + 1] in_bin = (all_probs.max(axis=1) >= bin_lower) & \ (all_probs.max(axis=1) < bin_upper) if in_bin.sum() > 0: bin_accuracy = (all_preds[in_bin] == all_labels[in_bin]).mean() bin_confidence = all_probs.max(axis=1)[in_bin].mean() ece += (in_bin.sum() / total) * abs(bin_accuracy - bin_confidence) return { 'expected_calibration_error': ece, 'n_samples': total } ``` ### Critical Failure Detection ```python def detect_critical_failures(self): """ Find cases where compressed model fails badly while original succeeds. """ critical_failures = [] with torch.no_grad(): for i, batch in enumerate(self.test_data): inputs = batch['input'] labels = batch['target'] orig_outputs = self.original(inputs) comp_outputs = self.compressed(inputs) orig_pred = orig_outputs.argmax(dim=1) comp_pred = comp_outputs.argmax(dim=1) # Critical failure: original correct, compressed wrong orig_correct = (orig_pred == labels) comp_wrong = (comp_pred != labels) critical_mask = orig_correct & comp_wrong if critical_mask.any(): indices = critical_mask.nonzero().squeeze() for idx in indices: confidence_original = F.softmax( orig_outputs[idx:idx+1], dim=1 ).max().item() confidence_compressed = F.softmax( comp_outputs[idx:idx+1], dim=1 ).max().item() critical_failures.append({ 'sample_idx': i * batch['input'].shape[0] + idx.item(), 'original_confidence': confidence_original, 'compressed_confidence': confidence_compressed, 'gap': confidence_original - confidence_compressed }) # Sort by confidence gap critical_failures.sort(key=lambda x: x['gap'], reverse=True) return critical_failures[:20] # Top 20 most critical ``` ### Fairness Evaluation ```python def evaluate_fairness(self, sensitive_attribute): """ Check if compression affects different groups equally. """ group_metrics = {} with torch.no_grad(): for batch in self.test_data: inputs = batch['input'] labels = batch['target'] groups = batch[sensitive_attribute] outputs = self.compressed(inputs) preds = outputs.argmax(dim=1) correct = (preds == labels) for group_id in torch.unique(groups): mask = (groups == group_id) if group_id.item() not in group_metrics: group_metrics[group_id.item()] = {'correct': 0, 'total': 0} group_metrics[group_id.item()]['correct'] += correct[mask].sum().item() group_metrics[group_id.item()]['total'] += mask.sum().item() # Compute per-group accuracy fairness_report = {} accuracies = [] for group_id, metrics in group_metrics.items(): acc = metrics['correct'] / metrics['total'] accuracies.append(acc) fairness_report[group_id] = { 'accuracy': acc, 'samples': metrics['total'] } # Disparate impact ratio min_acc = min(accuracies) max_acc = max(accuracies) fairness_report['disparate_impact_ratio'] = min_acc / max_acc return fairness_report ``` ### Evaluation Summary Report ```python def generate_evaluation_report(evaluator, original_model, compressed_model): """ Generate thorough evaluation report. """ results = evaluator.evaluate_behavioral_alignment() critical = evaluator.detect_critical_failures() fairness = evaluator.evaluate_fairness('group_id') report = [] report.append("=" * 60) report.append("COMPRESSION EVALUATION REPORT") report.append("=" * 60) report.append(f"\nBehavioral Alignment:") report.append(f" Prediction Agreement: {results['prediction_agreement']['mean_top1_agreement']:.4f}") report.append(f" Calibration ECE: {results['confidence_calibration']['expected_calibration_error']:.4f}") report.append(f"\nCritical Failures: {len(critical)}") if critical: report.append(f" Top failure confidence gap: {critical[0]['gap']:.4f}") report.append(f"\nFairness Analysis:") for group, metrics in fairness.items(): if isinstance(group, int): report.append(f" Group {group}: {metrics['accuracy']:.4f} ({metrics['samples']} samples)") else: report.append(f" {group}: {metrics:.4f}") report.append("=" * 60) return "\n".join(report) ```

EXERCISE

Write an evaluation suite that measures accuracy, inference latency, and model size for each compressed variant. Generate a summary table ranking variants by efficiency.