Chapter 3: Comparing Experiments

Effective Experiment Comparison

Comparing experiments effectively is crucial for selecting the best model and understanding what drives performance.

MLflow Experiment Comparison

import mlflow
from mlflow.tracking import MlflowClient

client = MlflowClient()

# Get all runs from an experiment
experiment = client.get_experiment_by_name("customer-churn")
runs = client.search_runs(
    experiment_ids=[experiment.experiment_id],
    filter_string="metrics.auc_roc > 0.8",
    order_by=["metrics.auc_roc DESC"],
    max_results=10
)

# Compare runs
comparison = []
for run in runs:
    comparison.append({
        "run_id": run.info.run_id,
        "run_name": run.data.tags.get("mlflow.runName", ""),
        "auc_roc": run.data.metrics.get("auc_roc"),
        "accuracy": run.data.metrics.get("accuracy"),
        "n_estimators": run.data.params.get("n_estimators"),
        "max_depth": run.data.params.get("max_depth")
    })

import pandas as pd
df_comparison = pd.DataFrame(comparison)
print(df_comparison.to_markdown())

Visualization Best Practices

import matplotlib.pyplot as plt
import seaborn as sns

# Parallel coordinates plot for hyperparameter analysis
from pandas.plotting import parallel_coordinates

fig, ax = plt.subplots(figsize=(12, 6))
parallel_coordinates(
    df_comparison[['n_estimators', 'max_depth', 'auc_roc']], 
    'auc_roc',
    colormap='viridis'
)
plt.title("Hyperparameter Impact on AUC-ROC")
plt.savefig("hyperparameter_analysis.png")
mlflow.log_artifact("hyperparameter_analysis.png")

Experiment Selection Framework

class ExperimentSelector:
    """Framework for selecting the best experiment."""
    
    def __init__(self, experiments: list, weights: dict):
        self.experiments = experiments
        self.weights = weights  # e.g., {"auc_roc": 0.5, "latency": 0.3, "model_size": 0.2}
    
    def calculate_score(self, exp: dict) -> float:
        """Calculate weighted score for experiment."""
        score = 0
        for metric, weight in self.weights.items():
            value = exp.get(metric, 0)
            # Normalize (assume higher is better except for latency/size)
            if metric in ["latency", "model_size"]:
                value = 1 / (value + 1)  # Invert
            score += value * weight
        return score
    
    def select_best(self) -> dict:
        """Select best experiment based on weighted criteria."""
        scored = [(exp, self.calculate_score(exp)) for exp in self.experiments]
        return max(scored, key=lambda x: x[1])[0]

Key Takeaways

Summary

1. Use structured queries to filter and sort experiments
2. Visualize hyperparameter impact with parallel coordinates
3. Define selection criteria with weighted metrics
4. Document decisions for reproducibility
5. Automate comparison in CI/CD pipelines

---

Next: Module 3 Quiz