Capítulo 2: Teste A/B de Prompts

Desenho do Experimento

from dataclasses import dataclass
import random

@dataclass
class PromptExperiment:
    name: str
    control_prompt: str
    treatment_prompt: str
    traffic_split: float  # % to treatment
    metrics: list[str]
    min_sample_size: int

class PromptABTester:
    def __init__(self, experiment: PromptExperiment):
        self.experiment = experiment
        self.results = {"control": [], "treatment": []}
    
    def get_prompt(self, user_id: str) -> tuple[str, str]:
        """Assign user to variant based on consistent hashing."""
        hash_value = hash(f"{self.experiment.name}:{user_id}") % 100
        
        if hash_value < self.experiment.traffic_split * 100:
            return "treatment", self.experiment.treatment_prompt
        return "control", self.experiment.control_prompt
    
    def record_metric(self, variant: str, metric: str, value: float):
        self.results[variant].append({"metric": metric, "value": value})
    
    def analyze(self) -> dict:
        """Analyze experiment results."""
        from scipy import stats
        
        control_values = [r["value"] for r in self.results["control"]]
        treatment_values = [r["value"] for r in self.results["treatment"]]
        
        t_stat, p_value = stats.ttest_ind(control_values, treatment_values)
        
        return {
            "control_mean": sum(control_values) / len(control_values),
            "treatment_mean": sum(treatment_values) / len(treatment_values),
            "p_value": p_value,
            "significant": p_value < 0.05
        }

---

Próximo: Capítulo 3: Otimização de Prompts