Chapter 3: Prompt Optimization Pipelines

DSPy for Prompt Optimization

import dspy

# Configure language model
lm = dspy.OpenAI(model="gpt-4")
dspy.configure(lm=lm)

class QuestionAnswering(dspy.Module):
    def __init__(self):
        super().__init__()
        self.generate_answer = dspy.ChainOfThought("context, question -> answer")
    
    def forward(self, context: str, question: str):
        return self.generate_answer(context=context, question=question)

# Optimize with examples
from dspy.teleprompt import BootstrapFewShot

optimizer = BootstrapFewShot(metric=answer_accuracy)
optimized_qa = optimizer.compile(QuestionAnswering(), trainset=train_examples)

Automated Optimization Loop

class PromptOptimizer:
    def __init__(self, base_prompt: str, evaluation_dataset: list):
        self.base_prompt = base_prompt
        self.evaluation_dataset = evaluation_dataset
        self.history = []
    
    def optimize(self, iterations: int = 10):
        current_prompt = self.base_prompt
        current_score = self.evaluate(current_prompt)
        
        for i in range(iterations):
            # Generate variations
            variations = self.generate_variations(current_prompt)
            
            # Evaluate each
            for variation in variations:
                score = self.evaluate(variation)
                
                if score > current_score:
                    current_prompt = variation
                    current_score = score
                    
            self.history.append({
                "iteration": i,
                "prompt": current_prompt,
                "score": current_score
            })
        
        return current_prompt, current_score

---

Next: Module 13 Quiz