Chapter 3: Safety Testing

Red Teaming

Red teaming involves deliberately trying to make the LLM produce harmful outputs.

class RedTeamer:
    def __init__(self, llm):
        self.llm = llm
        self.attack_categories = [
            "jailbreak_attempts",
            "prompt_injection",
            "harmful_content_elicitation",
            "pii_extraction",
            "bias_probing"
        ]
    
    def run_attack_suite(self, category: str) -> list:
        attacks = self.load_attacks(category)
        results = []
        
        for attack in attacks:
            response = self.llm.generate(attack["prompt"])
            is_safe = self.evaluate_safety(response)
            
            results.append({
                "attack": attack["name"],
                "category": category,
                "response": response[:200],
                "is_safe": is_safe
            })
        
        return results

Safety Benchmarks

Benchmark	Focus	Metrics
TruthfulQA	Truthfulness	MC accuracy
ToxiGen	Toxicity	Toxicity rate
BBQ	Social bias	Bias score
HarmBench	Harmful behaviors	Attack success rate

Content Filtering Pipeline

class ContentFilter:
    def __init__(self):
        self.toxicity_model = load_model("unitary/toxic-bert")
        
    def filter(self, text: str) -> dict:
        # Check toxicity
        toxicity_score = self.toxicity_model.predict(text)
        
        # Check for PII
        pii_found = self.detect_pii(text)
        
        # Check for blocked patterns
        blocked = self.check_blocked_patterns(text)
        
        return {
            "text": text,
            "is_safe": toxicity_score < 0.5 and not pii_found and not blocked,
            "toxicity": toxicity_score,
            "pii_detected": pii_found,
            "blocked_content": blocked
        }

---

Next: Module 12 Quiz