Regression tests for prompts and models

import json
import sys

def call_model(prompt: str) -> str:
    ...  # plug in your provider

def check(case, output):
    text = (output or "").lower()
    ok_in = all(p.lower() in text for p in case.get("must_include", []))
    ok_out = not any(p.lower() in text for p in case.get("must_not", []))
    return ok_in and ok_out

def run(cases_path, baseline_path, max_drop=0.03):
    with open(cases_path, encoding="utf-8") as f:
        cases = [json.loads(line) for line in f if line.strip()]
    results = {c["id"]: check(c, call_model(c["input"])) for c in cases}
    score = sum(results.values()) / len(results)
    must_fail = [c["id"] for c in cases if c.get("must_pass") and not results[c["id"]]]
    with open(baseline_path, encoding="utf-8") as f:
        baseline = json.load(f)["score"]
    print(f"score {score:.1%} (baseline {baseline:.1%}), must-pass failures: {must_fail}")
    if must_fail or score < baseline - max_drop:
        sys.exit(1)  # non-zero exit fails the CI job

if __name__ == "__main__":
    run("evals/cases.jsonl", "evals/baseline.json")