स्टेप 15 · रीज़निंग मॉडल्स (Reasoning models)

def grpo_step(model, ref_model, prompts, verify_fn, group=8, beta=0.04):
    losses = []
    for prompt in prompts:
        # 1. sample a group of independent attempts
        completions = [model.generate(prompt, temperature=1.0)
                       for _ in range(group)]

        # 2. score each one objectively -- no reward model needed
        rewards = torch.tensor([verify_fn(prompt, c) for c in completions])

        # 3. advantage = how much better than the group average
        adv = (rewards - rewards.mean()) / (rewards.std() + 1e-6)

        # 4. push probability toward above-average attempts,
        #    with a KL leash back to the reference model
        for c, a in zip(completions, adv):
            logp     = model.logprob(prompt, c)
            ref_logp = ref_model.logprob(prompt, c)
            losses.append(-(a * logp) + beta * (logp - ref_logp))
    return torch.stack(losses).mean()

def verify_fn(prompt, completion):
    """1.0 if the final answer is provably correct, else 0.0."""
    return float(extract_answer(completion) == known_answer(prompt))