स्टेप 1 · टोकनाइज़र

from collections import Counter

def get_pairs(ids):
    return Counter(zip(ids, ids[1:]))

def merge(ids, pair, new_id):
    out, i = [], 0
    while i < len(ids):
        if i < len(ids) - 1 and (ids[i], ids[i+1]) == pair:
            out.append(new_id); i += 2
        else:
            out.append(ids[i]); i += 1
    return out

def train_bpe(text, vocab_size=512):
    ids = list(text.encode("utf-8"))       # start: raw bytes, 0-255
    merges = {}
    for new_id in range(256, vocab_size):
        pairs = get_pairs(ids)
        if not pairs: break
        best = max(pairs, key=pairs.get)   # most frequent adjacent pair
        ids = merge(ids, best, new_id)
        merges[best] = new_id
    return merges

def encode(text, merges):
    ids = list(text.encode("utf-8"))
    for pair, new_id in merges.items():    # apply in training order
        ids = merge(ids, pair, new_id)
    return ids

def decode(ids, merges):
    vocab = {i: bytes([i]) for i in range(256)}
    for (a, b), new_id in merges.items():
        vocab[new_id] = vocab[a] + vocab[b]
    return b"".join(vocab[i] for i in ids).decode("utf-8", errors="replace")