अपने कंप्यूटर पर मॉडल चलाना

def estimate_gb(params_billion: float, bits: int, overhead: float = 1.2) -> float:
    """Weights only, times a rough overhead factor for runtime and context."""
    bytes_per_param = bits / 8
    weights_gb = params_billion * bytes_per_param   # 1B params x 1 byte = about 1 GB
    return round(weights_gb * overhead, 1)

for size in (1, 3, 7):
    for bits in (16, 8, 4):
        print(f"{size}B model at {bits}-bit: about {estimate_gb(size, bits)} GB")

# Ollama: download a model and chat with it in the terminal
ollama run <model-name>
ollama list                 # models you have downloaded

# llama.cpp: run a GGUF file you downloaded yourself
llama-cli -m ./my-model.gguf -p "Explain UPI in two sentences."

# llama.cpp: start a local server your own code can call
llama-server -m ./my-model.gguf --port 8080

import requests

resp = requests.post(
    "http://localhost:11434/api/generate",
    json={"model": "<model-name>", "prompt": "Give three tips for saving water at home.", "stream": False},
    timeout=120,
)
resp.raise_for_status()
print(resp.json()["response"])