Ollama: Run Open-Weight LLMs Locally, REST API and Modelfile Reference Guide

# macOS / Linux installer (auto-starts service)
curl -fsSL https://ollama.com/install.sh | sh

# brew (macOS)
brew install ollama && brew services start ollama

# Docker
docker run -d --gpus=all -v ollama:/root/.ollama -p 11434:11434 \
  --name ollama ollama/ollama

# Verify
curl http://127.0.0.1:11434/api/tags

# Pull a model
ollama pull llama3.2:3b
ollama pull nomic-embed-text

# Python client (optional)
pip install ollama

from ollama import Client

client = Client(host="http://127.0.0.1:11434")

# Chat — multi-turn
resp = client.chat(
    model="llama3.2:3b",
    messages=[
        {"role": "system", "content": "Answer in one sentence."},
        {"role": "user",   "content": "What is RAG?"},
    ],
    options={"temperature": 0.2, "num_ctx": 4096},
)
print(resp.message.content)

# Stream
for chunk in client.chat(model="llama3.2:3b",
                         messages=[{"role": "user", "content": "Count to 5"}],
                         stream=True):
    print(chunk.message.content, end="", flush=True)

# Embed
e = client.embed(model="nomic-embed-text", input=["hello", "world"])
print(len(e.embeddings), len(e.embeddings[0]))

# Modelfile — declarative spec for a custom model
FROM llama3.2:3b

# System prompt baked in
SYSTEM """
You are a senior Python engineer. Reply with code first, prose second.
"""

# Generation defaults
PARAMETER temperature 0.3
PARAMETER top_p 0.9
PARAMETER num_ctx 8192
PARAMETER repeat_penalty 1.1
PARAMETER stop "<|end|>"

# Optional: chat template override
TEMPLATE """{{ if .System }}<|system|>{{ .System }}<|end|>
{{ end }}<|user|>{{ .Prompt }}<|end|>
<|assistant|>"""

# Build it:
#   ollama create py-coder -f Modelfile
#   ollama run py-coder "factorial in python"

from ollama import Client
from pydantic import BaseModel

class Person(BaseModel):
    name: str
    age: int
    city: str

client = Client()

resp = client.chat(
    model="llama3.2:3b",
    messages=[{"role": "user",
               "content": "Extract: Ada is 29 and lives in Berlin."}],
    format=Person.model_json_schema(),     # constrained JSON output
    options={"temperature": 0},
)

person = Person.model_validate_json(resp.message.content)
print(person)
# Person(name='Ada', age=29, city='Berlin')

from ollama import Client

def get_weather(city: str) -> str:
    return f"{city}: 22C, clear"

tools = [{
    "type": "function",
    "function": {
        "name": "get_weather",
        "description": "Current weather for a city",
        "parameters": {
            "type": "object",
            "properties": {"city": {"type": "string"}},
            "required": ["city"],
        },
    },
}]

client = Client()
resp = client.chat(
    model="llama3.2:3b",        # must be a tool-calling capable model
    messages=[{"role": "user", "content": "Weather in Tokyo?"}],
    tools=tools,
)

for call in resp.message.tool_calls or []:
    name, args = call.function.name, call.function.arguments
    result = {"get_weather": get_weather}[name](**args)
    print(name, "->", result)

from ollama import Client
import numpy as np

client = Client(host="http://127.0.0.1:11434")

# 1. Embed a tiny corpus
corpus = [
    "Ollama runs LLMs locally.",
    "FastAPI is a Python web framework.",
    "Vector databases store embeddings for retrieval.",
]
M = np.array(
    client.embed(model="nomic-embed-text", input=corpus).embeddings,
    dtype=np.float32,
)
M /= np.linalg.norm(M, axis=1, keepdims=True)

# 2. Retrieve
question = "how do I run a model on my laptop?"
q = np.array(client.embed(model="nomic-embed-text", input=question).embeddings[0])
q /= np.linalg.norm(q)
top = (M @ q).argsort()[::-1][:2]
context = "\n".join(corpus[i] for i in top)

# 3. Answer
resp = client.chat(
    model="llama3.2:3b",
    messages=[
        {"role": "system", "content": f"Use context:\n{context}"},
        {"role": "user",   "content": question},
    ],
)
print(resp.message.content)