Ollama: Local LLMs

# Using Homebrew
brew install ollama

# Or download from ollama.ai

curl -fsSL https://ollama.ai/install.sh | sh

# Download installer from ollama.ai
# Or use WSL2 with Linux installation

docker run -d -v ollama:/root/.ollama -p 11434:11434 --name ollama ollama/ollama

# With GPU support
docker run -d --gpus=all -v ollama:/root/.ollama -p 11434:11434 --name ollama ollama/ollama

# Start Ollama service
ollama serve

# In another terminal, pull and run a model
ollama run llama2

# Chat with the model
>>> What is machine learning?
Machine learning is a subset of artificial intelligence...

# Popular models
ollama pull llama2              # Meta's Llama 2 (7B default)
ollama pull llama2:13b          # Larger Llama 2
ollama pull mistral             # Mistral 7B
ollama pull mixtral             # Mistral's MoE model
ollama pull codellama           # Code-specialized Llama
ollama pull phi                 # Microsoft's small model
ollama pull neural-chat         # Intel's chat model
ollama pull starling-lm         # Berkeley's model

# List downloaded models
ollama list

# Remove a model
ollama rm llama2

import ollama

# Simple generation
response = ollama.generate(
    model='llama2',
    prompt='Explain quantum computing in simple terms'
)
print(response['response'])

# Chat interface
response = ollama.chat(
    model='llama2',
    messages=[
        {'role': 'user', 'content': 'Why is the sky blue?'}
    ]
)
print(response['message']['content'])

import ollama

# Stream the response
for chunk in ollama.chat(
    model='llama2',
    messages=[{'role': 'user', 'content': 'Write a poem about AI'}],
    stream=True
):
    print(chunk['message']['content'], end='', flush=True)

import requests

# Ollama exposes an OpenAI-compatible API
response = requests.post(
    'http://localhost:11434/api/generate',
    json={
        'model': 'llama2',
        'prompt': 'Hello, how are you?',
        'stream': False
    }
)
print(response.json()['response'])

from langchain_community.llms import Ollama
from langchain_community.chat_models import ChatOllama
from langchain.prompts import ChatPromptTemplate
from langchain_core.output_parsers import StrOutputParser

# Basic LLM
llm = Ollama(model="llama2")
response = llm.invoke("What is the capital of France?")

# Chat model
chat = ChatOllama(model="llama2")
prompt = ChatPromptTemplate.from_messages([
    ("system", "You are a helpful assistant."),
    ("human", "{input}")
])

chain = prompt | chat | StrOutputParser()
response = chain.invoke({"input": "Explain Docker"})

from langchain_community.embeddings import OllamaEmbeddings
from langchain_community.vectorstores import Chroma
from langchain_community.chat_models import ChatOllama
from langchain.chains import RetrievalQA

# Use Ollama for embeddings too
embeddings = OllamaEmbeddings(model="llama2")

# Create vector store
vectorstore = Chroma.from_documents(
    documents=docs,
    embedding=embeddings
)

# Create RAG chain
llm = ChatOllama(model="llama2")
qa_chain = RetrievalQA.from_chain_type(
    llm=llm,
    retriever=vectorstore.as_retriever()
)

result = qa_chain.invoke("What does the document say about...")

# Create a Modelfile
FROM llama2

# Set parameters
PARAMETER temperature 0.7
PARAMETER top_p 0.9
PARAMETER num_ctx 4096

# Set system prompt
SYSTEM """
You are a helpful coding assistant specialized in Python.
Always provide code examples when relevant.
Explain concepts clearly and concisely.
"""

# Add custom template
TEMPLATE """{{ if .System }}{{ .System }}{{ end }}
{{ if .Prompt }}User: {{ .Prompt }}{{ end }}
Assistant: """

# Build and run custom model
ollama create python-assistant -f Modelfile
ollama run python-assistant

>>> How do I read a CSV file?

# Model sizes and use cases

| Model          | Size   | Best For                    |
|----------------|--------|------------------------------|
| phi            | 2.7B   | Fast, simple tasks          |
| llama2:7b      | 7B     | General purpose, balanced   |
| mistral        | 7B     | High quality, efficient     |
| codellama      | 7B     | Code generation             |
| llama2:13b     | 13B    | Better reasoning            |
| mixtral        | 47B*   | State-of-art open source    |

* Mixtral uses 12B active parameters (MoE)

from openai import OpenAI

# Point to local Ollama
client = OpenAI(
    base_url="http://localhost:11434/v1",
    api_key="ollama"  # Not used but required
)

response = client.chat.completions.create(
    model="llama2",
    messages=[
        {"role": "user", "content": "Hello!"}
    ]
)
print(response.choices[0].message.content)