ai.llm

This package runs large language models of the Llama family on the processor, in pure Salam with no outside library. Download a model in GGUF form, read it with LoadGgufModel and make text with Generate, or drive the loop yourself with Forward and a Sampler (temperature, top-k, top-p and a repetition penalty).

It is meant for small models running inside your own program: classifying or rewriting short texts offline, a local assistant in a tool, or learning how a transformer works step by step. For large models or high speed, call a server such as llama.cpp or an API over net.http instead.

What runs, and what it costs

• The layer layout is Llama's: RMSNorm, rotary positions, grouped-query attention and a SwiGLU feed forward block, with the optional attention biases that Qwen2 uses. Qwen2.5 Instruct is tested; Llama and Mistral use the same layout. Architectures with other layouts, such as Phi, fail to load or produce nonsense.
• The file limits of gguf apply: under 2 GB, and only F32, F16, Q8_0, Q4_0 and Q4_1 weights, not the K-quants.
• Every weight is expanded to f32 when the model loads, so Qwen2.5 0.5B, a 0.7 GB file, needs about 3 GB of memory. The key/value cache is sized for cfg.ctx, which starts as the model's full context (32768 for Qwen); lower it before NewKVCache to save memory.
• encode does not recognise special tokens such as <|im_start|> in text: it splits them into ordinary pieces, which the model does not understand. Look their ids up in tok.index and insert them yourself, as the chat example does.

The tokenizer, sampling and generation also work on a small random model built with RandomModel, which is how the reference example at the bottom of this page runs without a download.

A chat answer, streamed

Instruction tuned models expect the conversation in the exact format they were trained on. Qwen uses <|im_start|> and <|im_end|> around each turn, and the prompt ends by opening the assistant's turn so the model writes the reply. The format of any model is in its GGUF file under tokenizer.chat_template; see gguf for reading it. Those markers are single special tokens, so add_turn pushes their ids from tok.index and only encodes the text between them.

Instead of Generate, which returns the whole text at the end, this program runs the loop itself so every token is printed as soon as it is chosen. The first Forward call reads the whole prompt; after that each call feeds only the newest token, because the KVCache remembers everything before it. The loop stops at the end of turn token, after 80 tokens, or when the context is full. cfg.ctx is lowered to 4096 first, which keeps the cache at about 100 MB instead of 800.

mkdir -p models
curl -L -o models/qwen2.5-0.5b-instruct-q8_0.gguf \
  https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct-GGUF/resolve/main/qwen2.5-0.5b-instruct-q8_0.gguf
salam run llm-guide-chat.salam

import ai.llm
import ai.tensor

func add_turn(tok &: llm.Tokenizer, ids &: Vector<int>, role: str, text: str):
    ids.push(tok.index.get("<|im_start|>"))
    piece := tok.encode(role + "\n" + text)
    each id in piece:
        ids.push(id)
    end
    if text.len() > 0:
        ids.push(tok.index.get("<|im_end|>"))
        newline := tok.encode("\n")
        each id in newline:
            ids.push(id)
        end
    end
end

func main:
    path := "models/qwen2.5-0.5b-instruct-q8_0.gguf"
    mut model := llm.Model {}
    mut tok := llm.NewTokenizer(llm.TOK_BPE)
    if not llm.LoadGgufModel(path, model, tok):
        println "could not load", path
        ret 1
    end
    tok.add_bos = false
    end_of_turn := tok.index.get("<|im_end|>")
    model.cfg.ctx = 4096
    mut cache := llm.NewKVCache(model.cfg)
    mut sampler := llm.NewSampler(42)
    sampler.temperature = 0.7
    sampler.top_p = 0.9
    sampler.repeat_penalty = 1.1
    mut ids := Vector {} as Vector<int>
    add_turn(tok, ids, "system", "You are a helpful assistant. Answer in one sentence.")
    add_turn(tok, ids, "user", "Why is the sky blue?")
    add_turn(tok, ids, "assistant", "")
    mut logits := llm.Forward(model, cache, ids)
    mut said := Vector {} as Vector<int>
    mut step := Vector {} as Vector<int>
    step.push(0)
    until said.len() < 80 and cache.len < cache.ctx:
        next := sampler.sample(logits, logits.dim(0) - 1, said)
        tensor.Free(logits)
        if next == end_of_turn or next == tok.eos:
            break
        end
        said.push(next)
        step.set(0, next)
        print tok.decode(step)
        logits = llm.Forward(model, cache, step)
    end
    println ""
    println "tokens in prompt:", ids.len(), "answer:", said.len()
    cache.free()
    model.free()
    tok.free()
end

The sky appears blue because of the scattering of sunlight by tiny water droplets in the atmosphere, which causes them to bounce off each other and scatter into our eyes.
tokens in prompt: 30 answer: 33

A half billion weight model writes fluent sentences but gets facts wrong, as here: the real cause is scattering by air molecules, not water droplets. Use such a model for form, not for knowledge. Build with salam build --release for real use: the model runs on the processor, and a release build drops the bounds checks from the inner loops. Even so, expect a few seconds per token on an ordinary machine.

How text becomes tokens

A model never sees letters, only token ids. A BPE tokenizer starts from single characters and applies its list of merges in order, so frequent pieces such as lower become one token while rare words fall apart into many. Byte level BPE, the kind GPT 2, Llama 3 and Qwen use, writes a space as Ġ, so the space before a word is part of a token. This tokenizer is built by hand with five merges to make the steps visible.

import ai.llm

func show(tok &: llm.Tokenizer, text: str):
    ids := tok.encode(text)
    mut parts := ""
    each id in ids:
        parts = parts + "[" + tok.token_text(id) + "]"
    end
    println text, "->", ids.len(), "tokens", parts
end

func main:
    mut tok := llm.NewTokenizer(llm.TOK_BPE)
    each piece in ["l", "o", "w", "e", "r", "n", "s", "t", "Ġ", "lo", "low", "er", "lower"]:
        _ := tok.add_token(piece, 0.0)
    end
    tok.add_merge("l", "o")
    tok.add_merge("lo", "w")
    tok.add_merge("e", "r")
    tok.add_merge("low", "er")
    tok.finish()
    tok.add_bos = false
    show(tok, "low")
    show(tok, "lower")
    show(tok, "slower")
    show(tok, "newest")
    show(tok, "lower tower")
    println "round trip:", tok.decode(tok.encode("lower tower")) == "lower tower"
    tok.free()
end

tower is not merged into ow pieces because the merges only know lo and low. Counting tokens this way is also how you check that a prompt fits in the model's context before sending it.

Temperature and top-k

The model gives a score, a logit, for every token in the vocabulary, and the sampler turns the scores into a choice. Greedy always takes the highest. A Sampler draws at random in proportion to the probabilities: a temperature below one sharpens them toward the favourite, above one flattens them, and top_k drops all but the best k before drawing. The program draws 1000 times from four fixed scores with each setting and counts how often each token wins.

import ai.llm
import ai.tensor

func counts(sampler &: llm.Sampler, logits &: tensor.Tensor): str:
    mut seen := [0, 0, 0, 0]
    mut recent := Vector {} as Vector<int>
    repeat 1000:
        seen[sampler.sample(logits, 0, recent)] += 1
    end
    ret "" + seen[0] + " " + seen[1] + " " + seen[2] + " " + seen[3]
end

func main:
    mut data := Vector {} as Vector<f32>
    each v in [2.0, 1.5, 0.5, -1.0]:
        data.push(v as f32)
    end
    mut shape := tensor.Shape2(1, 4)
    mut logits := tensor.FromArray(shape, data)
    println "greedy always picks", llm.Greedy(logits, 0)
    mut plain := llm.NewSampler(1)
    println "temperature 1.0:", counts(plain, logits)
    mut cold := llm.NewSampler(1)
    cold.temperature = 0.3
    println "temperature 0.3:", counts(cold, logits)
    mut hot := llm.NewSampler(1)
    hot.temperature = 2.0
    println "temperature 2.0:", counts(hot, logits)
    mut top2 := llm.NewSampler(1)
    top2.top_k = 2
    println "top_k 2:        ", counts(top2, logits)
end

The seed passed to NewSampler makes the draws repeatable, which is why this output never changes. For factual answers use a low temperature; for varied writing, around 0.7 to 1.0 with top_p 0.9. repeat_penalty above 1 lowers the scores of recently used tokens and stops small models from looping.

Types

struct Config

import ai.llm
import ai.tensor

func main:
    mut tokenizer := llm.NewTokenizer(llm.TOK_BPE)
    each text in ["a", "b", "c", "ab"]:
        _ := tokenizer.add_token(text, 0.0)
    end
    tokenizer.add_merge("a", "b")
    tokenizer.finish()
    tokenizer.add_bos = false
    mut ids := tokenizer.encode("abcab")
    println tokenizer.size(), ids.len(), tokenizer.token_text(ids.get(0)), tokenizer.decode(ids)
    println llm.TOK_SPM, llm.ROPE_NORMAL, llm.ROPE_NEOX

    mut config := llm.Config { vocab = tokenizer.size(), dim = 8, layers = 2, heads = 2, kv_heads = 1, ff = 16, ctx = 32 }
    println config.head_dim()
    mut model := llm.RandomModel(config, 7)
    mut cache := llm.NewKVCache(config)
    println model.ok, model.layers.len(), cache.ctx

    mut logits := llm.Forward(model, cache, ids)
    println tensor.ShapeString(logits), cache.len
    mut last := llm.LogitsRow(logits, ids.len() - 1)
    greedy := llm.Greedy(logits, ids.len() - 1)
    println tensor.ShapeString(last), greedy == tensor.ArgMaxAll(last)

    mut sampler := llm.NewSampler(1)
    sampler.temperature = 0.8
    sampler.top_k = 3
    sampler.top_p = 0.9
    sampler.repeat_penalty = 1.1
    mut recent := Vector {} as Vector<int>
    next := sampler.sample(logits, ids.len() - 1, recent)
    println next >= 0 and next < tokenizer.size()

    cache.reset()
    generated := llm.GenerateIds(model, cache, ids, 4, sampler, -1)
    println generated.len()
    cache.reset()
    text := llm.Generate(model, cache, tokenizer, "ab", 3, sampler)
    println text.len() > 0
    tensor.Free(logits)
    tensor.Free(last)
    cache.free()
    model.free()
    tokenizer.free()
end

func head_dim(): int

struct Layer

func free()

struct Model

func free()

struct KVCache

func reset()

func free()

struct Tokenizer

func add_token(text: str, score: f64): int

func add_merge(a: str, b: str)

func finish()

func size(): int

func token_text(id: int): str

func encode(text: str): Vector<int>

func decode(ids &: Vector<int>): str

func free()

struct Sampler

func sample(logits &: tensor.Tensor, row: int, recent &: Vector<int>): int

Constants and variables

const ROPE_NORMAL: i32 = 0

const ROPE_NEOX: i32 = 1

const TOK_SPM: i32 = 0

const TOK_BPE: i32 = 1

Functions

func ConfigFromGguf(g &: gguf.Gguf): Config

import ai.llm
import ai.gguf

func main throws:
    path := "models/qwen2.5-0.5b-instruct-q8_0.gguf"
    mut model := llm.Model {}
    mut tokenizer := llm.NewTokenizer(llm.TOK_BPE)
    if not llm.LoadGgufModel(path, model, tokenizer):
        println "the model was not loaded:", path
        ret
    end
    mut cache := llm.NewKVCache(model.cfg)
    mut sampler := llm.NewSampler(42)
    sampler.temperature = 0.7
    println llm.Generate(model, cache, tokenizer, "The capital of France", 20, sampler)

    mut file := try gguf.LoadGguf(path)
    config := llm.ConfigFromGguf(file)
    println config.vocab, config.dim, config.layers, config.heads, config.kv_heads, config.ff, config.ctx, config.eps, config.rope_theta, config.rope_kind
    mut model2 := llm.ModelFromGguf(file)
    mut tokenizer2 := llm.TokenizerFromGguf(file)
    println tokenizer2.bos, tokenizer2.eos, tokenizer2.unk
    model2.free()
    tokenizer2.free()
    file.free()
    cache.free()
    model.free()
    tokenizer.free()
end

func ModelFromGguf(g &: gguf.Gguf): Model

func TokenizerFromGguf(g &: gguf.Gguf): Tokenizer

func LoadGgufModel(path: str, m &: Model, t &: Tokenizer): bool

func NewKVCache(cfg &: Config): KVCache

func RandomModel(cfg &: Config, seed: i64): Model

func NewTokenizer(kind: int): Tokenizer

func Forward(m &: Model, cache &: KVCache, tokens &: Vector<int>): tensor.Tensor

func LogitsRow(logits &: tensor.Tensor, row: int): tensor.Tensor

func NewSampler(seed: i64): Sampler

func Greedy(logits &: tensor.Tensor, row: int): int

func GenerateIds(m &: Model, cache &: KVCache, prompt_ids &: Vector<int>, max_new: int, sp &: Sampler, eos: int): Vector<int>

func Generate(m &: Model, cache &: KVCache, tok &: Tokenizer, prompt: str, max_new: int, sp &: Sampler): str