Standard library · AI

ai.gguf

Reading GGUF model files, the llama.cpp format.

import ai.gguf

On this page

What is in a model file
How much space a weight takes
Reading a weight
Bad files and missing keys
Types
Constants and variables
Functions

GGUF is the file format of llama.cpp for large language models. One file holds the metadata (the architecture, the sizes, the tokenizer vocabulary, the chat template) and every weight, usually quantized to 8 or 4 bits. This package reads such a file: LoadGguf parses the header and the tensor list, meta_int and meta_str read the metadata, and LoadTensorF32 turns one weight into a tensor of 32 bit floats when you need it.

It is a file reader, not an inference engine. To generate text from a model, use llm, which builds on it. Models exported from PyTorch as ONNX go through onnx instead.

Limits to know before you download a model

LoadGguf reads the whole file into memory first, and a buffer holds at most 2 GB. Pick files under 2 GB: TinyLlama 1.1B, Qwen 0.5B or 1.5B and other small models.
Only five weight types can be read: F32, F16, Q8_0, Q4_0 and Q4_1. The K-quants that most downloads use, such as Q4_K_M or Q5_K_S, and BF16 are not supported: TensorBytes returns -1 for them and LoadTensorF32 throws gguf.tensor_type. Download the Q8_0 or Q4_0 file of a model.
Dimensions are stored fastest first, the order llama.cpp uses. LoadTensorF32 reverses them, so a weight listed as 4096 x 32000 becomes a tensor of 32000 rows of 4096.
ParseGguf takes ownership of the buffer you give it; the model's free releases it. When it throws, the buffer stays yours.

The examples build small files in memory, so they run without a download.

What is in a model file

Everything llama.cpp needs to know about a model is in the metadata, under keys named after the architecture: llama.block_count is the number of layers, llama.context_length the longest prompt it was trained for. Read general.architecture first and build the other keys from it, so the same tool works for Qwen, Gemma or Phi files. meta_int and meta_str take a default or return an empty string for a missing key, so a key a file lacks never stops the program.

Download a small quantized model, for example a TinyLlama or Qwen Q4_0 or Q8_0 file from Hugging Face, and run:

salam build gguf-guide-inspect.salam
./gguf-guide-inspect tinyllama-1.1b-chat.Q8_0.gguf

gguf-guide-inspect.salam

import ai.gguf
import os
import str

func main:
    args := os.Args()
    if args.len() < 2:
        println "usage: gguf-guide-inspect model.gguf"
        ret 1
    end
    mut g := gguf.Gguf {}
    try:
        g = gguf.LoadGguf(args.get(1))
    catch e:
        println "not a GGUF model:", e.to_str()
        ret 1
    end
    arch := g.meta_str("general.architecture")
    println "name", g.meta_str("general.name")
    println "architecture", arch
    println "layers", g.meta_int(arch + ".block_count", 0)
    println "context", g.meta_int(arch + ".context_length", 0)
    println "embedding", g.meta_int(arch + ".embedding_length", 0)
    vocab := g.meta_index("tokenizer.ggml.tokens")
    if vocab >= 0:
        println "vocabulary", g.values.get(vocab).strs.len()
    end
    mut weights := 0 as i64
    mut bytes := 0 as i64
    mut kinds := HashMap {} as HashMap<int, int>
    repeat g.tensors.len() in i:
        info := g.tensors.get(i)
        weights += info.count() as i64
        bytes += gguf.TensorBytes(g, i) as i64
        if kinds.has(info.ggml_type):
            kinds.put(info.ggml_type, kinds.get(info.ggml_type) + 1)
        else:
            kinds.put(info.ggml_type, 1)
        end
    end
    println g.tensors.len(), "tensors,", weights / 1000000, "million weights,", bytes / 1048576, "MiB"
    println "tensors stored as Q8_0:", kinds.has(gguf.GGML_Q8_0), "Q4_0:", kinds.has(gguf.GGML_Q4_0), "F16:", kinds.has(gguf.GGML_F16)
    println "uses a chat template:", str.Contains(g.meta_str("tokenizer.chat_template"), "{%")
    g.free()
end

No weight is converted here, but LoadGguf still reads the whole file into memory, so just opening a 1 GB model takes 1 GB of memory. The vocabulary is a string array in tokenizer.ggml.tokens, and tokenizer.chat_template holds the Jinja template that turns a conversation into the model's prompt format.

How much space a weight takes

Quantization stores weights in fewer bits. Q8_0 keeps blocks of 32 weights as 8 bit integers plus one 16 bit scale, which is 34 bytes per block, or 8.5 bits per weight; Q4_0 packs the same block into 18 bytes, 4.5 bits per weight. TensorBytes gives the stored size of any tensor. The program below writes a file with one 64 weight tensor of each readable kind except Q4_1, which keeps a minimum next to the scale and takes 20 bytes per block.

gguf-guide-sizes.salam

import ai.gguf
import compress
import encoding.binary
import conv

func name(b &: compress.ByteBuf, text: str):
    binary.WriteU64LE(b, text.len() as i64)
    compress.BufPushBytes(b, text as void*, text.len())
end

func tensor_header(b &: compress.ByteBuf, label: str, kind: int, offset: int):
    name(b, label)
    binary.WriteU32LE(b, 1)
    binary.WriteU64LE(b, 64)
    binary.WriteU32LE(b, kind)
    binary.WriteU64LE(b, offset as i64)
end

func main throws:
    mut b := compress.NewByteBuf()
    compress.BufPushBytes(b, "GGUF" as void*, 4)
    binary.WriteU32LE(b, 3)
    binary.WriteU64LE(b, 4)
    binary.WriteU64LE(b, 0)
    tensor_header(b, "full", gguf.GGML_F32, 0)
    tensor_header(b, "half", gguf.GGML_F16, 256)
    tensor_header(b, "q8", gguf.GGML_Q8_0, 384)
    tensor_header(b, "q4", gguf.GGML_Q4_0, 480)
    until compress.BufLen(b) % 32 != 0:
        binary.WriteU8(b, 0)
    end
    repeat 544:
        binary.WriteU8(b, 0)
    end
    mut g := try gguf.ParseGguf(b)
    println "parsed:", g.tensors.len(), "tensors"
    repeat g.tensors.len() in i:
        info := g.tensors.get(i)
        bytes := gguf.TensorBytes(g, i)
        bits := (bytes * 8) as f64 / (info.count() as f64)
        println info.name, info.count(), "weights in", bytes, "bytes,", conv.FormatFloatPrec(bits, 1), "bits each"
    end
    g.free()
end

Outputparsed: 4 tensors full 64 weights in 256 bytes, 32.0 bits each half 64 weights in 128 bytes, 16.0 bits each q8 64 weights in 68 bytes, 8.5 bits each q4 64 weights in 36 bytes, 4.5 bits each

These numbers predict the memory a model needs: a seven billion weight model takes about 28 GB in F32, 14 GB in F16, 7.4 GB in Q8_0 and 3.9 GB in Q4_0, before the cache for the context. LoadTensorF32 expands any of these kinds back to 32 bit floats when you need the values. Those sizes are for the file; this package needs the whole file in memory and, for every tensor you load, four bytes per weight on top.

Reading a weight

tensor_index finds a tensor by name and LoadTensorF32 expands it. The program writes a Q8_0 token embedding table for two tokens of 32 values each: every block is one 16 bit scale (about 0.1 here) followed by 32 signed bytes, and each weight is the scale times its byte. Row 1 of the table is the embedding of token 1.

gguf-guide-weights.salam

import ai.gguf
import compress
import encoding.binary
import conv

func write_name(b &: compress.ByteBuf, text: str):
    binary.WriteU64LE(b, text.len() as i64)
    compress.BufPushBytes(b, text as void*, text.len())
end

func main throws:
    mut b := compress.NewByteBuf()
    compress.BufPushBytes(b, "GGUF" as void*, 4)
    binary.WriteU32LE(b, 3)
    binary.WriteU64LE(b, 1)
    binary.WriteU64LE(b, 0)
    write_name(b, "token_embd.weight")
    binary.WriteU32LE(b, 2)
    binary.WriteU64LE(b, 32)
    binary.WriteU64LE(b, 2)
    binary.WriteU32LE(b, gguf.GGML_Q8_0)
    binary.WriteU64LE(b, 0)
    until compress.BufLen(b) % 32 != 0:
        binary.WriteU8(b, 0)
    end
    repeat 2 in row:
        binary.WriteU16LE(b, 0x2E66)
        repeat 32 in j:
            q := (row * 32 + j) * 3 - 60
            binary.WriteU8(b, q & 255)
        end
    end
    mut model := try gguf.ParseGguf(b)
    index := model.tensor_index("token_embd.weight")
    info := model.tensors.get(index)
    println info.name, "dims", info.dims.get(0), "x", info.dims.get(1), "stored in", gguf.TensorBytes(model, index), "bytes"
    mut weights := try gguf.LoadTensorF32(model, index)
    println "loaded: shape", weights.shape.get(0), "x", weights.shape.get(1)
    mut row := ""
    repeat 4 in j:
        row = row + " " + conv.FormatFloatPrec(weights.at2(1, j), 3)
    end
    println "embedding of token 1 starts with" + row
    weights.free()
    model.free()
end

Outputtoken_embd.weight dims 32 x 2 stored in 68 bytes loaded: shape 2 x 32 embedding of token 1 starts with 3.599 3.899 4.199 4.499

The file lists the dimensions as 32 x 2, fastest first, and the loaded tensor has the shape 2 x 32, one row per token, so at2(token, j) reads one value. The values come out as 3.599 rather than 3.6 because the scale itself is stored as a 16 bit float: quantization error is part of the format. A real model's table has one row per vocabulary entry; look the token text up in the tokenizer.ggml.tokens array to find its row.

Bad files and missing keys

ParseGguf, LoadGguf and LoadTensorF32 are throws. A file that is not GGUF is gguf.bad_magic, a file cut off in the header is UnexpectedEnd with the byte offset (and the metadata key or tensor name in path when it happened inside one), and tensor data past the end of the file is gguf.truncated. A model with nothing in it is still valid; looking up a key or a tensor it does not have gives an empty string, your default, or -1 for an index. Check the index before you use it with values or tensors.

gguf-guide-errors.salam

import ai.gguf
import compress
import encoding.binary

func main throws:
    mut junk := compress.NewByteBuf()
    text := "PK this is a zip archive, not a model file"
    compress.BufPushBytes(junk, text as void*, text.len())
    try:
        gguf.ParseGguf(junk)
    catch e:
        println "zip file:", e.kind_name(), "|", e.to_str(), "|", e.code
    end
    compress.BufFree(junk)

    mut cut := compress.NewByteBuf()
    compress.BufPushBytes(cut, "GGUF" as void*, 4)
    binary.WriteU32LE(cut, 3)
    binary.WriteU64LE(cut, 1)
    try:
        gguf.ParseGguf(cut)
    catch e:
        println "cut short:", e.to_str()
    end
    compress.BufFree(cut)

    mut empty := compress.NewByteBuf()
    compress.BufPushBytes(empty, "GGUF" as void*, 4)
    binary.WriteU32LE(empty, 3)
    binary.WriteU64LE(empty, 0)
    binary.WriteU64LE(empty, 0)
    mut g := try gguf.ParseGguf(empty)
    println "empty model:", g.version, g.keys.len(), g.tensors.len()
    println "architecture: [" + g.meta_str("general.architecture") + "]"
    println "context length:", g.meta_int("llama.context_length", 2048)
    println "tensor index:", g.tensor_index("token_embd.weight"), "meta index:", g.meta_index("tokenizer.tokens")
    g.free()
end

Outputzip file: syntax error | gguf: not a GGUF file (bad magic bytes) (offset 0) | gguf.bad_magic cut short: gguf: unexpected end of data: needed 8 bytes, 0 left (offset 16) empty model: 3 0 0 architecture: [] context length: 2048 tensor index: -1 meta index: -1

ParseGguf takes ownership of the buffer you give it, and the model's free releases it. Do not also call BufFree on it: freeing it twice crashes the program. When ParseGguf throws, the buffer is still yours to free, as the program does with junk.

Types

`MetaValue`

struct MetaValue

One metadata value.

Name	Type	Description
`kind`	`int = 0`	The type.
`elem_kind`	`int = 0`	The type of the elements, for an array.
`i`	`i64 = 0`	The integer value.
`f`	`f64 = 0`	The float value.
`s`	`str = ""`	The string.
`ints`	`Vector`	The integer array.
`floats`	`Vector`	The float array.
`strs`	`Vector`	The string array, such as the tokenizer vocabulary.

ai-gguf-Read.salam

import ai.gguf
import compress
import encoding.binary
import archive
import os

func string(b &: compress.ByteBuf, text: str):
    binary.WriteU64LE(b, text.len() as i64)
    compress.BufPushBytes(b, text as void*, text.len())
end

func build(b &: compress.ByteBuf):
    compress.BufPushBytes(b, "GGUF" as void*, 4)
    binary.WriteU32LE(b, 3)
    binary.WriteU64LE(b, 1)
    binary.WriteU64LE(b, 3)
    string(b, "general.architecture")
    binary.WriteU32LE(b, gguf.META_STRING)
    string(b, "llama")
    string(b, "llama.context_length")
    binary.WriteU32LE(b, 4)
    binary.WriteU32LE(b, 2048)
    string(b, "tokenizer.tokens")
    binary.WriteU32LE(b, gguf.META_ARRAY)
    binary.WriteU32LE(b, gguf.META_STRING)
    binary.WriteU64LE(b, 2)
    string(b, "<s>")
    string(b, "hello")
    string(b, "w")
    binary.WriteU32LE(b, 2)
    binary.WriteU64LE(b, 3)
    binary.WriteU64LE(b, 2)
    binary.WriteU32LE(b, gguf.GGML_F32)
    binary.WriteU64LE(b, 0)
    until compress.BufLen(b) % 32 != 0:
        binary.WriteU8(b, 0)
    end
    repeat 6 in i:
        binary.WriteF32(b, (i as f64) * 0.5, false)
    end
end

func main throws:
    mut buffer := compress.NewByteBuf()
    build(buffer)
    path := os.TempDir() + "/salam_tiny_en.gguf"
    _ := archive.WriteBinaryFileBuf(path, buffer)
    mut in_memory := try gguf.ParseGguf(buffer)
    println in_memory.tensors.len()
    in_memory.free()

    mut g := try gguf.LoadGguf(path)
    println g.version, g.keys.len(), g.tensors.len(), g.alignment, g.data_offset
    println g.meta_str("general.architecture"), g.meta_int("llama.context_length", 0), g.meta_int("llama.block_count", 32)
    tokens := g.values.get(g.meta_index("tokenizer.tokens"))
    println tokens.kind == gguf.META_ARRAY, tokens.elem_kind == gguf.META_STRING, tokens.strs.get(1)
    idx := g.tensor_index("w")
    info := g.tensors.get(idx)
    println info.name, info.dims.len(), info.count(), info.ggml_type, info.offset, gguf.TensorBytes(g, idx)
    mut t := try gguf.LoadTensorF32(g, idx)
    println t.at(0), t.at(5)
    try:
        gguf.LoadTensorF32(g, 7)
    catch e:
        println e.code
    end
    t.free()
    println gguf.GGML_F16, gguf.GGML_Q4_0, gguf.GGML_Q4_1, gguf.GGML_Q8_0
    g.free()
end

Output1 3 3 1 32 224 llama 2048 32 true true hello w 2 6 0 0 24 0 2.5 gguf.index 1 2 3 8