Standard library · Machine learning

gguf

Reading GGUF model files, the llama.cpp format.

import gguf

On this page

Types
Constants and variables
Functions

GGUF is the llama.cpp file format for large language models. One file holds the metadata, such as the architecture and the tokenizer vocabulary, and the weights, usually quantised. LoadGguf reads only the header, and LoadTensorF32 turns each weight into a tensor whenever you need it.

The example below builds a small file so that it runs without downloading a model.

Types

`MetaValue`

struct MetaValue

One metadata value.

Name	Type	Description
`kind`	`int = 0`	The type.
`elem_kind`	`int = 0`	The type of the elements, for an array.
`i`	`i64 = 0`	The integer value.
`f`	`f64 = 0`	The float value.
`s`	`str = ""`	The string.
`ints`	`Vector`	The integer array.
`floats`	`Vector`	The float array.
`strs`	`Vector`	The string array, such as the tokenizer vocabulary.

gguf-Read.salam

import gguf
import compress
import encoding.binary
import archive
import os

func string(b &: compress.ByteBuf, text: str):
    binary.WriteU64LE(b, text.len() as i64)
    compress.BufPushBytes(b, text as void*, text.len())
end

func build(b &: compress.ByteBuf):
    compress.BufPushBytes(b, "GGUF" as void*, 4)
    binary.WriteU32LE(b, 3)
    binary.WriteU64LE(b, 1)
    binary.WriteU64LE(b, 3)
    string(b, "general.architecture")
    binary.WriteU32LE(b, gguf.META_STRING)
    string(b, "llama")
    string(b, "llama.context_length")
    binary.WriteU32LE(b, 4)
    binary.WriteU32LE(b, 2048)
    string(b, "tokenizer.tokens")
    binary.WriteU32LE(b, gguf.META_ARRAY)
    binary.WriteU32LE(b, gguf.META_STRING)
    binary.WriteU64LE(b, 2)
    string(b, "<s>")
    string(b, "hello")
    string(b, "w")
    binary.WriteU32LE(b, 2)
    binary.WriteU64LE(b, 3)
    binary.WriteU64LE(b, 2)
    binary.WriteU32LE(b, gguf.GGML_F32)
    binary.WriteU64LE(b, 0)
    until compress.BufLen(b) % 32 != 0:
        binary.WriteU8(b, 0)
    end
    repeat 6 in i:
        binary.WriteF32(b, (i as f64) * 0.5, false)
    end
end

func main:
    mut buffer := compress.NewByteBuf()
    build(buffer)
    path := os.TempDir() + "/salam_tiny_en.gguf"
    _ := archive.WriteBinaryFileBuf(path, buffer)
    mut good := false
    mut in_memory := gguf.ParseGguf(buffer, good)
    println good, in_memory.error == ""
    in_memory.free()

    mut g := gguf.LoadGguf(path, good)
    println good, g.ok, g.version, g.keys.len(), g.tensors.len(), g.alignment, g.data_offset
    println g.meta_str("general.architecture"), g.meta_int("llama.context_length", 0), g.meta_int("llama.block_count", 32)
    tokens := g.values.get(g.meta_index("tokenizer.tokens"))
    println tokens.kind == gguf.META_ARRAY, tokens.elem_kind == gguf.META_STRING, tokens.strs.get(1)
    idx := g.tensor_index("w")
    info := g.tensors.get(idx)
    println info.name, info.dims.len(), info.count(), info.ggml_type, info.offset, gguf.TensorBytes(g, idx)
    mut t := gguf.LoadTensorF32(g, idx, good)
    println good, t.at(0), t.at(5)
    t.free()
    println gguf.GGML_F16, gguf.GGML_Q4_0, gguf.GGML_Q4_1, gguf.GGML_Q8_0
    g.free()
end

Outputtrue true true true 3 3 1 32 224 llama 2048 32 true true hello w 2 6 0 0 24 true 0 2.5 1 2 3 8