encoding/serde

This package converts text from one data format to another in one call. Transcode reads the input with the matching decoder, holds it as a Value, and writes it with the target encoder. Detect guesses which format a text is in. The format names are the lowercase strings json, yaml, toml, xml and csv; "JSON" is not recognised.

It suits tools that take whatever format a user hands them, and one-off migrations. When your program needs the data itself, call the format's own package (encoding/json, encoding/yaml, encoding/toml, encoding/xml, encoding/csv): they return a Value or fill a struct and give real error messages, while this package only answers yes or no. MessagePack is binary and is not one of the names here; use encoding/msgpack.

What survives a conversion

Everything passes through a Value, so comments and formatting are always lost, while keys keep the order they were read in. Beyond that, each format can hold less than JSON:

What trips people up

• Detection is a guess. The order is XML (text starts with <), JSON, TOML, CSV, then YAML, and nearly any plain text such as hello world is valid YAML. Pass the format yourself whenever a file extension or Content-Type tells you.
• Transcode hides failures as an empty string. Use TranscodeStrict and check ok.
• CSV in never fails, so ok is true for any text read as CSV.

Guide: accept any format

A tool that takes settings or data from users does not have to insist on one format. Detect looks at the text and names the format: XML when it starts with <, JSON when it parses as JSON, then TOML, CSV and finally YAML. Pass that name to TranscodeStrict to turn whatever came in into the one format your code reads.

import encoding.serde

func to_json(text: str): str:
    kind := serde.Detect(text)
    if kind == "":
        ret "(unknown format)"
    end
    mut ok := false
    out := serde.TranscodeStrict(text, kind, "json", ok)
    if not ok:
        ret "(could not read " + kind + ")"
    end
    ret kind + " -> " + out
end

func main:
    println to_json("name: shop\nport: 8080\n")
    println to_json("name = \"shop\"\nport = 8080\n")
    println to_json("name,port\nshop,8080\n")
    println to_json(`{"name": "shop", "port": 8080}`)
    println to_json("")
end

Detection is a guess from the shape of the text. Very short inputs can fit more than one format, and almost any plain text is valid YAML, so when you know the format, from a file extension or a Content-Type header, pass it yourself.

Guide: move a config to another format

Changing a project from YAML to TOML, or the other way, is one call. Convert, then check the result: read both the old and the new text back as JSON, decode them, and compare the two values with value.Equal. Comparing the JSON text instead would fail on key order alone.

import encoding.serde
import encoding.json
import encoding.value
import encoding.encerr

func main:
    old := `server:
  host: 0.0.0.0
  port: 8080
log:
  level: info
`
    mut ok := false
    as_toml := serde.TranscodeStrict(old, "yaml", "toml", ok)
    println "converted:", ok
    print as_toml

    mut e := encerr.NoError()
    before := json.Decode(serde.Transcode(old, "yaml", "json"), e)
    after := json.Decode(serde.Transcode(as_toml, "toml", "json"), e)
    println "same data:", value.Equal(before, after)
end

Comments do not survive, because a Value holds data only, so tell people who edit the file by hand before you convert it.

Guide: XML in and out

XML has no arrays or numbers, so the mapping follows a few rules: an @name key becomes an attribute, an array becomes the same element repeated, #text becomes the element's text, and a converted document is wrapped in a <root> element. Reading XML gives every value as a string and attributes as @ keys.

import encoding.serde

func main:
    order := `{"@id": 1042, "customer": "Sara", "item": ["tea", "cups"], "total": 39.5, "paid": true}`
    x := serde.Transcode(order, "json", "xml")
    println x
    println "detected:", serde.Detect(x)
    println "back:", serde.Transcode(x, "xml", "json")
    println serde.Transcode("<book lang=\"fa\"><title>Salam</title><pages>153</pages></book>", "xml", "json")
end

Numbers and booleans are written as their text, so 39.5 and true arrive in the XML. The way back shows what XML cannot express: every value returns as a string, and the outer <root> or <book> is gone, leaving only its contents. When the element names, order or namespaces matter, build the document with encoding/xml directly.

Guide: handle failure

TranscodeStrict says through ok whether both halves worked: reading in the source format, and writing in the target. It fails on broken input, on a format name it does not know, and on data the target cannot hold, such as a top level array for TOML. Transcode just returns an empty string in all those cases.

import encoding.serde

func attempt(label: str, data: str, from: str, dest: str):
    mut ok := false
    out := serde.TranscodeStrict(data, from, dest, ok)
    if ok:
        println label, "ok:", out
    else:
        println label, "failed"
    end
end

func main:
    attempt("good yaml   ", "- 1\n- 2\n", "yaml", "json")
    attempt("broken json ", `{"a": 1`, "json", "yaml")
    attempt("unknown from", `a = 1`, "ini", "json")
    attempt("unknown dest", `{"a": 1}`, "json", "ini")
    attempt("toml needs a table", `[1, 2]`, "json", "toml")
    println "lenient gives [" + serde.Transcode(`{"a": 1`, "json", "yaml") + "]"
end

Constants and variables

const FormatUnknown: i32 = 0

import encoding.serde

func main:
    data := `{"name": "salam", "port": 8080, "tags": ["a", "b"]}`
    println serde.Detect(data), serde.Detect("a: 1\nb: 2\n"), serde.Detect("<a/>"), serde.Detect("[x]\ny = 1\n"), serde.Detect("a,b\n1,2\n")
    println serde.DetectFormat(data) == serde.FormatJSON, serde.FormatUnknown, serde.FormatYAML, serde.FormatTOML, serde.FormatXML, serde.FormatCSV
    println serde.Transcode(data, "json", "yaml")
    println serde.Transcode(data, "json", "toml")
    println serde.Transcode("name,age\nAli,30\n", "csv", "json")
    mut ok := false
    _ := serde.TranscodeStrict(data, "json", "pdf", ok)
    println ok
end

const FormatJSON: i32 = 1

const FormatYAML: i32 = 2

const FormatTOML: i32 = 3

const FormatXML: i32 = 4

const FormatCSV: i32 = 5

Functions

func DetectFormat(text: str): int

func Detect(text: str): str

func TranscodeStrict(data: str, from: str, dest: str, ok &: bool): str

func Transcode(data: str, from: str, dest: str): str