From 2171b71e920d286a9527a4dddd05c00eceb6af83 Mon Sep 17 00:00:00 2001
From: Rasmus Dahlberg <rasmus@mullvad.net>
Date: Mon, 20 Dec 2021 14:37:31 +0100
Subject: ascii: Add sigsum ASCII-parser and tests
---
pkg/ascii/ascii.go | 305 ++++++++++++++++++++++++++++++++
pkg/ascii/ascii_test.go | 455 ++++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 760 insertions(+)
create mode 100644 pkg/ascii/ascii.go
create mode 100644 pkg/ascii/ascii_test.go
(limited to 'pkg')
diff --git a/pkg/ascii/ascii.go b/pkg/ascii/ascii.go
new file mode 100644
index 0000000..350890c
--- /dev/null
+++ b/pkg/ascii/ascii.go
@@ -0,0 +1,305 @@
+// package ascii implements an ASCII key-value parser.
+//
+// The top-most (de)serialize must operate on a struct pointer. A struct may
+// contain other structs, in which case all tag names should be unique. Public
+// fields without tag names are ignored. Private fields are also ignored.
+//
+// The supported field types are:
+// - struct
+// - string (no empty strings)
+// - uint64 (only digits in ASCII representation)
+// - byte array (only lower-case hex in ASCII representation)
+// - slice of uint64 (no empty slices)
+// - slice of byte array (no empty slices)
+//
+// A key must not contain an encoding's end-of-key value.
+// A value must not contain an encoding's end-of-value value.
+//
+// For additional details, please refer to the Sigsum v0 API documentation.
+//
+package ascii
+
+import (
+ "bytes"
+ "fmt"
+ "io"
+ "reflect"
+ "strconv"
+ "strings"
+
+ "git.sigsum.org/sigsum-lib-go/pkg/hex"
+)
+
+var StdEncoding = NewEncoding("ascii", "=", "\n")
+
+type Encoding struct {
+ identifier string
+ endOfKey string
+ endOfValue string
+}
+
+func NewEncoding(id, eok, eov string) *Encoding {
+ return &Encoding{
+ identifier: id,
+ endOfKey: eok,
+ endOfValue: eov,
+ }
+}
+
+type someValue struct {
+ v reflect.Value
+ ok bool
+}
+
+// Serialize tries to serialize an interface as ASCII key-value pairs
+func (e *Encoding) Serialize(w io.Writer, i interface{}) error {
+ v, err := dereferenceStructPointer(i)
+ if err != nil {
+ return err
+ }
+
+ t := v.Type()
+ for i := 0; i < t.NumField(); i++ {
+ switch v.Field(i).Type().Kind() {
+ case reflect.Struct:
+ if err := e.Serialize(w, v.Field(i).Addr().Interface()); err != nil {
+ return err
+ }
+ default:
+ if t.Field(i).PkgPath != "" {
+ continue // skip private field
+ }
+ key, ok := t.Field(i).Tag.Lookup(e.identifier)
+ if !ok {
+ continue // skip public field without tag
+ }
+
+ if strings.Contains(key, e.endOfKey) {
+ return fmt.Errorf("ascii: key %q contains end-of-key character", key)
+ }
+ if err := e.write(w, key, v.Field(i)); err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+}
+
+func (e *Encoding) write(w io.Writer, key string, v reflect.Value) error {
+ t := v.Type()
+ switch t {
+ case reflect.TypeOf(uint64(0)):
+ val := fmt.Sprintf("%d", v.Uint())
+ return e.writeOne(w, key, val)
+ }
+
+ k := t.Kind()
+ switch k {
+ case reflect.Array:
+ if kind := t.Elem().Kind(); kind != reflect.Uint8 {
+ return fmt.Errorf("ascii: array kind not supported: %v", kind)
+ }
+
+ arrayLen := v.Len()
+ array := make([]byte, arrayLen, arrayLen)
+ for i := 0; i < arrayLen; i++ {
+ array[i] = uint8(v.Index(i).Uint())
+ }
+
+ val := hex.Serialize(array)
+ return e.writeOne(w, key, val)
+
+ case reflect.Slice:
+ kind := t.Elem().Kind()
+ if kind != reflect.Array && kind != reflect.Uint64 {
+ return fmt.Errorf("ascii: slice kind not supported: %v", kind)
+ }
+ if v.Len() == 0 {
+ return fmt.Errorf("ascii: slice must not be empty")
+ }
+
+ var err error
+ for i := 0; i < v.Len(); i++ {
+ err = e.write(w, key, v.Index(i))
+ }
+ return err
+
+ case reflect.String:
+ if v.Len() == 0 {
+ return fmt.Errorf("ascii: string must not be empty")
+ }
+ return e.writeOne(w, key, v.String())
+ }
+
+ return fmt.Errorf("ascii: unsupported type %v and kind %v", t, k)
+}
+
+func (e *Encoding) writeOne(w io.Writer, key, value string) error {
+ _, err := w.Write([]byte(key + e.endOfKey + value + e.endOfValue))
+ return err
+}
+
+// Deserialize tries to deserialize a buffer of ASCII key-value pairs
+func (e *Encoding) Deserialize(r io.Reader, i interface{}) error {
+ m := make(map[string]*someValue)
+ if err := e.mapKeys(i, m); err != nil {
+ return err
+ }
+
+ buf, err := io.ReadAll(r)
+ if err != nil {
+ return fmt.Errorf("ascii: failed reading incoming buffer")
+ }
+
+ // trim end of buffer so that loop does not run on an empty line
+ if len(buf) <= len(e.endOfValue) {
+ return fmt.Errorf("ascii: buffer contains no key-value pair")
+ }
+ offset := len(buf) - len(e.endOfValue)
+ if !bytes.Equal(buf[offset:], []byte(e.endOfValue)) {
+ return fmt.Errorf("ascii: buffer must end with endOfValue")
+ }
+ buf = buf[:offset]
+
+ for _, kv := range bytes.Split(buf, []byte(e.endOfValue)) {
+ split := bytes.Split(kv, []byte(e.endOfKey))
+ if len(split) == 1 {
+ return fmt.Errorf("ascii: missing key-value pair in %q", string(kv))
+ }
+
+ key := string(split[0])
+ value := string(bytes.Join(split[1:], nil))
+ ref, ok := m[key]
+ if !ok {
+ return fmt.Errorf("ascii: unexpected key %q", key)
+ }
+ if len(value) == 0 {
+ fmt.Errorf("ascii: missing value for key %q", key)
+ }
+ if err := setKey(ref, key, value); err != nil {
+ return err
+ }
+ }
+ return requireValues(m)
+}
+
+func (e *Encoding) mapKeys(i interface{}, m map[string]*someValue) error {
+ v, err := dereferenceStructPointer(i)
+ if err != nil {
+ return err
+ }
+
+ t := v.Type()
+ for i := 0; i < t.NumField(); i++ {
+ switch v.Field(i).Type().Kind() {
+ case reflect.Struct:
+ i := v.Field(i).Addr().Interface()
+ e.mapKeys(i, m) // return is always nil
+ default:
+ if t.Field(i).PkgPath != "" {
+ continue // skip private field
+ }
+ key, ok := t.Field(i).Tag.Lookup(e.identifier)
+ if !ok {
+ continue // skip public field without tag
+ }
+ m[key] = &someValue{
+ v: v.Field(i),
+ }
+ }
+ }
+ return nil
+}
+
+func setKey(ref *someValue, key, value string) error {
+ v := ref.v
+ if v.Kind() == reflect.Ptr && !v.IsNil() {
+ v = v.Elem()
+ }
+
+ t := v.Type()
+ switch t {
+ case reflect.TypeOf(uint64(0)):
+ num, err := strconv.ParseUint(value, 10, 64)
+ if err != nil {
+ return err
+ }
+
+ ref.ok = true
+ v.SetUint(num)
+ return nil
+ }
+
+ k := t.Kind()
+ switch k {
+ case reflect.Array:
+ arrayLen := v.Len()
+ b, err := hex.Deserialize(value)
+ if err != nil {
+ return err
+ }
+ if len(b) != arrayLen {
+ return fmt.Errorf("ascii: invalid array size for key %q", key)
+ }
+
+ ref.ok = true
+ reflect.Copy(v, reflect.ValueOf(b))
+ return nil
+
+ case reflect.Slice:
+ sliceType := t
+ kind := sliceType.Elem().Kind()
+ if kind != reflect.Array && kind != reflect.Uint64 {
+ return fmt.Errorf("ascii: slice kind not supported: %v", kind)
+ }
+
+ if v.IsNil() {
+ v.Set(reflect.MakeSlice(sliceType, 0, 0))
+ }
+ sv := &someValue{
+ v: reflect.New(sliceType.Elem()),
+ }
+ if err := setKey(sv, key, value); err != nil {
+ return err
+ }
+
+ ref.ok = true
+ v.Set(reflect.Append(v, sv.v.Elem()))
+ return nil
+
+ case reflect.String:
+ if len(value) == 0 {
+ return fmt.Errorf("ascii: string must not be empty")
+ }
+
+ ref.ok = true
+ v.SetString(value)
+ return nil
+ }
+
+ return fmt.Errorf("ascii: unsupported type %v and kind %v", t, k)
+}
+
+func requireValues(m map[string]*someValue) error {
+ for k, v := range m {
+ if !v.ok {
+ return fmt.Errorf("ascii: missing value for key %q", k)
+ }
+ }
+ return nil
+}
+
+func dereferenceStructPointer(i interface{}) (*reflect.Value, error) {
+ v := reflect.ValueOf(i)
+ if v.Kind() != reflect.Ptr {
+ return nil, fmt.Errorf("ascii: interface value must be pointer")
+ }
+ if v.IsNil() {
+ return nil, fmt.Errorf("ascii: interface value must be non-nil pointer")
+ }
+ v = v.Elem()
+ if v.Type().Kind() != reflect.Struct {
+ return nil, fmt.Errorf("ascii: interface value must point to struct")
+ }
+ return &v, nil
+}
diff --git a/pkg/ascii/ascii_test.go b/pkg/ascii/ascii_test.go
new file mode 100644
index 0000000..8aee70c
--- /dev/null
+++ b/pkg/ascii/ascii_test.go
@@ -0,0 +1,455 @@
+package ascii
+
+import (
+ "bytes"
+ "reflect"
+ "testing"
+)
+
+type testStruct struct {
+ Num uint64 `ascii/test:"num"`
+ Struct testStructOther
+ Skip uint64
+ skip uint64
+}
+
+type testStructOther struct {
+ Array testArray `ascii/test:"array"`
+ Slice []testArray `ascii/test:"slice"`
+ String string `ascii/test:"string"`
+}
+
+type testArray [2]byte
+
+type testStructUnsupportedType struct {
+ ByteSlice []byte `ascii/test:"byte_slice"`
+}
+
+func TestSerialize(t *testing.T) {
+ e := NewEncoding("ascii/test", "<--", ";;")
+ for _, table := range []struct {
+ desc string
+ want string
+ err bool
+ i interface{}
+ }{
+ {
+ desc: "invalid: not pointer to struct",
+ err: true,
+ i: testStruct{},
+ },
+ {
+ desc: "invalid: struct with invalid key",
+ err: true,
+ i: &struct {
+ Num uint64 `ascii/test:"num<--nom"`
+ }{
+ Num: 1,
+ },
+ },
+ {
+ desc: "invalid: struct with invalid type",
+ err: true,
+ i: &testStructUnsupportedType{
+ ByteSlice: []byte("hellow"),
+ },
+ },
+ {
+ desc: "invalid: struct with invalid type and kind",
+ err: true,
+ i: &struct {
+ Struct testStructUnsupportedType
+ }{
+ Struct: testStructUnsupportedType{
+ ByteSlice: []byte("hellow"),
+ },
+ },
+ },
+ {
+ desc: "valid",
+ want: "num<--1;;array<--01fe;;slice<--01fe;;slice<--00ff;;string<--hellow;;",
+ i: &testStruct{
+ Num: 1,
+ Struct: testStructOther{
+ Array: testArray{1, 254},
+ Slice: []testArray{
+ testArray{1, 254},
+ testArray{0, 255},
+ },
+ String: "hellow",
+ },
+ },
+ },
+ } {
+ b := bytes.NewBuffer(nil)
+ err := e.Serialize(b, table.i)
+ if got, want := err != nil, table.err; got != want {
+ t.Errorf("got error %v but wanted %v in test %q: %v", got, want, table.desc, err)
+ }
+ if err != nil {
+ continue
+ }
+ if got, want := string(b.Bytes()), table.want; got != want {
+ t.Errorf("got buf %s but wanted %s in test %q", got, want, table.desc)
+ }
+ }
+}
+
+func TestWrite(t *testing.T) {
+ e := NewEncoding("ascii/test", "<--", ";;")
+ for _, table := range []struct {
+ desc string
+ want string
+ err bool
+ i interface{}
+ }{
+ {
+ desc: "invalid: array with wrong type",
+ err: true,
+ i: [2]string{"first", "second"},
+ },
+ {
+ desc: "invalid: slice with wrong type",
+ err: true,
+ i: []string{"first", "second"},
+ },
+ {
+ desc: "invalid: empty slice with right type",
+ err: true,
+ i: make([][2]byte, 0),
+ },
+ {
+ desc: "invalid: empty string",
+ err: true,
+ i: "",
+ },
+ {
+ desc: "invalid: unsupported type and kind",
+ err: true,
+ i: int32(0),
+ },
+ {
+ desc: "valid: uint64",
+ want: "some key<--1;;",
+ i: uint64(1),
+ },
+ {
+ desc: "valid: byte array",
+ want: "some key<--01fe;;",
+ i: [2]byte{1, 254},
+ },
+ {
+ desc: "valid: slice array",
+ want: "some key<--01fe;;some key<--00ff;;",
+ i: [][2]byte{
+ [2]byte{1, 254},
+ [2]byte{0, 255},
+ },
+ },
+ {
+ desc: "valid: slice uint64",
+ want: "some key<--1;;some key<--2;;",
+ i: []uint64{1, 2},
+ },
+ {
+ desc: "valid: string",
+ want: "some key<--some value;;",
+ i: "some value",
+ },
+ } {
+ buf := bytes.NewBuffer(nil)
+ err := e.write(buf, "some key", reflect.ValueOf(table.i))
+ if got, want := err != nil, table.err; got != want {
+ t.Errorf("got error %v but wanted %v in test %q: %v", got, want, table.desc, err)
+ }
+ if err != nil {
+ continue
+ }
+ if got, want := string(buf.Bytes()), table.want; got != want {
+ t.Errorf("got buf %s but wanted %s in test %q", got, want, table.desc)
+ }
+ }
+}
+
+func TestWriteOne(t *testing.T) {
+ buf := bytes.NewBuffer(nil)
+ e := NewEncoding("ascii/test", "<--", ";;")
+ e.writeOne(buf, "some key", "some value")
+ want := "some key<--some value;;"
+ if got := string(buf.Bytes()); got != want {
+ t.Errorf("got buf %s but wanted %s", got, want)
+ }
+}
+
+func TestDeserialize(t *testing.T) {
+ e := NewEncoding("ascii/test", "<--", ";;")
+ for _, table := range []struct {
+ desc string
+ buf string
+ want interface{}
+ err bool
+ }{
+ {
+ desc: "invalid: interface must be pointer to struct",
+ buf: ";",
+ want: uint64(0),
+ err: true,
+ },
+ {
+ desc: "invalid: buffer too small",
+ buf: ";",
+ want: testStruct{},
+ err: true,
+ },
+ {
+ desc: "invalid: buffer must end with endOfValue",
+ buf: "num<--1;;string<--hellow;;array<--01fe;;slice<--01fe;;slice<--00ff^^",
+ want: testStruct{},
+ err: true,
+ },
+ {
+ desc: "invalid: missing key num",
+ buf: "string<--hellow;;array<--01fe;;slice<--01fe;;slice<--00ff;;",
+ want: testStruct{},
+ err: true,
+ },
+ {
+ desc: "invalid: missing key-value pair on num line",
+ buf: "string<--hellow;;num;;array<--01fe;;slice<--01fe;;slice<--00ff;;",
+ want: testStruct{},
+ err: true,
+ },
+ {
+ desc: "invalid: missing value for key num",
+ buf: "num<--;;string<--hellow;;array<--01fe;;slice<--01fe;;slice<--00ff;;",
+ want: testStruct{},
+ err: true,
+ },
+ {
+ desc: "invalid: value for key num must be digits only",
+ buf: "num<--+1;;string<--hellow;;array<--01fe;;slice<--01fe;;slice<--00ff;;",
+ want: testStruct{},
+ err: true,
+ },
+ {
+ desc: "invalid: missing field for key num2",
+ buf: "num<--1;;string<--hellow;;num2<--2;;array<--01fe;;slice<--01fe;;slice<--00ff;;",
+ want: testStruct{},
+ err: true,
+ },
+ {
+ desc: "valid",
+ buf: "num<--1;;string<--hellow;;array<--01fe;;slice<--01fe;;slice<--00ff;;",
+ want: testStruct{
+ Num: 1,
+ Struct: testStructOther{
+ Array: testArray{1, 254},
+ Slice: []testArray{
+ testArray{1, 254},
+ testArray{0, 255},
+ },
+ String: "hellow",
+ },
+ },
+ },
+ } {
+ v := reflect.New(reflect.TypeOf(table.want))
+ err := e.Deserialize(bytes.NewBuffer([]byte(table.buf)), v.Interface())
+ if got, want := err != nil, table.err; got != want {
+ t.Errorf("got error %v but wanted %v in test %q: %v", got, want, table.desc, err)
+ }
+ if err != nil {
+ continue
+ }
+
+ v = v.Elem() // have pointer to struct, get just struct as in table
+ if got, want := v.Interface(), table.want; !reflect.DeepEqual(got, want) {
+ t.Errorf("got interface %v but wanted %v in test %q", got, want, table.desc)
+ }
+ }
+
+}
+
+func TestMapKeys(t *testing.T) {
+ s := testStruct{}
+ m := make(map[string]*someValue)
+ e := NewEncoding("ascii/test", "<--", ";;")
+ if err := e.mapKeys(s, m); err == nil {
+ t.Errorf("expected mapping to fail without pointer")
+ }
+ if err := e.mapKeys(&s, m); err != nil {
+ t.Errorf("expected mapping to succeed")
+ return
+ }
+
+ wantKeys := []string{"num", "array", "slice", "string"}
+ if got, want := len(m), len(wantKeys); got != want {
+ t.Errorf("got %d keys, wanted %d", got, want)
+ }
+ for _, key := range wantKeys {
+ if _, ok := m[key]; !ok {
+ t.Errorf("expected key %q in map", key)
+ }
+ }
+}
+
+func TestSetKey(t *testing.T) {
+ for _, table := range []struct {
+ desc string
+ key string
+ value string
+ want interface{}
+ err bool
+ }{
+ {
+ desc: "invalid: unsupported type and kind",
+ key: "num",
+ value: "1",
+ want: uint32(1),
+ err: true,
+ },
+ // uint64
+ {
+ desc: "invalid: uint64: underflow",
+ key: "num",
+ value: "-1",
+ want: uint64(0),
+ err: true,
+ },
+ {
+ desc: "invalid: uint64: overflow",
+ key: "num",
+ value: "18446744073709551616",
+ want: uint64(0),
+ err: true,
+ },
+ {
+ desc: "invalid: uint64: not a number",
+ key: "num",
+ value: "+1",
+ want: uint64(0),
+ err: true,
+ },
+ {
+ desc: "invalid: uint64: number with white space",
+ key: "num",
+ value: "1 ",
+ want: uint64(0),
+ err: true,
+ },
+ {
+ desc: "valid: uint64",
+ key: "num",
+ value: "1",
+ want: uint64(1),
+ },
+ // string
+ {
+ desc: "invalid: string: empty",
+ key: "string",
+ value: "",
+ want: "",
+ err: true,
+ },
+ {
+ desc: "valid: string",
+ key: "string",
+ value: "hellow",
+ want: "hellow",
+ },
+ // array
+ {
+ desc: "invalid: array: bad hex",
+ key: "array",
+ value: "00fE",
+ want: [2]byte{},
+ err: true,
+ },
+ {
+ desc: "invalid: array: wrong size",
+ key: "array",
+ value: "01fe",
+ want: [3]byte{},
+ err: true,
+ },
+ {
+ desc: "valid: array",
+ key: "num",
+ value: "01fe",
+ want: [2]byte{1, 254},
+ },
+ // slice
+ {
+ desc: "invalid: slice: bad type",
+ key: "slice",
+ value: "01fe",
+ want: []string{
+ "hello",
+ },
+ err: true,
+ },
+ {
+ desc: "invalid: bad hex",
+ key: "slice",
+ value: "01FE",
+ want: [][2]byte{
+ [2]byte{1, 254},
+ },
+ err: true,
+ },
+ {
+ desc: "valid: slice",
+ key: "slice",
+ value: "01fe",
+ want: [][2]byte{
+ [2]byte{1, 254},
+ },
+ },
+ {
+ desc: "valid: slice",
+ key: "slice",
+ value: "4711",
+ want: []uint64{4711},
+ },
+ } {
+ ref := &someValue{
+ v: reflect.New(reflect.TypeOf(table.want)),
+ }
+ err := setKey(ref, table.key, table.value)
+ if got, want := err != nil, table.err; got != want {
+ t.Errorf("got error %v but wanted %v in test %q: %v", got, want, table.desc, err)
+ }
+ if err != nil {
+ continue
+ }
+
+ ref.v = ref.v.Elem() // get same type as table
+ if got, want := ref.v.Interface(), table.want; !reflect.DeepEqual(got, want) {
+ t.Errorf("got interface %v but wanted %v in test %q", got, want, table.desc)
+ }
+ if got, want := ref.ok, true; got != want {
+ t.Errorf("got ok %v but wanted %v in test %q", got, want, table.desc)
+ }
+ }
+}
+
+func TestDereferenceStructPointer(t *testing.T) {
+ var ts testStruct
+ if _, err := dereferenceStructPointer(ts); err == nil {
+ t.Errorf("should have failed dereferencing non-pointer")
+ }
+
+ var tsp *testStruct
+ if _, err := dereferenceStructPointer(tsp); err == nil {
+ t.Errorf("should have failed dereferencing nil-pointer")
+ }
+
+ var ta testArray
+ if _, err := dereferenceStructPointer(&ta); err == nil {
+ t.Errorf("should have failed dereferencing non-struct pointer")
+ }
+
+ if _, err := dereferenceStructPointer(&ts); err != nil {
+ t.Errorf("should have succeeded dereferencing pointer to struct")
+ }
+}
--
cgit v1.2.3