From 2171b71e920d286a9527a4dddd05c00eceb6af83 Mon Sep 17 00:00:00 2001 From: Rasmus Dahlberg Date: Mon, 20 Dec 2021 14:37:31 +0100 Subject: ascii: Add sigsum ASCII-parser and tests --- pkg/ascii/ascii.go | 305 ++++++++++++++++++++++++++++++++ pkg/ascii/ascii_test.go | 455 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 760 insertions(+) create mode 100644 pkg/ascii/ascii.go create mode 100644 pkg/ascii/ascii_test.go diff --git a/pkg/ascii/ascii.go b/pkg/ascii/ascii.go new file mode 100644 index 0000000..350890c --- /dev/null +++ b/pkg/ascii/ascii.go @@ -0,0 +1,305 @@ +// package ascii implements an ASCII key-value parser. +// +// The top-most (de)serialize must operate on a struct pointer. A struct may +// contain other structs, in which case all tag names should be unique. Public +// fields without tag names are ignored. Private fields are also ignored. +// +// The supported field types are: +// - struct +// - string (no empty strings) +// - uint64 (only digits in ASCII representation) +// - byte array (only lower-case hex in ASCII representation) +// - slice of uint64 (no empty slices) +// - slice of byte array (no empty slices) +// +// A key must not contain an encoding's end-of-key value. +// A value must not contain an encoding's end-of-value value. +// +// For additional details, please refer to the Sigsum v0 API documentation. +// +package ascii + +import ( + "bytes" + "fmt" + "io" + "reflect" + "strconv" + "strings" + + "git.sigsum.org/sigsum-lib-go/pkg/hex" +) + +var StdEncoding = NewEncoding("ascii", "=", "\n") + +type Encoding struct { + identifier string + endOfKey string + endOfValue string +} + +func NewEncoding(id, eok, eov string) *Encoding { + return &Encoding{ + identifier: id, + endOfKey: eok, + endOfValue: eov, + } +} + +type someValue struct { + v reflect.Value + ok bool +} + +// Serialize tries to serialize an interface as ASCII key-value pairs +func (e *Encoding) Serialize(w io.Writer, i interface{}) error { + v, err := dereferenceStructPointer(i) + if err != nil { + return err + } + + t := v.Type() + for i := 0; i < t.NumField(); i++ { + switch v.Field(i).Type().Kind() { + case reflect.Struct: + if err := e.Serialize(w, v.Field(i).Addr().Interface()); err != nil { + return err + } + default: + if t.Field(i).PkgPath != "" { + continue // skip private field + } + key, ok := t.Field(i).Tag.Lookup(e.identifier) + if !ok { + continue // skip public field without tag + } + + if strings.Contains(key, e.endOfKey) { + return fmt.Errorf("ascii: key %q contains end-of-key character", key) + } + if err := e.write(w, key, v.Field(i)); err != nil { + return err + } + } + } + return nil +} + +func (e *Encoding) write(w io.Writer, key string, v reflect.Value) error { + t := v.Type() + switch t { + case reflect.TypeOf(uint64(0)): + val := fmt.Sprintf("%d", v.Uint()) + return e.writeOne(w, key, val) + } + + k := t.Kind() + switch k { + case reflect.Array: + if kind := t.Elem().Kind(); kind != reflect.Uint8 { + return fmt.Errorf("ascii: array kind not supported: %v", kind) + } + + arrayLen := v.Len() + array := make([]byte, arrayLen, arrayLen) + for i := 0; i < arrayLen; i++ { + array[i] = uint8(v.Index(i).Uint()) + } + + val := hex.Serialize(array) + return e.writeOne(w, key, val) + + case reflect.Slice: + kind := t.Elem().Kind() + if kind != reflect.Array && kind != reflect.Uint64 { + return fmt.Errorf("ascii: slice kind not supported: %v", kind) + } + if v.Len() == 0 { + return fmt.Errorf("ascii: slice must not be empty") + } + + var err error + for i := 0; i < v.Len(); i++ { + err = e.write(w, key, v.Index(i)) + } + return err + + case reflect.String: + if v.Len() == 0 { + return fmt.Errorf("ascii: string must not be empty") + } + return e.writeOne(w, key, v.String()) + } + + return fmt.Errorf("ascii: unsupported type %v and kind %v", t, k) +} + +func (e *Encoding) writeOne(w io.Writer, key, value string) error { + _, err := w.Write([]byte(key + e.endOfKey + value + e.endOfValue)) + return err +} + +// Deserialize tries to deserialize a buffer of ASCII key-value pairs +func (e *Encoding) Deserialize(r io.Reader, i interface{}) error { + m := make(map[string]*someValue) + if err := e.mapKeys(i, m); err != nil { + return err + } + + buf, err := io.ReadAll(r) + if err != nil { + return fmt.Errorf("ascii: failed reading incoming buffer") + } + + // trim end of buffer so that loop does not run on an empty line + if len(buf) <= len(e.endOfValue) { + return fmt.Errorf("ascii: buffer contains no key-value pair") + } + offset := len(buf) - len(e.endOfValue) + if !bytes.Equal(buf[offset:], []byte(e.endOfValue)) { + return fmt.Errorf("ascii: buffer must end with endOfValue") + } + buf = buf[:offset] + + for _, kv := range bytes.Split(buf, []byte(e.endOfValue)) { + split := bytes.Split(kv, []byte(e.endOfKey)) + if len(split) == 1 { + return fmt.Errorf("ascii: missing key-value pair in %q", string(kv)) + } + + key := string(split[0]) + value := string(bytes.Join(split[1:], nil)) + ref, ok := m[key] + if !ok { + return fmt.Errorf("ascii: unexpected key %q", key) + } + if len(value) == 0 { + fmt.Errorf("ascii: missing value for key %q", key) + } + if err := setKey(ref, key, value); err != nil { + return err + } + } + return requireValues(m) +} + +func (e *Encoding) mapKeys(i interface{}, m map[string]*someValue) error { + v, err := dereferenceStructPointer(i) + if err != nil { + return err + } + + t := v.Type() + for i := 0; i < t.NumField(); i++ { + switch v.Field(i).Type().Kind() { + case reflect.Struct: + i := v.Field(i).Addr().Interface() + e.mapKeys(i, m) // return is always nil + default: + if t.Field(i).PkgPath != "" { + continue // skip private field + } + key, ok := t.Field(i).Tag.Lookup(e.identifier) + if !ok { + continue // skip public field without tag + } + m[key] = &someValue{ + v: v.Field(i), + } + } + } + return nil +} + +func setKey(ref *someValue, key, value string) error { + v := ref.v + if v.Kind() == reflect.Ptr && !v.IsNil() { + v = v.Elem() + } + + t := v.Type() + switch t { + case reflect.TypeOf(uint64(0)): + num, err := strconv.ParseUint(value, 10, 64) + if err != nil { + return err + } + + ref.ok = true + v.SetUint(num) + return nil + } + + k := t.Kind() + switch k { + case reflect.Array: + arrayLen := v.Len() + b, err := hex.Deserialize(value) + if err != nil { + return err + } + if len(b) != arrayLen { + return fmt.Errorf("ascii: invalid array size for key %q", key) + } + + ref.ok = true + reflect.Copy(v, reflect.ValueOf(b)) + return nil + + case reflect.Slice: + sliceType := t + kind := sliceType.Elem().Kind() + if kind != reflect.Array && kind != reflect.Uint64 { + return fmt.Errorf("ascii: slice kind not supported: %v", kind) + } + + if v.IsNil() { + v.Set(reflect.MakeSlice(sliceType, 0, 0)) + } + sv := &someValue{ + v: reflect.New(sliceType.Elem()), + } + if err := setKey(sv, key, value); err != nil { + return err + } + + ref.ok = true + v.Set(reflect.Append(v, sv.v.Elem())) + return nil + + case reflect.String: + if len(value) == 0 { + return fmt.Errorf("ascii: string must not be empty") + } + + ref.ok = true + v.SetString(value) + return nil + } + + return fmt.Errorf("ascii: unsupported type %v and kind %v", t, k) +} + +func requireValues(m map[string]*someValue) error { + for k, v := range m { + if !v.ok { + return fmt.Errorf("ascii: missing value for key %q", k) + } + } + return nil +} + +func dereferenceStructPointer(i interface{}) (*reflect.Value, error) { + v := reflect.ValueOf(i) + if v.Kind() != reflect.Ptr { + return nil, fmt.Errorf("ascii: interface value must be pointer") + } + if v.IsNil() { + return nil, fmt.Errorf("ascii: interface value must be non-nil pointer") + } + v = v.Elem() + if v.Type().Kind() != reflect.Struct { + return nil, fmt.Errorf("ascii: interface value must point to struct") + } + return &v, nil +} diff --git a/pkg/ascii/ascii_test.go b/pkg/ascii/ascii_test.go new file mode 100644 index 0000000..8aee70c --- /dev/null +++ b/pkg/ascii/ascii_test.go @@ -0,0 +1,455 @@ +package ascii + +import ( + "bytes" + "reflect" + "testing" +) + +type testStruct struct { + Num uint64 `ascii/test:"num"` + Struct testStructOther + Skip uint64 + skip uint64 +} + +type testStructOther struct { + Array testArray `ascii/test:"array"` + Slice []testArray `ascii/test:"slice"` + String string `ascii/test:"string"` +} + +type testArray [2]byte + +type testStructUnsupportedType struct { + ByteSlice []byte `ascii/test:"byte_slice"` +} + +func TestSerialize(t *testing.T) { + e := NewEncoding("ascii/test", "<--", ";;") + for _, table := range []struct { + desc string + want string + err bool + i interface{} + }{ + { + desc: "invalid: not pointer to struct", + err: true, + i: testStruct{}, + }, + { + desc: "invalid: struct with invalid key", + err: true, + i: &struct { + Num uint64 `ascii/test:"num<--nom"` + }{ + Num: 1, + }, + }, + { + desc: "invalid: struct with invalid type", + err: true, + i: &testStructUnsupportedType{ + ByteSlice: []byte("hellow"), + }, + }, + { + desc: "invalid: struct with invalid type and kind", + err: true, + i: &struct { + Struct testStructUnsupportedType + }{ + Struct: testStructUnsupportedType{ + ByteSlice: []byte("hellow"), + }, + }, + }, + { + desc: "valid", + want: "num<--1;;array<--01fe;;slice<--01fe;;slice<--00ff;;string<--hellow;;", + i: &testStruct{ + Num: 1, + Struct: testStructOther{ + Array: testArray{1, 254}, + Slice: []testArray{ + testArray{1, 254}, + testArray{0, 255}, + }, + String: "hellow", + }, + }, + }, + } { + b := bytes.NewBuffer(nil) + err := e.Serialize(b, table.i) + if got, want := err != nil, table.err; got != want { + t.Errorf("got error %v but wanted %v in test %q: %v", got, want, table.desc, err) + } + if err != nil { + continue + } + if got, want := string(b.Bytes()), table.want; got != want { + t.Errorf("got buf %s but wanted %s in test %q", got, want, table.desc) + } + } +} + +func TestWrite(t *testing.T) { + e := NewEncoding("ascii/test", "<--", ";;") + for _, table := range []struct { + desc string + want string + err bool + i interface{} + }{ + { + desc: "invalid: array with wrong type", + err: true, + i: [2]string{"first", "second"}, + }, + { + desc: "invalid: slice with wrong type", + err: true, + i: []string{"first", "second"}, + }, + { + desc: "invalid: empty slice with right type", + err: true, + i: make([][2]byte, 0), + }, + { + desc: "invalid: empty string", + err: true, + i: "", + }, + { + desc: "invalid: unsupported type and kind", + err: true, + i: int32(0), + }, + { + desc: "valid: uint64", + want: "some key<--1;;", + i: uint64(1), + }, + { + desc: "valid: byte array", + want: "some key<--01fe;;", + i: [2]byte{1, 254}, + }, + { + desc: "valid: slice array", + want: "some key<--01fe;;some key<--00ff;;", + i: [][2]byte{ + [2]byte{1, 254}, + [2]byte{0, 255}, + }, + }, + { + desc: "valid: slice uint64", + want: "some key<--1;;some key<--2;;", + i: []uint64{1, 2}, + }, + { + desc: "valid: string", + want: "some key<--some value;;", + i: "some value", + }, + } { + buf := bytes.NewBuffer(nil) + err := e.write(buf, "some key", reflect.ValueOf(table.i)) + if got, want := err != nil, table.err; got != want { + t.Errorf("got error %v but wanted %v in test %q: %v", got, want, table.desc, err) + } + if err != nil { + continue + } + if got, want := string(buf.Bytes()), table.want; got != want { + t.Errorf("got buf %s but wanted %s in test %q", got, want, table.desc) + } + } +} + +func TestWriteOne(t *testing.T) { + buf := bytes.NewBuffer(nil) + e := NewEncoding("ascii/test", "<--", ";;") + e.writeOne(buf, "some key", "some value") + want := "some key<--some value;;" + if got := string(buf.Bytes()); got != want { + t.Errorf("got buf %s but wanted %s", got, want) + } +} + +func TestDeserialize(t *testing.T) { + e := NewEncoding("ascii/test", "<--", ";;") + for _, table := range []struct { + desc string + buf string + want interface{} + err bool + }{ + { + desc: "invalid: interface must be pointer to struct", + buf: ";", + want: uint64(0), + err: true, + }, + { + desc: "invalid: buffer too small", + buf: ";", + want: testStruct{}, + err: true, + }, + { + desc: "invalid: buffer must end with endOfValue", + buf: "num<--1;;string<--hellow;;array<--01fe;;slice<--01fe;;slice<--00ff^^", + want: testStruct{}, + err: true, + }, + { + desc: "invalid: missing key num", + buf: "string<--hellow;;array<--01fe;;slice<--01fe;;slice<--00ff;;", + want: testStruct{}, + err: true, + }, + { + desc: "invalid: missing key-value pair on num line", + buf: "string<--hellow;;num;;array<--01fe;;slice<--01fe;;slice<--00ff;;", + want: testStruct{}, + err: true, + }, + { + desc: "invalid: missing value for key num", + buf: "num<--;;string<--hellow;;array<--01fe;;slice<--01fe;;slice<--00ff;;", + want: testStruct{}, + err: true, + }, + { + desc: "invalid: value for key num must be digits only", + buf: "num<--+1;;string<--hellow;;array<--01fe;;slice<--01fe;;slice<--00ff;;", + want: testStruct{}, + err: true, + }, + { + desc: "invalid: missing field for key num2", + buf: "num<--1;;string<--hellow;;num2<--2;;array<--01fe;;slice<--01fe;;slice<--00ff;;", + want: testStruct{}, + err: true, + }, + { + desc: "valid", + buf: "num<--1;;string<--hellow;;array<--01fe;;slice<--01fe;;slice<--00ff;;", + want: testStruct{ + Num: 1, + Struct: testStructOther{ + Array: testArray{1, 254}, + Slice: []testArray{ + testArray{1, 254}, + testArray{0, 255}, + }, + String: "hellow", + }, + }, + }, + } { + v := reflect.New(reflect.TypeOf(table.want)) + err := e.Deserialize(bytes.NewBuffer([]byte(table.buf)), v.Interface()) + if got, want := err != nil, table.err; got != want { + t.Errorf("got error %v but wanted %v in test %q: %v", got, want, table.desc, err) + } + if err != nil { + continue + } + + v = v.Elem() // have pointer to struct, get just struct as in table + if got, want := v.Interface(), table.want; !reflect.DeepEqual(got, want) { + t.Errorf("got interface %v but wanted %v in test %q", got, want, table.desc) + } + } + +} + +func TestMapKeys(t *testing.T) { + s := testStruct{} + m := make(map[string]*someValue) + e := NewEncoding("ascii/test", "<--", ";;") + if err := e.mapKeys(s, m); err == nil { + t.Errorf("expected mapping to fail without pointer") + } + if err := e.mapKeys(&s, m); err != nil { + t.Errorf("expected mapping to succeed") + return + } + + wantKeys := []string{"num", "array", "slice", "string"} + if got, want := len(m), len(wantKeys); got != want { + t.Errorf("got %d keys, wanted %d", got, want) + } + for _, key := range wantKeys { + if _, ok := m[key]; !ok { + t.Errorf("expected key %q in map", key) + } + } +} + +func TestSetKey(t *testing.T) { + for _, table := range []struct { + desc string + key string + value string + want interface{} + err bool + }{ + { + desc: "invalid: unsupported type and kind", + key: "num", + value: "1", + want: uint32(1), + err: true, + }, + // uint64 + { + desc: "invalid: uint64: underflow", + key: "num", + value: "-1", + want: uint64(0), + err: true, + }, + { + desc: "invalid: uint64: overflow", + key: "num", + value: "18446744073709551616", + want: uint64(0), + err: true, + }, + { + desc: "invalid: uint64: not a number", + key: "num", + value: "+1", + want: uint64(0), + err: true, + }, + { + desc: "invalid: uint64: number with white space", + key: "num", + value: "1 ", + want: uint64(0), + err: true, + }, + { + desc: "valid: uint64", + key: "num", + value: "1", + want: uint64(1), + }, + // string + { + desc: "invalid: string: empty", + key: "string", + value: "", + want: "", + err: true, + }, + { + desc: "valid: string", + key: "string", + value: "hellow", + want: "hellow", + }, + // array + { + desc: "invalid: array: bad hex", + key: "array", + value: "00fE", + want: [2]byte{}, + err: true, + }, + { + desc: "invalid: array: wrong size", + key: "array", + value: "01fe", + want: [3]byte{}, + err: true, + }, + { + desc: "valid: array", + key: "num", + value: "01fe", + want: [2]byte{1, 254}, + }, + // slice + { + desc: "invalid: slice: bad type", + key: "slice", + value: "01fe", + want: []string{ + "hello", + }, + err: true, + }, + { + desc: "invalid: bad hex", + key: "slice", + value: "01FE", + want: [][2]byte{ + [2]byte{1, 254}, + }, + err: true, + }, + { + desc: "valid: slice", + key: "slice", + value: "01fe", + want: [][2]byte{ + [2]byte{1, 254}, + }, + }, + { + desc: "valid: slice", + key: "slice", + value: "4711", + want: []uint64{4711}, + }, + } { + ref := &someValue{ + v: reflect.New(reflect.TypeOf(table.want)), + } + err := setKey(ref, table.key, table.value) + if got, want := err != nil, table.err; got != want { + t.Errorf("got error %v but wanted %v in test %q: %v", got, want, table.desc, err) + } + if err != nil { + continue + } + + ref.v = ref.v.Elem() // get same type as table + if got, want := ref.v.Interface(), table.want; !reflect.DeepEqual(got, want) { + t.Errorf("got interface %v but wanted %v in test %q", got, want, table.desc) + } + if got, want := ref.ok, true; got != want { + t.Errorf("got ok %v but wanted %v in test %q", got, want, table.desc) + } + } +} + +func TestDereferenceStructPointer(t *testing.T) { + var ts testStruct + if _, err := dereferenceStructPointer(ts); err == nil { + t.Errorf("should have failed dereferencing non-pointer") + } + + var tsp *testStruct + if _, err := dereferenceStructPointer(tsp); err == nil { + t.Errorf("should have failed dereferencing nil-pointer") + } + + var ta testArray + if _, err := dereferenceStructPointer(&ta); err == nil { + t.Errorf("should have failed dereferencing non-struct pointer") + } + + if _, err := dereferenceStructPointer(&ts); err != nil { + t.Errorf("should have succeeded dereferencing pointer to struct") + } +} -- cgit v1.2.3