From f393059c0be6f5e9f1ae4ca90443043f0f69788b Mon Sep 17 00:00:00 2001 From: Roman Sharkov Date: Fri, 26 Jan 2024 00:30:28 +0100 Subject: [PATCH] feat: Add token value getter methods --- bench_test.go | 2 +- example_test.go | 46 +-- internal/atoi/atoi.go | 675 +++++++++++++++++++++++++++++++++ internal/atoi/atoi_test.go | 671 +++++++++++++++++++++++++++++++++ internal/atoi/doc.go | 8 + jscan_test.go | 749 +++++++++++++++++++++++++++++++++++-- tokenize.go | 336 ++++++++++++++++- 7 files changed, 2403 insertions(+), 84 deletions(-) create mode 100644 internal/atoi/atoi.go create mode 100644 internal/atoi/atoi_test.go create mode 100644 internal/atoi/doc.go diff --git a/bench_test.go b/bench_test.go index bb22cca..4ff510f 100644 --- a/bench_test.go +++ b/bench_test.go @@ -71,7 +71,7 @@ func MustCalcStatsJscan(p *jscan.Parser[[]byte], str []byte) (s Stats) { func MustCalcStatsJscanTokenizer[S []byte | string](p *jscan.Tokenizer[S], str S) (s Stats) { if err := p.Tokenize( str, - func(tokens []jscan.Token) (err bool) { + func(tokens []jscan.Token[S]) (err bool) { depth := 0 for i := range tokens { switch tokens[i].Type { diff --git a/example_test.go b/example_test.go index 92c3e65..fb511b7 100644 --- a/example_test.go +++ b/example_test.go @@ -254,68 +254,68 @@ func ExampleTokenizer_decodeVector3DArray() { var data []Vector3D var err error - errTokenizer := tokenizer.Tokenize(src, func(tokens []jscan.Token) (errTok bool) { - if tokens[0].Type != jscan.TokenTypeArray { - err = fmt.Errorf("expected array at index %d", tokens[0].Index) + errTokenizer := tokenizer.Tokenize(src, func(t []jscan.Token[string]) (errTok bool) { + if t[0].Type != jscan.TokenTypeArray { + err = fmt.Errorf("expected array at index %d", t[0].Index) return true } - tokens = tokens[1 : len(tokens)-1] + t = t[1 : len(t)-1] // Preallocate slice since we know the number of objects in advance. - data = make([]Vector3D, tokens[0].Elements) + data = make([]Vector3D, t[0].Elements) - mustParseField := func(defined bool, val jscan.Token) (float64, error) { + mustParseField := func(defined bool, val jscan.Token[string]) (float64, error) { if defined { - return 0, fmt.Errorf("duplicated field at index %d", tokens[0].Index) + return 0, fmt.Errorf("duplicated field at index %d", t[0].Index) } if val.Type != jscan.TokenTypeNumber && val.Type != jscan.TokenTypeInteger { - return 0, fmt.Errorf("expected number at index %d", tokens[0].Index) + return 0, fmt.Errorf("expected number at index %d", t[0].Index) } v, errParse := strconv.ParseFloat(src[val.Index:val.End], 64) if errParse != nil { return 0, fmt.Errorf("parsing number at index %d: %v", - tokens[0].Index, err) + t[0].Index, err) } return v, nil } for i := 0; i < len(data); i++ { - if tokens[0].Type != jscan.TokenTypeObject { - err = fmt.Errorf("expected object at index %d", tokens[0].Index) + if t[0].Type != jscan.TokenTypeObject { + err = fmt.Errorf("expected object at index %d", t[0].Index) return true } - tokens = tokens[1:] // Skip object start token + t = t[1:] // Skip object start token hasX, hasY, hasZ := false, false, false for k := 0; k < 3; k++ { - fieldName := src[tokens[0].Index:tokens[0].End] + fieldName := src[t[0].Index:t[0].End] switch string(fieldName) { case `"x"`: - if data[i].X, err = mustParseField(hasX, tokens[1]); err != nil { + if data[i].X, err = mustParseField(hasX, t[1]); err != nil { return true } - hasX, tokens = true, tokens[2:] // Skip key and value + hasX, t = true, t[2:] // Skip key and value case `"y"`: - if data[i].Y, err = mustParseField(hasY, tokens[1]); err != nil { + if data[i].Y, err = mustParseField(hasY, t[1]); err != nil { return true } - hasY, tokens = true, tokens[2:] + hasY, t = true, t[2:] case `"z"`: - if data[i].Z, err = mustParseField(hasZ, tokens[1]); err != nil { + if data[i].Z, err = mustParseField(hasZ, t[1]); err != nil { return true } - hasZ, tokens = true, tokens[2:] + hasZ, t = true, t[2:] default: err = fmt.Errorf("unknown field %q at index %d", - string(fieldName), tokens[0].Index) + string(fieldName), t[0].Index) return true } } - if tokens[0].Type != jscan.TokenTypeObjectEnd { + if t[0].Type != jscan.TokenTypeObjectEnd { err = fmt.Errorf("unknown extra field %q in object at index %d", - string(src[tokens[0].Index:tokens[0].End]), tokens[0].Index) + string(src[t[0].Index:t[0].End]), t[0].Index) } - tokens = tokens[1:] // Skip object end + t = t[1:] // Skip object end } return false diff --git a/internal/atoi/atoi.go b/internal/atoi/atoi.go new file mode 100644 index 0000000..2248179 --- /dev/null +++ b/internal/atoi/atoi.go @@ -0,0 +1,675 @@ +package atoi + +// U8 assumes that s is a valid unsigned 8-bit integer. +// Returns (0, true) if the value would overflow. +func U8[S ~string | ~[]byte](s S) (n uint8, overflow bool) { + d := func(index int) uint8 { return uint8(s[index] - '0') } + switch len(s) { + case 1: + return d(0), false + case 2: + return d(0)*1e1 + d(1), false + case 3: // This case can overflow + if s[0]-'0' > 2 { + return 0, true + } + p := d(1)*1e1 + d(2) + n = p + d(0)*1e2 + if n <= p { + return 0, true + } + return n, false + } + return 0, true // Anything above 3 digits overflows uint8 +} + +// I8 assumes that s is a valid 8-bit signed or unsigned integer (where +// only '-' is accepted). Returns (0, true) if the value would overflow. +func I8[S ~string | ~[]byte](s S) (n int8, overflow bool) { + d := func(index int) int8 { return int8(s[index] - '0') } + if s[0] == '-' { + switch len(s) { + case 2: + return -d(1), false + case 3: + return -(d(1)*1e1 + d(2)), false + case 4: // This case can overflow + if n = -(d(1)*1e2 + d(2)*1e1 + d(3)); n > 0 { + return 0, true + } + return n, false + } + } else { + switch len(s) { + case 1: + return d(0), false + case 2: + return d(0)*1e1 + d(1), false + case 3: // This case can overflow + if n = d(0)*1e2 + d(1)*1e1 + d(2); n < 0 { + return 0, true + } + return n, false + } + } + return 0, true // Anything above 3 digits overflows int8 +} + +// U16 assumes that s is a valid unsigned 16-bit integer. +// Returns (0, true) if the value would overflow. +func U16[S ~string | ~[]byte](s S) (n uint16, overflow bool) { + d := func(index int) uint16 { return uint16(s[index] - '0') } + switch len(s) { + case 1: + return d(0), false + case 2: + return d(0)*1e1 + d(1), false + case 3: + return d(0)*1e2 + d(1)*1e1 + d(2), false + case 4: + return d(0)*1e3 + d(1)*1e2 + d(2)*1e1 + d(3), false + case 5: // This case can overflow + if s[0]-'0' > 6 { + return 0, true + } + p := d(1)*1e3 + d(2)*1e2 + d(3)*1e1 + d(4) + n = p + d(0)*1e4 + if n <= p { + return 0, true + } + return n, false + } + return 0, true // Anything above 5 digits overflows uint16 +} + +// I16 assumes that s is a valid 16-bit signed or unsigned integer (where +// only '-' is accepted). Returns (0, true) if the value would overflow. +func I16[S ~string | ~[]byte](s S) (n int16, overflow bool) { + d := func(index int) int16 { return int16(s[index] - '0') } + if s[0] == '-' { + switch len(s) { + case 2: + return -d(1), false + case 3: + return -(n*1e2 + d(1)*1e1 + d(2)), false + case 4: + return -(d(1)*1e2 + d(2)*1e1 + d(3)), false + case 5: + return -(d(1)*1e3 + d(2)*1e2 + d(3)*1e1 + d(4)), false + case 6: // This case can overflow + n = -(d(1)*1e4 + d(2)*1e3 + d(3)*1e2 + d(4)*1e1 + d(5)) + if n > 0 { + return 0, true + } + return n, false + } + } else { + switch len(s) { + case 1: + return d(0), false + case 2: + return d(0)*1e1 + d(1), false + case 3: + return d(0)*1e2 + d(1)*1e1 + d(2), false + case 4: + return d(0)*1e3 + d(1)*1e2 + d(2)*1e1 + d(3), false + case 5: // This case can overflow + n = d(0)*1e4 + d(1)*1e3 + d(2)*1e2 + d(3)*1e1 + d(4) + if n < 0 { + return 0, true + } + return n, false + } + } + return 0, true // Anything above 5 digits overflows int16 +} + +// U32 assumes that s is a valid unsigned 32-bit integer. +// Returns (0, true) if the value would overflow. +func U32[S ~string | ~[]byte](s S) (n uint32, overflow bool) { + d := func(index int) uint32 { return uint32(s[index] - '0') } + switch len(s) { + case 1: + return d(0), false + case 2: + return d(0)*1e1 + d(1), false + case 3: + return d(0)*1e2 + d(1)*1e1 + d(2), false + case 4: + return d(0)*1e3 + d(1)*1e2 + d(2)*1e1 + d(3), false + case 5: + return d(0)*1e4 + d(1)*1e3 + d(2)*1e2 + d(3)*1e1 + d(4), false + case 6: + return d(0)*1e5 + d(1)*1e4 + + d(2)*1e3 + d(3)*1e2 + + d(4)*1e1 + d(5), false + case 7: + return d(0)*1e6 + d(1)*1e5 + + d(2)*1e4 + d(3)*1e3 + + d(4)*1e2 + d(5)*1e1 + + d(6), false + case 8: + return d(0)*1e7 + d(1)*1e6 + + d(2)*1e5 + d(3)*1e4 + + d(4)*1e3 + d(5)*1e2 + + d(6)*1e1 + d(7), false + case 9: + return d(0)*1e8 + d(1)*1e7 + + d(2)*1e6 + d(3)*1e5 + + d(4)*1e4 + d(5)*1e3 + + d(6)*1e2 + d(7)*1e1 + + d(8), false + case 10: // This case can overflow + if s[0]-'0' > 4 { + return 0, true + } + p := d(1)*1e8 + d(2)*1e7 + + d(3)*1e6 + d(4)*1e5 + + d(5)*1e4 + d(6)*1e3 + + d(7)*1e2 + d(8)*1e1 + + d(9) + n = p + d(0)*1e9 + if n <= p { + return 0, true + } + return n, false + } + return 0, true // Anything above 10 digits overflows uint32 +} + +// I32 assumes that s is a valid 32-bit signed or unsigned integer (where +// only '-' is accepted). Returns (0, true) if the value would overflow. +func I32[S ~string | ~[]byte](s S) (n int32, overflow bool) { + d := func(index int) int32 { return int32(s[index] - '0') } + if s[0] == '-' { + switch len(s) { + case 2: + return -d(1), false + case 3: + return -(n*1e2 + d(1)*1e1 + d(2)), false + case 4: + return -(d(1)*1e2 + d(2)*1e1 + d(3)), false + case 5: + return -(d(1)*1e3 + d(2)*1e2 + d(3)*1e1 + d(4)), false + case 6: + return -(d(1)*1e4 + d(2)*1e3 + d(3)*1e2 + d(4)*1e1 + d(5)), false + case 7: + return -(d(1)*1e5 + d(2)*1e4 + + d(3)*1e3 + d(4)*1e2 + + d(5)*1e1 + d(6)), false + case 8: + return -(d(1)*1e6 + d(2)*1e5 + + d(3)*1e4 + d(4)*1e3 + + d(5)*1e2 + d(6)*1e1 + + d(7)), false + case 9: + return -(d(1)*1e7 + d(2)*1e6 + + d(3)*1e5 + d(4)*1e4 + + d(5)*1e3 + d(6)*1e2 + + d(7)*1e1 + d(8)), false + case 10: + return -(d(1)*1e8 + d(2)*1e7 + + d(3)*1e6 + d(4)*1e5 + + d(5)*1e4 + d(6)*1e3 + + d(7)*1e2 + d(8)*1e1 + + d(9)), false + case 11: // This case can overflow + n = d(1)*1e9 + d(2)*1e8 + + d(3)*1e7 + d(4)*1e6 + + d(5)*1e5 + d(6)*1e4 + + d(7)*1e3 + d(8)*1e2 + + d(9)*1e1 + d(10) + n = -n + if n > 0 { + return 0, true + } + return n, false + } + } else { + switch len(s) { + case 1: + return d(0), false + case 2: + return d(0)*1e1 + d(1), false + case 3: + return d(0)*1e2 + d(1)*1e1 + d(2), false + case 4: + return d(0)*1e3 + d(1)*1e2 + d(2)*1e1 + d(3), false + case 5: + return d(0)*1e4 + d(1)*1e3 + d(2)*1e2 + d(3)*1e1 + d(4), false + case 6: + return d(0)*1e5 + d(1)*1e4 + + d(2)*1e3 + d(3)*1e2 + + d(4)*1e1 + d(5), false + case 7: + return d(0)*1e6 + d(1)*1e5 + + d(2)*1e4 + d(3)*1e3 + + d(4)*1e2 + d(5)*1e1 + + d(6), false + case 8: + return d(0)*1e7 + d(1)*1e6 + + d(2)*1e5 + d(3)*1e4 + + d(4)*1e3 + d(5)*1e2 + + d(6)*1e1 + d(7), false + case 9: + return d(0)*1e8 + d(1)*1e7 + + d(2)*1e6 + d(3)*1e5 + + d(4)*1e4 + d(5)*1e3 + + d(6)*1e2 + d(7)*1e1 + + d(8), false + case 10: // This case can overflow + n = d(0)*1e9 + d(1)*1e8 + + d(2)*1e7 + d(3)*1e6 + + d(4)*1e5 + d(5)*1e4 + + d(6)*1e3 + d(7)*1e2 + + d(8)*1e1 + d(9) + if n < 0 { + return 0, true + } + return n, false + } + } + return 0, true // Anything above 10 digits overflows int32 +} + +// U64 assumes that s is a valid unsigned 64-bit integer. +// Returns (0, true) if the value would overflow. +func U64[S ~string | ~[]byte](s S) (n uint64, overflow bool) { + d := func(index int) uint64 { return uint64(s[index] - '0') } + switch len(s) { + case 1: + return d(0), false + case 2: + return d(0)*1e1 + d(1), false + case 3: + return d(0)*1e2 + d(1)*1e1 + + d(2), false + case 4: + return d(0)*1e3 + d(1)*1e2 + + d(2)*1e1 + d(3), false + case 5: + return d(0)*1e4 + d(1)*1e3 + + d(2)*1e2 + d(3)*1e1 + + d(4), false + case 6: + return d(0)*1e5 + d(1)*1e4 + + d(2)*1e3 + d(3)*1e2 + + d(4)*1e1 + d(5), false + case 7: + return d(0)*1e6 + d(1)*1e5 + + d(2)*1e4 + d(3)*1e3 + + d(4)*1e2 + d(5)*1e1 + + d(6), false + case 8: + return d(0)*1e7 + d(1)*1e6 + + d(2)*1e5 + d(3)*1e4 + + d(4)*1e3 + d(5)*1e2 + + d(6)*1e1 + d(7), false + case 9: + return d(0)*1e8 + d(1)*1e7 + + d(2)*1e6 + d(3)*1e5 + + d(4)*1e4 + d(5)*1e3 + + d(6)*1e2 + d(7)*1e1 + + d(8), false + case 10: + return d(0)*1e9 + d(1)*1e8 + + d(2)*1e7 + d(3)*1e6 + + d(4)*1e5 + d(5)*1e4 + + d(6)*1e3 + d(7)*1e2 + + d(8)*1e1 + d(9), false + case 11: + return d(0)*1e10 + d(1)*1e9 + + d(2)*1e8 + d(3)*1e7 + + d(4)*1e6 + d(5)*1e5 + + d(6)*1e4 + d(7)*1e3 + + d(8)*1e2 + d(9)*1e1 + + d(10), false + case 12: + return d(0)*1e11 + d(1)*1e10 + + d(2)*1e9 + d(3)*1e8 + + d(4)*1e7 + d(5)*1e6 + + d(6)*1e5 + d(7)*1e4 + + d(8)*1e3 + d(9)*1e2 + + d(10)*1e1 + d(11), false + case 13: + return d(0)*1e12 + d(1)*1e11 + + d(2)*1e10 + d(3)*1e9 + + d(4)*1e8 + d(5)*1e7 + + d(6)*1e6 + d(7)*1e5 + + d(8)*1e4 + d(9)*1e3 + + d(10)*1e2 + d(11)*1e1 + + d(12), false + case 14: + return d(0)*1e13 + d(1)*1e12 + + d(2)*1e11 + d(3)*1e10 + + d(4)*1e9 + d(5)*1e8 + + d(6)*1e7 + d(7)*1e6 + + d(8)*1e5 + d(9)*1e4 + + d(10)*1e3 + d(11)*1e2 + + d(12)*1e1 + d(13), false + case 15: + return d(0)*1e14 + d(1)*1e13 + + d(2)*1e12 + d(3)*1e11 + + d(4)*1e10 + d(5)*1e9 + + d(6)*1e8 + d(7)*1e7 + + d(8)*1e6 + d(9)*1e5 + + d(10)*1e4 + d(11)*1e3 + + d(12)*1e2 + d(13)*1e1 + + d(14), false + case 16: + return d(0)*1e15 + d(1)*1e14 + + d(2)*1e13 + d(3)*1e12 + + d(4)*1e11 + d(5)*1e10 + + d(6)*1e9 + d(7)*1e8 + + d(8)*1e7 + d(9)*1e6 + + d(10)*1e5 + d(11)*1e4 + + d(12)*1e3 + d(13)*1e2 + + d(14)*1e1 + d(15), false + case 17: + return d(0)*1e16 + d(1)*1e15 + + d(2)*1e14 + d(3)*1e13 + + d(4)*1e12 + d(5)*1e11 + + d(6)*1e10 + d(7)*1e9 + + d(8)*1e8 + d(9)*1e7 + + d(10)*1e6 + d(11)*1e5 + + d(12)*1e4 + d(13)*1e3 + + d(14)*1e2 + d(15)*1e1 + + d(16), false + case 18: + return d(0)*1e17 + d(1)*1e16 + + d(2)*1e15 + d(3)*1e14 + + d(4)*1e13 + d(5)*1e12 + + d(6)*1e11 + d(7)*1e10 + + d(8)*1e9 + d(9)*1e8 + + d(10)*1e7 + d(11)*1e6 + + d(12)*1e5 + d(13)*1e4 + + d(14)*1e3 + d(15)*1e2 + + d(16)*1e1 + d(17), false + case 19: + return d(0)*1e18 + d(1)*1e17 + + d(2)*1e16 + d(3)*1e15 + + d(4)*1e14 + d(5)*1e13 + + d(6)*1e12 + d(7)*1e11 + + d(8)*1e10 + d(9)*1e9 + + d(10)*1e8 + d(11)*1e7 + + d(12)*1e6 + d(13)*1e5 + + d(14)*1e4 + d(15)*1e3 + + d(16)*1e2 + d(17)*1e1 + + d(18), false + case 20: // This case can overflow + if s[0] != '1' { + return 0, true + } + n = d(1)*1e18 + d(2)*1e17 + + d(3)*1e16 + d(4)*1e15 + + d(5)*1e14 + d(6)*1e13 + + d(7)*1e12 + d(8)*1e11 + + d(9)*1e10 + d(10)*1e9 + + d(11)*1e8 + d(12)*1e7 + + d(13)*1e6 + d(14)*1e5 + + d(15)*1e4 + d(16)*1e3 + + d(17)*1e2 + d(18)*1e1 + + d(19) + if n > 8446744073709551615 { + return 0, true + } + return 1e19 + n, false + } + return 0, true // Anything above 20 digits overflows uint64 +} + +// I64 assumes that s is a valid 64-bit signed or unsigned integer (where +// only '-' is accepted). Returns (0, true) if the value would overflow. +func I64[S ~string | ~[]byte](s S) (n int64, overflow bool) { + d := func(index int) int64 { return int64(s[index] - '0') } + if s[0] == '-' { + switch len(s) { + case 2: + return -d(1), false + case 3: + return -(n*1e2 + d(1)*1e1 + d(2)), false + case 4: + return -(d(1)*1e2 + d(2)*1e1 + + d(3)), false + case 5: + return -(d(1)*1e3 + d(2)*1e2 + + d(3)*1e1 + d(4)), false + case 6: + return -(d(1)*1e4 + d(2)*1e3 + + d(3)*1e2 + d(4)*1e1 + + d(5)), false + case 7: + return -(d(1)*1e5 + d(2)*1e4 + + d(3)*1e3 + d(4)*1e2 + + d(5)*1e1 + d(6)), false + case 8: + return -(d(1)*1e6 + d(2)*1e5 + + d(3)*1e4 + d(4)*1e3 + + d(5)*1e2 + d(6)*1e1 + + d(7)), false + case 9: + return -(d(1)*1e7 + d(2)*1e6 + + d(3)*1e5 + d(4)*1e4 + + d(5)*1e3 + d(6)*1e2 + + d(7)*1e1 + d(8)), false + case 10: + return -(d(1)*1e8 + d(2)*1e7 + + d(3)*1e6 + d(4)*1e5 + + d(5)*1e4 + d(6)*1e3 + + d(7)*1e2 + d(8)*1e1 + + d(9)), false + case 11: + return -(d(1)*1e9 + d(2)*1e8 + + d(3)*1e7 + d(4)*1e6 + + d(5)*1e5 + d(6)*1e4 + + d(7)*1e3 + d(8)*1e2 + + d(9)*1e1 + d(10)), false + case 12: + return -(d(1)*1e10 + d(2)*1e9 + + d(3)*1e8 + d(4)*1e7 + + d(5)*1e6 + d(6)*1e5 + + d(7)*1e4 + d(8)*1e3 + + d(9)*1e2 + d(10)*1e1 + + d(11)), false + case 13: + return -(d(1)*1e11 + d(2)*1e10 + + d(3)*1e9 + d(4)*1e8 + + d(5)*1e7 + d(6)*1e6 + + d(7)*1e5 + d(8)*1e4 + + d(9)*1e3 + d(10)*1e2 + + d(11)*1e1 + d(12)), false + case 14: + return -(d(1)*1e12 + d(2)*1e11 + + d(3)*1e10 + d(4)*1e9 + + d(5)*1e8 + d(6)*1e7 + + d(7)*1e6 + d(8)*1e5 + + d(9)*1e4 + d(10)*1e3 + + d(11)*1e2 + d(12)*1e1 + + d(13)), false + case 15: + return -(d(1)*1e13 + d(2)*1e12 + + d(3)*1e11 + d(4)*1e10 + + d(5)*1e9 + d(6)*1e8 + + d(7)*1e7 + d(8)*1e6 + + d(9)*1e5 + d(10)*1e4 + + d(11)*1e3 + d(12)*1e2 + + d(13)*1e1 + d(14)), false + case 16: + return -(d(1)*1e14 + d(2)*1e13 + + d(3)*1e12 + d(4)*1e11 + + d(5)*1e10 + d(6)*1e9 + + d(7)*1e8 + d(8)*1e7 + + d(9)*1e6 + d(10)*1e5 + + d(11)*1e4 + d(12)*1e3 + + d(13)*1e2 + d(14)*1e1 + + d(15)), false + case 17: + return -(d(1)*1e15 + d(2)*1e14 + + d(3)*1e13 + d(4)*1e12 + + d(5)*1e11 + d(6)*1e10 + + d(7)*1e9 + d(8)*1e8 + + d(9)*1e7 + d(10)*1e6 + + d(11)*1e5 + d(12)*1e4 + + d(13)*1e3 + d(14)*1e2 + + d(15)*1e1 + d(16)), false + case 18: + return -(d(1)*1e16 + d(2)*1e15 + + d(3)*1e14 + d(4)*1e13 + + d(5)*1e12 + d(6)*1e11 + + d(7)*1e10 + d(8)*1e9 + + d(9)*1e8 + d(10)*1e7 + + d(11)*1e6 + d(12)*1e5 + + d(13)*1e4 + d(14)*1e3 + + d(15)*1e2 + d(16)*1e1 + + d(17)), false + case 19: + return -(d(1)*1e17 + d(2)*1e16 + + d(3)*1e15 + d(4)*1e14 + + d(5)*1e13 + d(6)*1e12 + + d(7)*1e11 + d(8)*1e10 + + d(9)*1e9 + d(10)*1e8 + + d(11)*1e7 + d(12)*1e6 + + d(13)*1e5 + d(14)*1e4 + + d(15)*1e3 + d(16)*1e2 + + d(17)*1e1 + d(18)), false + case 20: // This case can overflow + n = d(1)*1e18 + d(2)*1e17 + d(3)*1e16 + d(4)*1e15 + + d(5)*1e14 + d(6)*1e13 + d(7)*1e12 + d(8)*1e11 + + d(9)*1e10 + d(10)*1e9 + d(11)*1e8 + d(12)*1e7 + + d(13)*1e6 + d(14)*1e5 + d(15)*1e4 + d(16)*1e3 + + d(17)*1e2 + d(18)*1e1 + d(19) + n = -n + if n > 0 { + return 0, true + } + return n, false + } + } else { + switch len(s) { + case 1: + return d(0), false + case 2: + return d(0)*1e1 + d(1), false + case 3: + return d(0)*1e2 + d(1)*1e1 + + d(2), false + case 4: + return d(0)*1e3 + d(1)*1e2 + + d(2)*1e1 + d(3), false + case 5: + return d(0)*1e4 + d(1)*1e3 + + d(2)*1e2 + d(3)*1e1 + + d(4), false + case 6: + return d(0)*1e5 + d(1)*1e4 + + d(2)*1e3 + d(3)*1e2 + + d(4)*1e1 + d(5), false + case 7: + return d(0)*1e6 + d(1)*1e5 + + d(2)*1e4 + d(3)*1e3 + + d(4)*1e2 + d(5)*1e1 + + d(6), false + case 8: + return d(0)*1e7 + d(1)*1e6 + + d(2)*1e5 + d(3)*1e4 + + d(4)*1e3 + d(5)*1e2 + + d(6)*1e1 + d(7), false + case 9: + return d(0)*1e8 + d(1)*1e7 + + d(2)*1e6 + d(3)*1e5 + + d(4)*1e4 + d(5)*1e3 + + d(6)*1e2 + d(7)*1e1 + + d(8), false + case 10: + return d(0)*1e9 + d(1)*1e8 + + d(2)*1e7 + d(3)*1e6 + + d(4)*1e5 + d(5)*1e4 + + d(6)*1e3 + d(7)*1e2 + + d(8)*1e1 + d(9), false + case 11: + return d(0)*1e10 + d(1)*1e9 + + d(2)*1e8 + d(3)*1e7 + + d(4)*1e6 + d(5)*1e5 + + d(6)*1e4 + d(7)*1e3 + + d(8)*1e2 + d(9)*1e1 + + d(10), false + case 12: + return d(0)*1e11 + d(1)*1e10 + + d(2)*1e9 + d(3)*1e8 + + d(4)*1e7 + d(5)*1e6 + + d(6)*1e5 + d(7)*1e4 + + d(8)*1e3 + d(9)*1e2 + + d(10)*1e1 + d(11), false + case 13: + return d(0)*1e12 + d(1)*1e11 + + d(2)*1e10 + d(3)*1e9 + + d(4)*1e8 + d(5)*1e7 + + d(6)*1e6 + d(7)*1e5 + + d(8)*1e4 + d(9)*1e3 + + d(10)*1e2 + d(11)*1e1 + + d(12), false + case 14: + return d(0)*1e13 + d(1)*1e12 + + d(2)*1e11 + d(3)*1e10 + + d(4)*1e9 + d(5)*1e8 + + d(6)*1e7 + d(7)*1e6 + + d(8)*1e5 + d(9)*1e4 + + d(10)*1e3 + d(11)*1e2 + + d(12)*1e1 + d(13), false + case 15: + return d(0)*1e14 + d(1)*1e13 + + d(2)*1e12 + d(3)*1e11 + + d(4)*1e10 + d(5)*1e9 + + d(6)*1e8 + d(7)*1e7 + + d(8)*1e6 + d(9)*1e5 + + d(10)*1e4 + d(11)*1e3 + + d(12)*1e2 + d(13)*1e1 + + d(14), false + case 16: + return d(0)*1e15 + d(1)*1e14 + + d(2)*1e13 + d(3)*1e12 + + d(4)*1e11 + d(5)*1e10 + + d(6)*1e9 + d(7)*1e8 + + d(8)*1e7 + d(9)*1e6 + + d(10)*1e5 + d(11)*1e4 + + d(12)*1e3 + d(13)*1e2 + + d(14)*1e1 + d(15), false + case 17: + return d(0)*1e16 + d(1)*1e15 + + d(2)*1e14 + d(3)*1e13 + + d(4)*1e12 + d(5)*1e11 + + d(6)*1e10 + d(7)*1e9 + + d(8)*1e8 + d(9)*1e7 + + d(10)*1e6 + d(11)*1e5 + + d(12)*1e4 + d(13)*1e3 + + d(14)*1e2 + d(15)*1e1 + + d(16), false + case 18: + return d(0)*1e17 + d(1)*1e16 + + d(2)*1e15 + d(3)*1e14 + + d(4)*1e13 + d(5)*1e12 + + d(6)*1e11 + d(7)*1e10 + + d(8)*1e9 + d(9)*1e8 + + d(10)*1e7 + d(11)*1e6 + + d(12)*1e5 + d(13)*1e4 + + d(14)*1e3 + d(15)*1e2 + + d(16)*1e1 + d(17), false + case 19: // This case can overflow + n = d(0)*1e18 + d(1)*1e17 + + d(2)*1e16 + d(3)*1e15 + + d(4)*1e14 + d(5)*1e13 + + d(6)*1e12 + d(7)*1e11 + + d(8)*1e10 + d(9)*1e9 + + d(10)*1e8 + d(11)*1e7 + + d(12)*1e6 + d(13)*1e5 + + d(14)*1e4 + d(15)*1e3 + + d(16)*1e2 + d(17)*1e1 + + d(18) + if n < 0 { + return 0, true + } + return n, false + } + } + return 0, true // Anything above 19 digits overflows int64 +} diff --git a/internal/atoi/atoi_test.go b/internal/atoi/atoi_test.go new file mode 100644 index 0000000..eae8ee5 --- /dev/null +++ b/internal/atoi/atoi_test.go @@ -0,0 +1,671 @@ +package atoi_test + +import ( + "math" + "runtime" + "strconv" + "testing" + + "github.com/romshark/jscan/v2/internal/atoi" + + "github.com/stretchr/testify/require" +) + +func TestU64(t *testing.T) { + for _, td := range []struct { + name string + input string + expect uint64 + overflow bool + }{ + {name: "0", input: `0`, expect: 0}, + {name: "1", input: `1`, expect: 1}, + {name: "12", input: `12`, expect: 12}, + {name: "123", input: `123`, expect: 123}, + {name: "1234", input: `1234`, expect: 1234}, + {name: "12345", input: `12345`, expect: 12345}, + {name: "123456", input: `123456`, expect: 123456}, + {name: "1234567", input: `1234567`, expect: 1234567}, + {name: "12345678", input: `12345678`, expect: 12345678}, + {name: "123456789", input: `123456789`, expect: 123456789}, + {name: "1234567891", input: `1234567891`, expect: 1234567891}, + {name: "12345678912", input: `12345678912`, expect: 12345678912}, + {name: "123456789123", input: `123456789123`, expect: 123456789123}, + {name: "1234567891234", input: `1234567891234`, expect: 1234567891234}, + {name: "12345678912345", input: `12345678912345`, expect: 12345678912345}, + { + name: "123456789123456", + input: `123456789123456`, expect: 123456789123456, + }, + { + name: "1234567891234567", + input: `1234567891234567`, expect: 1234567891234567, + }, + { + name: "12345678912345678", + input: `12345678912345678`, expect: 12345678912345678, + }, + { + name: "123456789123456789", + input: `123456789123456789`, expect: 123456789123456789, + }, + { + name: "1234567891234567891", + input: `1234567891234567891`, expect: 1234567891234567891, + }, + + {name: "int32_max", input: `2147483647`, expect: math.MaxInt32}, + {name: "uint64_max", input: `18446744073709551615`, expect: math.MaxUint64}, + + {name: "overflow_hi", input: `18446744073709551616`, overflow: true}, + {name: "overflow_hi2", input: `22222222222222222222`, overflow: true}, + {name: "overflow_l21", input: `222222222222222222222`, overflow: true}, + } { + t.Run(td.name, func(t *testing.T) { + a, overflow := atoi.U64(td.input) + if td.overflow { + require.True(t, overflow) + require.Zero(t, a) + } else { + require.False(t, overflow) + require.Equal(t, td.expect, a) + } + }) + } +} + +func TestI64(t *testing.T) { + for _, td := range []struct { + name string + input string + expect int64 + err bool + }{ + {name: "0", input: `0`, expect: 0}, + {name: "1", input: `1`, expect: 1}, + {name: "12", input: `12`, expect: 12}, + {name: "123", input: `123`, expect: 123}, + {name: "1234", input: `1234`, expect: 1234}, + {name: "12345", input: `12345`, expect: 12345}, + {name: "123456", input: `123456`, expect: 123456}, + {name: "1234567", input: `1234567`, expect: 1234567}, + {name: "12345678", input: `12345678`, expect: 12345678}, + {name: "123456789", input: `123456789`, expect: 123456789}, + {name: "1234567891", input: `1234567891`, expect: 1234567891}, + {name: "12345678912", input: `12345678912`, expect: 12345678912}, + {name: "123456789123", input: `123456789123`, expect: 123456789123}, + {name: "1234567891234", input: `1234567891234`, expect: 1234567891234}, + {name: "12345678912345", input: `12345678912345`, expect: 12345678912345}, + { + name: "123456789123456", + input: `123456789123456`, expect: 123456789123456, + }, + { + name: "1234567891234567", + input: `1234567891234567`, expect: 1234567891234567, + }, + { + name: "12345678912345678", + input: `12345678912345678`, expect: 12345678912345678, + }, + { + name: "123456789123456789", + input: `123456789123456789`, expect: 123456789123456789, + }, + { + name: "1234567891234567891", + input: `1234567891234567891`, expect: 1234567891234567891, + }, + + {name: "-1", input: `-1`, expect: -1}, + {name: "-12", input: `-12`, expect: -12}, + {name: "-123", input: `-123`, expect: -123}, + {name: "-1234", input: `-1234`, expect: -1234}, + {name: "-12345", input: `-12345`, expect: -12345}, + {name: "-123456", input: `-123456`, expect: -123456}, + {name: "-1234567", input: `-1234567`, expect: -1234567}, + {name: "-12345678", input: `-12345678`, expect: -12345678}, + {name: "-123456789", input: `-123456789`, expect: -123456789}, + {name: "-1234567891", input: `-1234567891`, expect: -1234567891}, + {name: "-12345678912", input: `-12345678912`, expect: -12345678912}, + {name: "-123456789123", input: `-123456789123`, expect: -123456789123}, + {name: "-1234567891234", input: `-1234567891234`, expect: -1234567891234}, + { + name: "-12345678912345", + input: `-12345678912345`, expect: -12345678912345, + }, + { + name: "-123456789123456", + input: `-123456789123456`, expect: -123456789123456, + }, + { + name: "-1234567891234567", + input: `-1234567891234567`, expect: -1234567891234567, + }, + { + name: "-12345678912345678", + input: `-12345678912345678`, expect: -12345678912345678, + }, + { + name: "-123456789123456789", + input: `-123456789123456789`, expect: -123456789123456789, + }, + { + name: "-1234567891234567891", + input: `-1234567891234567891`, expect: -1234567891234567891, + }, + + {name: "int32_min", input: `-2147483648`, expect: math.MinInt32}, + {name: "int32_max", input: `2147483647`, expect: math.MaxInt32}, + {name: "int64_min", input: `-9223372036854775808`, expect: math.MinInt64}, + {name: "int64_max", input: `9223372036854775807`, expect: math.MaxInt64}, + + {name: "overflow_lo", input: `-9223372036854775809`, err: true}, + {name: "overflow_hi", input: `9223372036854775808`, err: true}, + {name: "overflow_l20_neg", input: `-11111111111111111111`, err: true}, + {name: "overflow_l20_pos", input: `11111111111111111111`, err: true}, + } { + t.Run(td.name, func(t *testing.T) { + a, overflow := atoi.I64(td.input) + if td.err { + require.True(t, overflow) + require.Zero(t, a) + } else { + require.False(t, overflow) + require.Equal(t, td.expect, a) + } + }) + } +} + +func TestU32(t *testing.T) { + for _, td := range []struct { + name string + input string + expect uint32 + err bool + }{ + {name: "0", input: `0`, expect: 0}, + {name: "1", input: `1`, expect: 1}, + {name: "12", input: `12`, expect: 12}, + {name: "123", input: `123`, expect: 123}, + {name: "1234", input: `1234`, expect: 1234}, + {name: "12345", input: `12345`, expect: 12345}, + {name: "123456", input: `123456`, expect: 123456}, + {name: "1234567", input: `1234567`, expect: 1234567}, + {name: "12345678", input: `12345678`, expect: 12345678}, + {name: "123456789", input: `123456789`, expect: 123456789}, + {name: "1234567891", input: `1234567891`, expect: 1234567891}, + {name: "uint32_max", input: `4294967295`, expect: math.MaxUint32}, + + {name: "overflow_hi", input: `4294967296`, err: true}, + {name: "overflow_hi1", input: `4333333333`, err: true}, + {name: "overflow_hi2", input: `5555555555`, err: true}, + {name: "overflow_hi3", input: `6666666666`, err: true}, + {name: "overflow_hi4", input: `7777777777`, err: true}, + {name: "overflow_hi5", input: `8888888888`, err: true}, + {name: "overflow_hi6", input: `9999999999`, err: true}, + {name: "overflow_l11", input: `11111111111`, err: true}, + } { + t.Run(td.name, func(t *testing.T) { + a, overflow := atoi.U32(td.input) + if td.err { + require.True(t, overflow) + require.Zero(t, a) + } else { + require.False(t, overflow) + require.Equal(t, td.expect, a) + } + }) + } +} + +func TestI32(t *testing.T) { + for _, td := range []struct { + name string + input string + expect int32 + err bool + }{ + {name: "0", input: `0`, expect: 0}, + {name: "1", input: `1`, expect: 1}, + {name: "12", input: `12`, expect: 12}, + {name: "123", input: `123`, expect: 123}, + {name: "1234", input: `1234`, expect: 1234}, + {name: "12345", input: `12345`, expect: 12345}, + {name: "123456", input: `123456`, expect: 123456}, + {name: "1234567", input: `1234567`, expect: 1234567}, + {name: "12345678", input: `12345678`, expect: 12345678}, + {name: "123456789", input: `123456789`, expect: 123456789}, + {name: "1234567891", input: `1234567891`, expect: 1234567891}, + + {name: "-1", input: `-1`, expect: -1}, + {name: "-12", input: `-12`, expect: -12}, + {name: "-123", input: `-123`, expect: -123}, + {name: "-1234", input: `-1234`, expect: -1234}, + {name: "-12345", input: `-12345`, expect: -12345}, + {name: "-123456", input: `-123456`, expect: -123456}, + {name: "-1234567", input: `-1234567`, expect: -1234567}, + {name: "-12345678", input: `-12345678`, expect: -12345678}, + {name: "-123456789", input: `-123456789`, expect: -123456789}, + {name: "-1234567891", input: `-1234567891`, expect: -1234567891}, + + {name: "int32_min", input: `-2147483648`, expect: math.MinInt32}, + {name: "int32_max", input: `2147483647`, expect: math.MaxInt32}, + + {name: "overflow_int32_lo", input: `-2147483649`, err: true}, + {name: "overflow_int32_hi", input: `2147483648`, err: true}, + {name: "overflow_int64_min", input: `-9223372036854775808`, err: true}, + {name: "overflow_int64_max", input: `9223372036854775807`, err: true}, + {name: "overflow_int64_lo", input: `-9223372036854775809`, err: true}, + {name: "overflow_int64_hi", input: `9223372036854775808`, err: true}, + {name: "overflow_l20_neg", input: `-11111111111111111111`, err: true}, + {name: "overflow_l20_pos", input: `11111111111111111111`, err: true}, + } { + t.Run(td.name, func(t *testing.T) { + a, overflow := atoi.I32(td.input) + if td.err { + require.True(t, overflow) + require.Zero(t, a) + } else { + require.False(t, overflow) + require.Equal(t, td.expect, a) + } + }) + } +} + +func TestI8(t *testing.T) { + for _, td := range []struct { + name string + input string + expect int8 + err bool + }{ + {name: "0", input: `0`, expect: 0}, + {name: "1", input: `1`, expect: 1}, + {name: "12", input: `12`, expect: 12}, + {name: "123", input: `123`, expect: 123}, + {name: "-1", input: `-1`, expect: -1}, + {name: "-12", input: `-12`, expect: -12}, + {name: "-123", input: `-123`, expect: -123}, + + {name: "int8_min", input: `-128`, expect: math.MinInt8}, + {name: "int8_max", input: `127`, expect: math.MaxInt8}, + + {name: "overflow_lo", input: `-129`, err: true}, + {name: "overflow_hi", input: `128`, err: true}, + {name: "overflow_int32_lo", input: `-2147483649`, err: true}, + {name: "overflow_int32_hi", input: `2147483648`, err: true}, + {name: "overflow_int64_min", input: `-9223372036854775808`, err: true}, + {name: "overflow_int64_max", input: `9223372036854775807`, err: true}, + {name: "overflow_int64_lo", input: `-9223372036854775809`, err: true}, + {name: "overflow_int64_hi", input: `9223372036854775808`, err: true}, + } { + t.Run(td.name, func(t *testing.T) { + a, overflow := atoi.I8(td.input) + if td.err { + require.True(t, overflow) + require.Zero(t, a) + } else { + require.False(t, overflow) + require.Equal(t, td.expect, a) + } + }) + } +} + +func TestU8(t *testing.T) { + for _, td := range []struct { + name string + input string + expect uint8 + err bool + }{ + {name: "0", input: `0`, expect: 0}, + {name: "1", input: `1`, expect: 1}, + {name: "12", input: `12`, expect: 12}, + {name: "123", input: `123`, expect: 123}, + {name: "max_uint8", input: `255`, expect: 255}, + + {name: "overflow_hi", input: `256`, err: true}, + {name: "overflow_hi1", input: `300`, err: true}, + {name: "overflow_hi1", input: `999`, err: true}, + {name: "overflow_l4", input: `1111`, err: true}, + } { + t.Run(td.name, func(t *testing.T) { + a, overflow := atoi.U8(td.input) + if td.err { + require.True(t, overflow) + require.Zero(t, a) + } else { + require.False(t, overflow) + require.Equal(t, td.expect, a) + } + }) + } +} + +func TestI16(t *testing.T) { + for _, td := range []struct { + name string + input string + expect int16 + err bool + }{ + {name: "0", input: `0`, expect: 0}, + {name: "1", input: `1`, expect: 1}, + {name: "12", input: `12`, expect: 12}, + {name: "123", input: `123`, expect: 123}, + {name: "1234", input: `1234`, expect: 1234}, + {name: "12345", input: `12345`, expect: 12345}, + {name: "-1", input: `-1`, expect: -1}, + {name: "-12", input: `-12`, expect: -12}, + {name: "-123", input: `-123`, expect: -123}, + {name: "-1234", input: `-1234`, expect: -1234}, + {name: "-12345", input: `-12345`, expect: -12345}, + + {name: "int16_min", input: `-32768`, expect: math.MinInt16}, + {name: "int16_max", input: `32767`, expect: math.MaxInt16}, + + {name: "overflow_lo", input: `-32769`, err: true}, + {name: "overflow_hi", input: `32768`, err: true}, + {name: "overflow_int32_lo", input: `-2147483649`, err: true}, + {name: "overflow_int32_hi", input: `2147483648`, err: true}, + {name: "overflow_int64_min", input: `-9223372036854775808`, err: true}, + {name: "overflow_int64_max", input: `9223372036854775807`, err: true}, + {name: "overflow_int64_lo", input: `-9223372036854775809`, err: true}, + {name: "overflow_int64_hi", input: `9223372036854775808`, err: true}, + } { + t.Run(td.name, func(t *testing.T) { + a, overflow := atoi.I16(td.input) + if td.err { + require.True(t, overflow) + require.Zero(t, a) + } else { + require.False(t, overflow) + require.Equal(t, td.expect, a) + } + }) + } +} + +func TestU16(t *testing.T) { + for _, td := range []struct { + name string + input string + expect uint16 + err bool + }{ + {name: "0", input: `0`, expect: 0}, + {name: "1", input: `1`, expect: 1}, + {name: "12", input: `12`, expect: 12}, + {name: "123", input: `123`, expect: 123}, + {name: "1234", input: `1234`, expect: 1234}, + {name: "12345", input: `12345`, expect: 12345}, + {name: "uint16_max", input: `65535`, expect: math.MaxUint16}, + + {name: "overflow_hi", input: `65536`, err: true}, + {name: "overflow_hi1", input: `77777`, err: true}, + {name: "overflow_int32_hi", input: `2147483648`, err: true}, + {name: "overflow_int64_max", input: `9223372036854775807`, err: true}, + } { + t.Run(td.name, func(t *testing.T) { + a, overflow := atoi.U16(td.input) + if td.err { + require.True(t, overflow) + require.Zero(t, a) + } else { + require.False(t, overflow) + require.Equal(t, td.expect, a) + } + }) + } +} + +func BenchmarkAtoi64(b *testing.B) { + for _, bd := range []struct { + name string + input string + }{ + {name: "min_int64", input: `-9223372036854775808`}, + {name: "max_int64", input: `9223372036854775807`}, + {name: "1to9_____", input: `123456789`}, + {name: "neg_one__", input: `-1`}, + {name: "zero_____", input: `0`}, + } { + b.Run(bd.name, func(b *testing.B) { + var x int + var x64 int64 + b.Run("strconv_atoi", func(b *testing.B) { + for n := 0; n < b.N; n++ { + var err error + if x, err = strconv.Atoi(bd.input); err != nil { + b.Fatal(err) + } + } + }) + b.Run("strconv_parse", func(b *testing.B) { + for n := 0; n < b.N; n++ { + var err error + if x64, err = strconv.ParseInt(bd.input, 10, 0); err != nil { + b.Fatal(err) + } + } + }) + b.Run("I64", func(b *testing.B) { + for n := 0; n < b.N; n++ { + var overflow bool + if x64, overflow = atoi.I64(bd.input); overflow { + b.Fatal("overflow") + } + } + }) + runtime.KeepAlive(x) + runtime.KeepAlive(x64) + }) + } +} + +func BenchmarkI8(b *testing.B) { + for _, bd := range []struct { + name string + input string + }{ + {name: "min_int8", input: `-128`}, + {name: "max_int8", input: `127`}, + {name: "-1______", input: `-1`}, + {name: "zero____", input: `0`}, + } { + b.Run(bd.name, func(b *testing.B) { + var x int + var x8 int8 + var x64 int64 + b.Run("strconv_atoi", func(b *testing.B) { + for n := 0; n < b.N; n++ { + var err error + if x, err = strconv.Atoi(bd.input); err != nil { + b.Fatal(err) + } + } + }) + b.Run("strconv_parse", func(b *testing.B) { + for n := 0; n < b.N; n++ { + var err error + if x64, err = strconv.ParseInt(bd.input, 10, 16); err != nil { + b.Fatal(err) + } + } + }) + b.Run("I16", func(b *testing.B) { + for n := 0; n < b.N; n++ { + var overflow bool + if x8, overflow = atoi.I8(bd.input); overflow { + b.Fatal("overflow") + } + } + }) + runtime.KeepAlive(x) + runtime.KeepAlive(x8) + runtime.KeepAlive(x64) + }) + } +} + +func BenchmarkU8(b *testing.B) { + for _, bd := range []struct { + name string + input string + }{ + {name: "max_uint8", input: `255`}, + {name: "123______", input: `123`}, + {name: "zero_____", input: `0`}, + } { + b.Run(bd.name, func(b *testing.B) { + var x int + var x8 uint8 + var x64 uint64 + b.Run("strconv_atoi", func(b *testing.B) { + for n := 0; n < b.N; n++ { + var err error + if x, err = strconv.Atoi(bd.input); err != nil { + b.Fatal(err) + } + } + }) + b.Run("strconv_parse", func(b *testing.B) { + for n := 0; n < b.N; n++ { + var err error + if x64, err = strconv.ParseUint(bd.input, 10, 16); err != nil { + b.Fatal(err) + } + } + }) + b.Run("U8", func(b *testing.B) { + for n := 0; n < b.N; n++ { + var overflow bool + if x8, overflow = atoi.U8(bd.input); overflow { + b.Fatal("overflow") + } + } + }) + runtime.KeepAlive(x) + runtime.KeepAlive(x8) + runtime.KeepAlive(x64) + }) + } +} + +func BenchmarkI16(b *testing.B) { + for _, bd := range []struct { + name string + input string + }{ + {name: "min_int16", input: `-32768`}, + {name: "max_int16", input: `32767`}, + {name: "-1_______", input: `-1`}, + {name: "zero_____", input: `0`}, + } { + b.Run(bd.name, func(b *testing.B) { + var x int + var x16 int16 + var x64 int64 + b.Run("strconv_atoi", func(b *testing.B) { + for n := 0; n < b.N; n++ { + var err error + if x, err = strconv.Atoi(bd.input); err != nil { + b.Fatal(err) + } + } + }) + b.Run("strconv_parse", func(b *testing.B) { + for n := 0; n < b.N; n++ { + var err error + if x64, err = strconv.ParseInt(bd.input, 10, 16); err != nil { + b.Fatal(err) + } + } + }) + b.Run("I16", func(b *testing.B) { + for n := 0; n < b.N; n++ { + var overflow bool + if x16, overflow = atoi.I16(bd.input); overflow { + b.Fatal("overflow") + } + } + }) + runtime.KeepAlive(x) + runtime.KeepAlive(x16) + runtime.KeepAlive(x64) + }) + } +} + +func BenchmarkU16(b *testing.B) { + for _, bd := range []struct { + name string + input string + }{ + {name: "max_int16", input: `65535`}, + {name: "32767____s", input: `32767`}, + {name: "zero_____", input: `0`}, + } { + b.Run(bd.name, func(b *testing.B) { + var x int + var x16 uint16 + var x64 uint64 + b.Run("strconv_atoi", func(b *testing.B) { + for n := 0; n < b.N; n++ { + var err error + if x, err = strconv.Atoi(bd.input); err != nil { + b.Fatal(err) + } + } + }) + b.Run("strconv_parse", func(b *testing.B) { + for n := 0; n < b.N; n++ { + var err error + if x64, err = strconv.ParseUint(bd.input, 10, 16); err != nil { + b.Fatal(err) + } + } + }) + b.Run("U16", func(b *testing.B) { + for n := 0; n < b.N; n++ { + var overflow bool + if x16, overflow = atoi.U16(bd.input); overflow { + b.Fatal("overflow") + } + } + }) + runtime.KeepAlive(x) + runtime.KeepAlive(x16) + runtime.KeepAlive(x64) + }) + } +} + +func BenchmarkU16Overflow(b *testing.B) { + var x uint16 + var x64 uint64 + var err error + inputOverflow := "65536" + b.Run("U16", func(b *testing.B) { + for n := 0; n < b.N; n++ { + var overflow bool + if x, overflow = atoi.U16(inputOverflow); !overflow { + panic(err) + } + } + }) + b.Run("strconv", func(b *testing.B) { + for n := 0; n < b.N; n++ { + if x64, err = strconv.ParseUint(inputOverflow, 10, 16); err == nil { + panic(err) + } + } + }) + runtime.KeepAlive(x) + runtime.KeepAlive(x64) + runtime.KeepAlive(err) +} diff --git a/internal/atoi/doc.go b/internal/atoi/doc.go new file mode 100644 index 0000000..8427f7a --- /dev/null +++ b/internal/atoi/doc.go @@ -0,0 +1,8 @@ +// Package atoi provides integer parsing functions optimized specifically for +// jscan since the parser validates the input before the integer parser is invoked. +// +// WARNING: These functions are not a replacement for strconv.ParseInt or strconv.Atoi +// because they don't validate the input and assumes valid input instead. +// The jscan tokenizer is guaranteed to provide only valid values which +// are only not guaranteed to not overflow. +package atoi diff --git a/jscan_test.go b/jscan_test.go index 4a3c6bf..8b649b4 100644 --- a/jscan_test.go +++ b/jscan_test.go @@ -3,8 +3,10 @@ package jscan_test import ( "encoding/json" "fmt" + "math" "os" "path/filepath" + "strconv" "strings" "testing" @@ -82,7 +84,7 @@ func testStrictOK[S ~string | ~[]byte](t *testing.T, input S) { k := jscan.NewTokenizer[string](64, 128) err := k.Tokenize( string(input), - func(tokens []jscan.Token) (err bool) { return false }, + func(tokens []jscan.Token[string]) (err bool) { return false }, ) require.False(t, err.IsErr()) }) @@ -90,7 +92,7 @@ func testStrictOK[S ~string | ~[]byte](t *testing.T, input S) { k := jscan.NewTokenizer[string](64, 128) _, err := k.TokenizeOne( string(input), - func(tokens []jscan.Token) (err bool) { return false }, + func(tokens []jscan.Token[string]) (err bool) { return false }, ) require.False(t, err.IsErr()) }) @@ -126,7 +128,7 @@ func testOKOrErr[S ~string | ~[]byte](t *testing.T, input S) { }) t.Run("TokenizerTokenize", func(t *testing.T) { v := jscan.NewTokenizer[string](1024, 4*1024) - err := v.Tokenize(string(input), func(tokens []jscan.Token) (err bool) { + err := v.Tokenize(string(input), func(tokens []jscan.Token[string]) (err bool) { return false }) if err.IsErr() { @@ -170,7 +172,7 @@ func testStrictErr[S ~string | ~[]byte](t *testing.T, input S) { }) t.Run("TokenizerTokenize", func(t *testing.T) { err := jscan.NewTokenizer[S](1024, 4*1024).Tokenize( - input, func(tokens []jscan.Token) (err bool) { return false }, + input, func(tokens []jscan.Token[S]) (err bool) { return false }, ) require.True(t, err.IsErr()) require.NotEqual(t, err.Code, jscan.ErrorCodeCallback) @@ -187,15 +189,15 @@ type Record struct { Pointer string } -type ScanTest struct { +type ScanTest[S ~string | ~[]byte] struct { name string - input string + input S expect []Record - expectTokens []jscan.Token + expectTokens []jscan.Token[S] } func TestParsingValid(t *testing.T) { - for _, td := range []ScanTest{ + for _, td := range []ScanTest[string]{ { name: "null", input: "null", @@ -206,7 +208,7 @@ func TestParsingValid(t *testing.T) { Value: "null", }, }, - expectTokens: []jscan.Token{ + expectTokens: []jscan.Token[string]{ {Type: jscan.TokenTypeNull, Index: 0, End: 4}, }, }, @@ -220,7 +222,7 @@ func TestParsingValid(t *testing.T) { Value: "true", }, }, - expectTokens: []jscan.Token{ + expectTokens: []jscan.Token[string]{ {Type: jscan.TokenTypeTrue, Index: 0, End: 4}, }, }, @@ -234,7 +236,7 @@ func TestParsingValid(t *testing.T) { Value: "false", }, }, - expectTokens: []jscan.Token{ + expectTokens: []jscan.Token[string]{ {Type: jscan.TokenTypeFalse, Index: 0, End: 5}, }, }, @@ -248,7 +250,7 @@ func TestParsingValid(t *testing.T) { ArrayIndex: -1, }, }, - expectTokens: []jscan.Token{ + expectTokens: []jscan.Token[string]{ {Type: jscan.TokenTypeInteger, Index: 0, End: 2}, }, }, @@ -262,7 +264,7 @@ func TestParsingValid(t *testing.T) { ArrayIndex: -1, }, }, - expectTokens: []jscan.Token{ + expectTokens: []jscan.Token[string]{ {Type: jscan.TokenTypeNumber, Index: 0, End: 4}, }, }, @@ -276,7 +278,7 @@ func TestParsingValid(t *testing.T) { ArrayIndex: -1, }, }, - expectTokens: []jscan.Token{ + expectTokens: []jscan.Token[string]{ {Type: jscan.TokenTypeNumber, Index: 0, End: 5}, }, }, @@ -290,7 +292,7 @@ func TestParsingValid(t *testing.T) { ArrayIndex: -1, }, }, - expectTokens: []jscan.Token{ + expectTokens: []jscan.Token[string]{ {Type: jscan.TokenTypeNumber, Index: 0, End: 12}, }, }, @@ -304,7 +306,7 @@ func TestParsingValid(t *testing.T) { ArrayIndex: -1, }, }, - expectTokens: []jscan.Token{ + expectTokens: []jscan.Token[string]{ {Type: jscan.TokenTypeString, Index: 0, End: 4}, }, }, @@ -318,7 +320,7 @@ func TestParsingValid(t *testing.T) { ArrayIndex: -1, }, }, - expectTokens: []jscan.Token{ + expectTokens: []jscan.Token[string]{ {Type: jscan.TokenTypeString, Index: 0, End: len(`"жш\"ц\\\\\""`)}, }, }, @@ -331,7 +333,7 @@ func TestParsingValid(t *testing.T) { ArrayIndex: -1, }, }, - expectTokens: []jscan.Token{ + expectTokens: []jscan.Token[string]{ {Type: jscan.TokenTypeArray, Index: 0, End: 1}, {Type: jscan.TokenTypeArrayEnd, Index: 1, End: 0}, }, @@ -345,7 +347,7 @@ func TestParsingValid(t *testing.T) { ArrayIndex: -1, }, }, - expectTokens: []jscan.Token{ + expectTokens: []jscan.Token[string]{ {Type: jscan.TokenTypeObject, Index: 0, End: 1}, {Type: jscan.TokenTypeObjectEnd, Index: 1, End: 0}, }, @@ -398,7 +400,7 @@ func TestParsingValid(t *testing.T) { Pointer: "/1", }, }, - expectTokens: []jscan.Token{ + expectTokens: []jscan.Token[string]{ {Type: jscan.TokenTypeArray, Index: 0, Elements: 2, End: 12}, // <────┐ 0 {Type: jscan.TokenTypeArray, Index: 1, Elements: 2, End: 9}, // <───┐│ 1 {Type: jscan.TokenTypeNull, Index: 2, End: 6}, // ││ 2 @@ -431,7 +433,7 @@ func TestParsingValid(t *testing.T) { Pointer: `/\\`, }, }, - expectTokens: []jscan.Token{ + expectTokens: []jscan.Token[string]{ {Type: jscan.TokenTypeObject, Index: 0, Elements: 1, End: 3}, // <─┐ 0 {Type: jscan.TokenTypeKey, Index: 1, End: 5}, // │ 1 {Type: jscan.TokenTypeNull, Index: 6, End: 10}, // │ 2 @@ -455,7 +457,7 @@ func TestParsingValid(t *testing.T) { Pointer: `/\"`, }, }, - expectTokens: []jscan.Token{ + expectTokens: []jscan.Token[string]{ {Type: jscan.TokenTypeObject, Index: 0, Elements: 1, End: 3}, // <─┐ 0 {Type: jscan.TokenTypeKey, Index: 1, End: 5}, // │ 1 {Type: jscan.TokenTypeNull, Index: 6, End: 10}, // │ 2 @@ -499,7 +501,7 @@ func TestParsingValid(t *testing.T) { Pointer: `/~1/1`, }, }, - expectTokens: []jscan.Token{ + expectTokens: []jscan.Token[string]{ {Type: jscan.TokenTypeObject, Index: 0, Elements: 1, End: 9}, // <───┐ 0 {Type: jscan.TokenTypeKey, Index: 1, End: 4}, // │ 1 {Type: jscan.TokenTypeArray, Index: 5, Elements: 2, End: 8}, // <──┐│ 2 @@ -693,7 +695,7 @@ func TestParsingValid(t *testing.T) { Pointer: "/a3/1/a3", }, }, - expectTokens: []jscan.Token{ + expectTokens: []jscan.Token[string]{ {Type: jscan.TokenTypeObject, Index: 0, Elements: 9, End: 41}, // <───┐ 0 {Type: jscan.TokenTypeKey, Index: 6, End: 9}, // │ 1 {Type: jscan.TokenTypeString, Index: 11, End: 18}, // │ 2 @@ -748,7 +750,7 @@ func TestParsingValid(t *testing.T) { Value: "null", }, }, - expectTokens: []jscan.Token{ + expectTokens: []jscan.Token[string]{ {Type: jscan.TokenTypeNull, Index: 0, End: 4}, }, }, @@ -762,7 +764,7 @@ func TestParsingValid(t *testing.T) { Value: "null", }, }, - expectTokens: []jscan.Token{ + expectTokens: []jscan.Token[string]{ {Type: jscan.TokenTypeNull, Index: 0, End: 4}, }, }, @@ -776,7 +778,7 @@ func TestParsingValid(t *testing.T) { Value: "null", }, }, - expectTokens: []jscan.Token{ + expectTokens: []jscan.Token[string]{ {Type: jscan.TokenTypeNull, Index: 0, End: 4}, }, }, @@ -790,7 +792,7 @@ func TestParsingValid(t *testing.T) { Value: "null", }, }, - expectTokens: []jscan.Token{ + expectTokens: []jscan.Token[string]{ {Type: jscan.TokenTypeNull, Index: 0, End: 4}, }, }, @@ -798,12 +800,11 @@ func TestParsingValid(t *testing.T) { t.Run(td.name, func(t *testing.T) { require.True(t, json.Valid([]byte(td.input))) testParsingValid[string](t, td) - testParsingValid[[]byte](t, td) }) } } -func testParsingValid[S ~string | ~[]byte](t *testing.T, td ScanTest) { +func testParsingValid[S ~string | ~[]byte](t *testing.T, td ScanTest[S]) { t.Run(testDataType(td.input), func(t *testing.T) { j := 0 check := func(t *testing.T) func(i *jscan.Iterator[S]) bool { @@ -858,19 +859,19 @@ func testParsingValid[S ~string | ~[]byte](t *testing.T, td ScanTest) { }) t.Run("TokenizerTokenize", func(t *testing.T) { k := jscan.NewTokenizer[S](64, 1024) - var cp []jscan.Token - err := k.Tokenize(S(td.input), func(tokens []jscan.Token) (err bool) { - cp = make([]jscan.Token, len(tokens)) + var cp []jscan.Token[S] + err := k.Tokenize(S(td.input), func(tokens []jscan.Token[S]) (err bool) { + cp = make([]jscan.Token[S], len(tokens)) copy(cp, tokens) return false }) require.False(t, err.IsErr(), "unexpected error: %s", err) - compareTokens(t, td.expectTokens, cp) + compareTokens[S](t, td.expectTokens, cp) }) }) } -func compareTokens(t *testing.T, expected, actual []jscan.Token) { +func compareTokens[S ~string | ~[]byte](t *testing.T, expected, actual []jscan.Token[S]) { t.Helper() assert.Len(t, actual, len(expected)) for i, e := range expected { @@ -1274,7 +1275,7 @@ func testError[S ~string | ~[]byte](t *testing.T, td ErrorTest) { k := jscan.NewTokenizer[S](64, 128) err := k.Tokenize( S(td.input), - func(tokens []jscan.Token) (err bool) { return false }, + func(tokens []jscan.Token[S]) (err bool) { return false }, ) require.Equal(t, td.expect, err.Error()) require.True(t, err.IsErr()) @@ -1503,7 +1504,7 @@ func testControlCharacters[S ~string | ~[]byte](t *testing.T, input S, expectErr k := jscan.NewTokenizer[S](64, 128) err := k.Tokenize( S(input), - func(tokens []jscan.Token) (err bool) { return false }, + func(tokens []jscan.Token[S]) (err bool) { return false }, ) require.Equal(t, expectErr, err.Error()) require.True(t, err.IsErr()) @@ -1513,7 +1514,7 @@ func testControlCharacters[S ~string | ~[]byte](t *testing.T, input S, expectErr k := jscan.NewTokenizer[S](64, 128) _, err := k.TokenizeOne( S(input), - func(tokens []jscan.Token) (err bool) { return false }, + func(tokens []jscan.Token[S]) (err bool) { return false }, ) require.Equal(t, expectErr, err.Error()) require.True(t, err.IsErr()) @@ -1771,7 +1772,7 @@ func testStrings[S ~string | ~[]byte](t *testing.T, input S) { t.Run("TokenizerTokenize", func(t *testing.T) { k := jscan.NewTokenizer[S](64, 1024) c := 0 - err := k.Tokenize(input, func(tokens []jscan.Token) (err bool) { + err := k.Tokenize(input, func(tokens []jscan.Token[S]) (err bool) { require.Len(t, tokens, 1) require.Equal(t, 0, tokens[0].Index) require.Equal(t, jscan.TokenTypeString, tokens[0].Type) @@ -1785,7 +1786,7 @@ func testStrings[S ~string | ~[]byte](t *testing.T, input S) { t.Run("TokenizerTokenizeOne", func(t *testing.T) { k := jscan.NewTokenizer[S](64, 1024) c := 0 - tail, err := k.TokenizeOne(input, func(tokens []jscan.Token) (err bool) { + tail, err := k.TokenizeOne(input, func(tokens []jscan.Token[S]) (err bool) { require.Len(t, tokens, 1) require.Equal(t, 0, tokens[0].Index) require.Equal(t, jscan.TokenTypeString, tokens[0].Type) @@ -1895,3 +1896,671 @@ func TestDerivedTypes(t *testing.T) { }) }) } + +func TestTokenBool(t *testing.T) { + src := `null` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Bool(src) + require.NoError(t, err) + require.Zero(t, v) + }) + + src = `true` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Bool(src) + require.NoError(t, err) + require.Equal(t, true, v) + }) + + src = `false` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Bool(src) + require.NoError(t, err) + require.Equal(t, false, v) + }) + + src = `42` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Bool(src) + require.ErrorIs(t, err, jscan.ErrWrongType) + require.Zero(t, v) + }) +} + +func TestTokenString(t *testing.T) { + src := `null` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.String(src) + require.NoError(t, err) + require.Zero(t, v) + }) + + src = `"text"` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.String(src) + require.NoError(t, err) + require.Equal(t, "text", v) + }) + + src = `""` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.String(src) + require.NoError(t, err) + require.Equal(t, "", v) + }) + + src = `42` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.String(src) + require.ErrorIs(t, err, jscan.ErrWrongType) + require.Zero(t, v) + }) +} + +func TestTokenFloat32(t *testing.T) { + src := `null` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Float32(src) + require.NoError(t, err) + require.Zero(t, v) + }) + + src = `3.1415` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Float32(src) + require.NoError(t, err) + require.Equal(t, float32(3.1415), v) + }) + + src = `0` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Float32(src) + require.NoError(t, err) + require.Equal(t, float32(0), v) + }) + + src = `42` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Float32(src) + require.NoError(t, err) + require.Equal(t, float32(42), v) + }) + + src = `-42.0` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Float32(src) + require.NoError(t, err) + require.Equal(t, float32(-42), v) + }) + + src = `123456e123456` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Float32(src) + require.ErrorIs(t, err, strconv.ErrRange) + require.Zero(t, v) + }) + + src = `false` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Float32(src) + require.ErrorIs(t, err, jscan.ErrWrongType) + require.Zero(t, v) + }) + + { // Bytes + src := []byte(`3.1415`) + testValueToken(t, src, func(t *testing.T, token jscan.Token[[]byte]) { + v, err := token.Float32(src) + require.NoError(t, err) + require.Equal(t, float32(3.1415), v) + }) + } +} + +func TestTokenFloat64(t *testing.T) { + src := `null` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Float64(src) + require.NoError(t, err) + require.Zero(t, v) + }) + + src = `3.1415` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Float64(src) + require.NoError(t, err) + require.Equal(t, float64(3.1415), v) + }) + + src = `0` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Float64(src) + require.NoError(t, err) + require.Equal(t, float64(0), v) + }) + + src = `42` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Float64(src) + require.NoError(t, err) + require.Equal(t, float64(42), v) + }) + + src = `-42.0` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Float64(src) + require.NoError(t, err) + require.Equal(t, float64(-42), v) + }) + + src = `123456e123456` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Float64(src) + require.ErrorIs(t, err, strconv.ErrRange) + require.Zero(t, v) + }) + + src = `false` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Float64(src) + require.ErrorIs(t, err, jscan.ErrWrongType) + require.Zero(t, v) + }) + + { // Bytes + src := []byte(`3.1415`) + testValueToken(t, src, func(t *testing.T, token jscan.Token[[]byte]) { + v, err := token.Float64(src) + require.NoError(t, err) + require.Equal(t, float64(3.1415), v) + }) + } +} + +func TestTokenInt(t *testing.T) { + src := `null` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Int(src) + require.NoError(t, err) + require.Zero(t, v) + }) + + src = `0` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Int(src) + require.NoError(t, err) + require.Equal(t, int(0), v) + }) + + src = `42` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Int(src) + require.NoError(t, err) + require.Equal(t, int(42), v) + }) + + src = `-42` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Int(src) + require.NoError(t, err) + require.Equal(t, int(-42), v) + }) + + src = `9223372036854775808` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Int(src) + require.ErrorIs(t, err, jscan.ErrOverflow) + require.Zero(t, v) + }) + + src = `-9223372036854775809` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Int(src) + require.ErrorIs(t, err, jscan.ErrOverflow) + require.Zero(t, v) + }) + + src = `3.14` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Int(src) + require.ErrorIs(t, err, jscan.ErrWrongType) + require.Zero(t, v) + }) +} + +func TestTokenInt8(t *testing.T) { + src := `null` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Int8(src) + require.NoError(t, err) + require.Zero(t, v) + }) + + src = `0` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Int8(src) + require.NoError(t, err) + require.Equal(t, int8(0), v) + }) + + src = `42` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Int8(src) + require.NoError(t, err) + require.Equal(t, int8(42), v) + }) + + src = `-42` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Int8(src) + require.NoError(t, err) + require.Equal(t, int8(-42), v) + }) + + src = fmt.Sprintf("%d", math.MaxInt8+1) + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Int8(src) + require.ErrorIs(t, err, jscan.ErrOverflow) + require.Zero(t, v) + }) + + src = fmt.Sprintf("%d", math.MinInt8-1) + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Int8(src) + require.ErrorIs(t, err, jscan.ErrOverflow) + require.Zero(t, v) + }) + + src = `3.14` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Int8(src) + require.ErrorIs(t, err, jscan.ErrWrongType) + require.Zero(t, v) + }) +} + +func TestTokenInt16(t *testing.T) { + src := `null` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Int16(src) + require.NoError(t, err) + require.Zero(t, v) + }) + + src = `0` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Int16(src) + require.NoError(t, err) + require.Equal(t, int16(0), v) + }) + + src = `42` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Int16(src) + require.NoError(t, err) + require.Equal(t, int16(42), v) + }) + + src = `-42` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Int16(src) + require.NoError(t, err) + require.Equal(t, int16(-42), v) + }) + + src = fmt.Sprintf("%d", math.MaxInt16+1) + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Int16(src) + require.ErrorIs(t, err, jscan.ErrOverflow) + require.Zero(t, v) + }) + + src = fmt.Sprintf("%d", math.MinInt16-1) + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Int16(src) + require.ErrorIs(t, err, jscan.ErrOverflow) + require.Zero(t, v) + }) + + src = `3.14` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Int16(src) + require.ErrorIs(t, err, jscan.ErrWrongType) + require.Zero(t, v) + }) +} + +func TestTokenInt32(t *testing.T) { + src := `null` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Int32(src) + require.NoError(t, err) + require.Zero(t, v) + }) + + src = `0` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Int32(src) + require.NoError(t, err) + require.Equal(t, int32(0), v) + }) + + src = `42` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Int32(src) + require.NoError(t, err) + require.Equal(t, int32(42), v) + }) + + src = `-42` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Int32(src) + require.NoError(t, err) + require.Equal(t, int32(-42), v) + }) + + src = fmt.Sprintf("%d", math.MaxInt32+1) + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Int32(src) + require.ErrorIs(t, err, jscan.ErrOverflow) + require.Zero(t, v) + }) + + src = fmt.Sprintf("%d", math.MinInt32-1) + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Int32(src) + require.ErrorIs(t, err, jscan.ErrOverflow) + require.Zero(t, v) + }) + + src = `3.14` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Int32(src) + require.ErrorIs(t, err, jscan.ErrWrongType) + require.Zero(t, v) + }) +} + +func TestTokenInt64(t *testing.T) { + src := `null` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Int64(src) + require.NoError(t, err) + require.Zero(t, v) + }) + + src = `0` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Int64(src) + require.NoError(t, err) + require.Equal(t, int64(0), v) + }) + + src = `42` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Int64(src) + require.NoError(t, err) + require.Equal(t, int64(42), v) + }) + + src = `-42` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Int64(src) + require.NoError(t, err) + require.Equal(t, int64(-42), v) + }) + + src = `9223372036854775808` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Int64(src) + require.ErrorIs(t, err, jscan.ErrOverflow) + require.Zero(t, v) + }) + + src = `-9223372036854775809` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Int64(src) + require.ErrorIs(t, err, jscan.ErrOverflow) + require.Zero(t, v) + }) + + src = `3.14` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Int64(src) + require.ErrorIs(t, err, jscan.ErrWrongType) + require.Zero(t, v) + }) +} + +func TestTokenUint(t *testing.T) { + src := `null` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Uint(src) + require.NoError(t, err) + require.Zero(t, v) + }) + + src = `0` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Uint(src) + require.NoError(t, err) + require.Equal(t, uint(0), v) + }) + + src = `42` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Uint(src) + require.NoError(t, err) + require.Equal(t, uint(42), v) + }) + + src = fmt.Sprintf("%d", math.MaxUint32) + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Uint(src) + require.NoError(t, err) + require.Equal(t, uint(math.MaxUint32), v) + }) + + src = `18446744073709551616` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Uint(src) + require.ErrorIs(t, err, jscan.ErrOverflow) + require.Zero(t, v) + }) + + src = `-42` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Uint(src) + require.ErrorIs(t, err, jscan.ErrWrongType) + require.Zero(t, v) + }) +} + +func TestTokenUint8(t *testing.T) { + src := `null` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Uint8(src) + require.NoError(t, err) + require.Zero(t, v) + }) + + src = `0` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Uint8(src) + require.NoError(t, err) + require.Equal(t, uint8(0), v) + }) + + src = `42` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Uint8(src) + require.NoError(t, err) + require.Equal(t, uint8(42), v) + }) + + src = fmt.Sprintf("%d", uint8(math.MaxUint8)) + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Uint8(src) + require.NoError(t, err) + require.Equal(t, uint8(math.MaxUint8), v) + }) + + src = fmt.Sprintf("%d", math.MaxUint8+1) + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Uint8(src) + require.ErrorIs(t, err, jscan.ErrOverflow) + require.Zero(t, v) + }) + + src = `-42` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Uint8(src) + require.ErrorIs(t, err, jscan.ErrWrongType) + require.Zero(t, v) + }) +} + +func TestTokenUint16(t *testing.T) { + src := `null` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Uint16(src) + require.NoError(t, err) + require.Zero(t, v) + }) + + src = `0` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Uint16(src) + require.NoError(t, err) + require.Equal(t, uint16(0), v) + }) + + src = `42` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Uint16(src) + require.NoError(t, err) + require.Equal(t, uint16(42), v) + }) + + src = fmt.Sprintf("%d", uint16(math.MaxUint16)) + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Uint16(src) + require.NoError(t, err) + require.Equal(t, uint16(math.MaxUint16), v) + }) + + src = fmt.Sprintf("%d", math.MaxUint16+1) + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Uint16(src) + require.ErrorIs(t, err, jscan.ErrOverflow) + require.Zero(t, v) + }) + + src = `-42` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Uint16(src) + require.ErrorIs(t, err, jscan.ErrWrongType) + require.Zero(t, v) + }) +} + +func TestTokenUint32(t *testing.T) { + src := `null` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Uint32(src) + require.NoError(t, err) + require.Zero(t, v) + }) + + src = `0` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Uint32(src) + require.NoError(t, err) + require.Equal(t, uint32(0), v) + }) + + src = `42` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Uint32(src) + require.NoError(t, err) + require.Equal(t, uint32(42), v) + }) + + src = fmt.Sprintf("%d", uint32(math.MaxUint32)) + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Uint32(src) + require.NoError(t, err) + require.Equal(t, uint32(math.MaxUint32), v) + }) + + src = fmt.Sprintf("%d", math.MaxUint32+1) + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Uint32(src) + require.ErrorIs(t, err, jscan.ErrOverflow) + require.Zero(t, v) + }) + + src = `-42` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Uint32(src) + require.ErrorIs(t, err, jscan.ErrWrongType) + require.Zero(t, v) + }) +} + +func TestTokenUint64(t *testing.T) { + src := `null` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Uint64(src) + require.NoError(t, err) + require.Zero(t, v) + }) + + src = `0` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Uint64(src) + require.NoError(t, err) + require.Equal(t, uint64(0), v) + }) + + src = `42` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Uint64(src) + require.NoError(t, err) + require.Equal(t, uint64(42), v) + }) + + src = fmt.Sprintf("%d", uint64(math.MaxUint64)) + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Uint64(src) + require.NoError(t, err) + require.Equal(t, uint64(math.MaxUint64), v) + }) + + src = `18446744073709551616` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Uint64(src) + require.ErrorIs(t, err, jscan.ErrOverflow) + require.Zero(t, v) + }) + + src = `-42` + testValueToken(t, src, func(t *testing.T, token jscan.Token[string]) { + v, err := token.Uint64(src) + require.ErrorIs(t, err, jscan.ErrWrongType) + require.Zero(t, v) + }) +} + +func testValueToken[S ~string | ~[]byte]( + t *testing.T, input S, check func(t *testing.T, token jscan.Token[S]), +) { + t.Helper() + tok := jscan.NewTokenizer[S](4, 16) + err := tok.Tokenize(input, func(tokens []jscan.Token[S]) (err bool) { + require.Len(t, tokens, 1) + check(t, tokens[0]) + return false + }) + require.False(t, err.IsErr()) +} diff --git a/tokenize.go b/tokenize.go index 1a70153..901c165 100644 --- a/tokenize.go +++ b/tokenize.go @@ -1,6 +1,11 @@ package jscan import ( + "fmt" + "strconv" + "unsafe" + + "github.com/romshark/jscan/v2/internal/atoi" "github.com/romshark/jscan/v2/internal/jsonnum" "github.com/romshark/jscan/v2/internal/strfind" ) @@ -74,7 +79,7 @@ func (t TokenType) String() string { } // Token is any JSON token except comma, colon and space. -type Token struct { +type Token[S ~string | ~[]byte] struct { // Index declares the start byte index in the source. Index int @@ -102,10 +107,301 @@ type Token struct { Type TokenType } +var ( + ErrOverflow = fmt.Errorf("token value overflows integer type") + ErrWrongType = fmt.Errorf("token value has different type") +) + +const intSize = unsafe.Sizeof(int(0)) + +// Int returns the int value of the token. +// Expects src to be the source string provided to the tokenizer. +// Returns int(0) if the token is a null value. +// Returns ErrWrongType if the token isn't an integer value. +// Returns ErrOverflow if the value would overflow type int. +func (t Token[S]) Int(src S) (int, error) { + if t.Type == TokenTypeNull { + return 0, nil + } + if t.Type != TokenTypeInteger { + return 0, ErrWrongType + } + if intSize != 8 { + v, overflow := atoi.I32(src[t.Index:t.End]) + if overflow { + return 0, ErrOverflow + } + return int(v), nil + } + v, overflow := atoi.I64(src[t.Index:t.End]) + if overflow { + return 0, ErrOverflow + } + return int(v), nil +} + +// Int8 returns the int8 value of the token. +// Expects src to be the source string provided to the tokenizer. +// Returns int8(0) if the token is a null value. +// Returns ErrWrongType if the token isn't an integer value. +// Returns ErrOverflow if the value would overflow type int8. +func (t Token[S]) Int8(src S) (int8, error) { + if t.Type == TokenTypeNull { + return 0, nil + } + if t.Type != TokenTypeInteger { + return 0, ErrWrongType + } + v, overflow := atoi.I8(src[t.Index:t.End]) + if overflow { + return 0, ErrOverflow + } + return v, nil +} + +// Int16 returns the int16 value of the token. +// Expects src to be the source string provided to the tokenizer. +// Returns int16(0) if the token is a null value. +// Returns ErrWrongType if the token isn't an integer value. +// Returns ErrOverflow if the value would overflow type int16. +func (t Token[S]) Int16(src S) (int16, error) { + if t.Type == TokenTypeNull { + return 0, nil + } + if t.Type != TokenTypeInteger { + return 0, ErrWrongType + } + v, overflow := atoi.I16(src[t.Index:t.End]) + if overflow { + return 0, ErrOverflow + } + return v, nil +} + +// Int32 returns the int32 value of the token. +// Expects src to be the source string provided to the tokenizer. +// Returns int32(0) if the token is a null value. +// Returns ErrWrongType if the token isn't an integer value. +// Returns ErrOverflow if the value would overflow type int32. +func (t Token[S]) Int32(src S) (int32, error) { + if t.Type == TokenTypeNull { + return 0, nil + } + if t.Type != TokenTypeInteger { + return 0, ErrWrongType + } + v, overflow := atoi.I32(src[t.Index:t.End]) + if overflow { + return 0, ErrOverflow + } + return v, nil +} + +// Int64 returns the int64 value of the token. +// Expects src to be the source string provided to the tokenizer. +// Returns int64(0) if the token is a null value. +// Returns ErrWrongType if the token isn't an integer value. +// Returns ErrOverflow if the value would overflow type int64. +func (t Token[S]) Int64(src S) (int64, error) { + if t.Type == TokenTypeNull { + return 0, nil + } + if t.Type != TokenTypeInteger { + return 0, ErrWrongType + } + v, overflow := atoi.I64(src[t.Index:t.End]) + if overflow { + return 0, ErrOverflow + } + return v, nil +} + +// Uint returns the uint value of the token. +// Expects src to be the source string provided to the tokenizer. +// Returns uint(0) if the token is a null value. +// Returns ErrWrongType if the token isn't an integer value. +// Returns ErrOverflow if the value would overflow type uint. +func (t Token[S]) Uint(src S) (uint, error) { + if t.Type == TokenTypeNull { + return 0, nil + } + if t.Type != TokenTypeInteger || src[t.Index] == '-' { + return 0, ErrWrongType + } + if intSize != 8 { + v, overflow := atoi.U32(src[t.Index:t.End]) + if overflow { + return 0, ErrOverflow + } + return uint(v), nil + } + v, overflow := atoi.U64(src[t.Index:t.End]) + if overflow { + return 0, ErrOverflow + } + return uint(v), nil +} + +// Uint8 returns the uint8 value of the token. +// Expects src to be the source string provided to the tokenizer. +// Returns uint8(0) if the token is a null value. +// Returns ErrWrongType if the token isn't an integer value. +// Returns ErrOverflow if the value would overflow type uint8. +func (t Token[S]) Uint8(src S) (uint8, error) { + if t.Type == TokenTypeNull { + return 0, nil + } + if t.Type != TokenTypeInteger || src[t.Index] == '-' { + return 0, ErrWrongType + } + v, overflow := atoi.U8(src[t.Index:t.End]) + if overflow { + return 0, ErrOverflow + } + return uint8(v), nil +} + +// Uint16 returns the uint16 value of the token. +// Expects src to be the source string provided to the tokenizer. +// Returns uint16(0) if the token is a null value. +// Returns ErrWrongType if the token isn't an integer value. +// Returns ErrOverflow if the value would overflow type uint16. +func (t Token[S]) Uint16(src S) (uint16, error) { + if t.Type == TokenTypeNull { + return 0, nil + } + if t.Type != TokenTypeInteger || src[t.Index] == '-' { + return 0, ErrWrongType + } + v, overflow := atoi.U16(src[t.Index:t.End]) + if overflow { + return 0, ErrOverflow + } + return uint16(v), nil +} + +// Uint32 returns the uint32 value of the token. +// Expects src to be the source string provided to the tokenizer. +// Returns uint32(0) if the token is a null value. +// Returns ErrWrongType if the token isn't an integer value. +// Returns ErrOverflow if the value would overflow type uint32. +func (t Token[S]) Uint32(src S) (uint32, error) { + if t.Type == TokenTypeNull { + return 0, nil + } + if t.Type != TokenTypeInteger || src[t.Index] == '-' { + return 0, ErrWrongType + } + v, overflow := atoi.U32(src[t.Index:t.End]) + if overflow { + return 0, ErrOverflow + } + return uint32(v), nil +} + +// Uint64 returns the uint64 value of the token. +// Expects src to be the source string provided to the tokenizer. +// Returns uint64(0) if the token is a null value. +// Returns ErrWrongType if the token isn't an integer value. +// Returns ErrOverflow if the value would overflow type uint64. +func (t Token[S]) Uint64(src S) (uint64, error) { + if t.Type == TokenTypeNull { + return 0, nil + } + if t.Type != TokenTypeInteger || src[t.Index] == '-' { + return 0, ErrWrongType + } + v, overflow := atoi.U64(src[t.Index:t.End]) + if overflow { + return 0, ErrOverflow + } + return uint64(v), nil +} + +// Float32 returns the float32 value of the token. +// Expects src to be the source string provided to the tokenizer. +// Returns float32(0) if the token is a null value. +// Returns ErrWrongType if the token isn't a number value. +func (t Token[S]) Float32(src S) (float32, error) { + if t.Type == TokenTypeNull { + return 0, nil + } + if t.Type != TokenTypeNumber && t.Type != TokenTypeInteger { + return 0, ErrWrongType + } + var sz S + var s string + if _, ok := any(sz).([]byte); ok { + b := []byte(src[t.Index:t.End]) + s = unsafe.String(unsafe.SliceData(b), len(b)) + } else { + s = string(src[t.Index:t.End]) + } + v, err := strconv.ParseFloat(s, 32) + if err != nil { + return 0, err + } + return float32(v), nil +} + +// Float64 returns the float64 value of the token. +// Expects src to be the source string provided to the tokenizer. +// Returns float64(0) if the token is a null value. +// Returns ErrWrongType if the token isn't a number value. +func (t Token[S]) Float64(src S) (float64, error) { + if t.Type == TokenTypeNull { + return 0, nil + } + if t.Type != TokenTypeNumber && t.Type != TokenTypeInteger { + return 0, ErrWrongType + } + var sz S + var s string + if _, ok := any(sz).([]byte); ok { + b := []byte(src[t.Index:t.End]) + s = unsafe.String(unsafe.SliceData(b), len(b)) + } else { + s = string(src[t.Index:t.End]) + } + v, err := strconv.ParseFloat(s, 64) + if err != nil { + return 0, err + } + return v, nil +} + +// Bool returns the bool value of the token. +// Expects src to be the source string provided to the tokenizer. +// Returns bool(false) if the token is a null value. +// Returns ErrWrongType if the token isn't a boolean value. +func (t Token[S]) Bool(src S) (bool, error) { + switch t.Type { + case TokenTypeTrue: + return true, nil + case TokenTypeFalse, TokenTypeNull: + return false, nil + } + return false, ErrWrongType +} + +// String returns the string value of the token. +// Expects src to be the source string provided to the tokenizer. +// Returns string("") if the token is a null value. +// Returns ErrWrongType if the token isn't a string value. +func (t Token[S]) String(src S) (string, error) { + switch t.Type { + case TokenTypeNull: + return "", nil + case TokenTypeString: + return string(src[t.Index+1 : t.End-1]), nil + } + return "", ErrWrongType +} + // Tokenizer is a reusable tokenizer instance holding a stack and a token buffer // which are reused across method calls. type Tokenizer[S ~string | ~[]byte] struct { - buffer []Token + buffer []Token[S] stack []int // Buffer index } @@ -124,7 +420,7 @@ func NewTokenizer[S ~string | ~[]byte]( preallocStackFrames, preallocTokenBuffer int, ) *Tokenizer[S] { t := &Tokenizer[S]{ - buffer: make([]Token, preallocTokenBuffer), + buffer: make([]Token[S], preallocTokenBuffer), stack: make([]int, preallocStackFrames), } return t @@ -140,7 +436,7 @@ func NewTokenizer[S ~string | ~[]byte]( // // WARNING: Don't use or alias tokens after fn returns! func (t *Tokenizer[S]) TokenizeOne( - s S, fn func(tokens []Token) (err bool), + s S, fn func(tokens []Token[S]) (err bool), ) (trailing S, err Error[S]) { return t.tokenize(s, fn) } @@ -153,7 +449,7 @@ func (t *Tokenizer[S]) topStackType() TokenType { // // WARNING: Don't use or alias tokens after fn returns! func (t *Tokenizer[S]) Tokenize( - s S, fn func(tokens []Token) (err bool), + s S, fn func(tokens []Token[S]) (err bool), ) Error[S] { tail, err := t.tokenize(s, fn) if err.IsErr() { @@ -172,7 +468,7 @@ func (t *Tokenizer[S]) Tokenize( // tokenize calls fn once all tokens are parsed to the buffer. // Returns the remainder of src and an error if any is encountered. -func (t *Tokenizer[S]) tokenize(src S, fn func(tokens []Token) (err bool)) (S, Error[S]) { +func (t *Tokenizer[S]) tokenize(src S, fn func(tokens []Token[S]) (err bool)) (S, Error[S]) { // Reset tokenizer t.buffer = t.buffer[:0] t.stack = t.stack[:0] @@ -243,13 +539,13 @@ VALUE_OBJECT: if s[0] == '}' { t.buffer = append( t.buffer, - Token{ + Token[S]{ Index: index, End: len(t.buffer) + 1, Type: TokenTypeObject, Elements: 0, }, - Token{ + Token[S]{ Index: len(src) - len(s), End: len(t.buffer), Type: TokenTypeObjectEnd, @@ -261,7 +557,7 @@ VALUE_OBJECT: } t.stack = append(t.stack, len(t.buffer)) - t.buffer = append(t.buffer, Token{ + t.buffer = append(t.buffer, Token[S]{ Index: index, End: 0, // To be set once the end is discovered. Type: TokenTypeObject, @@ -293,13 +589,13 @@ VALUE_ARRAY: if s[0] == ']' { t.buffer = append( t.buffer, - Token{ + Token[S]{ Index: index, End: len(t.buffer) + 1, Type: TokenTypeArray, Elements: 0, }, - Token{ + Token[S]{ Index: len(src) - len(s), End: len(t.buffer), Type: TokenTypeArrayEnd, @@ -313,7 +609,7 @@ VALUE_ARRAY: t.stack = append(t.stack, len(t.buffer)) t.buffer = append( t.buffer, - Token{ + Token[S]{ Index: index, End: 0, // To be set once the end is discovered. Type: TokenTypeArray, @@ -352,7 +648,7 @@ VALUE_NUMBER: if s, rc = jsonnum.ReadNumber(s); rc == jsonnum.ReturnCodeErr { return s, getError(ErrorCodeMalformedNumber, src, rollback) } - t.buffer = append(t.buffer, Token{ + t.buffer = append(t.buffer, Token[S]{ Index: index, End: len(src) - len(s), Type: TokenType(rc), @@ -460,7 +756,7 @@ VALUE_STRING: case '"': s = s[1:] - t.buffer = append(t.buffer, Token{ + t.buffer = append(t.buffer, Token[S]{ Type: TokenTypeString, Index: index, Elements: 0, @@ -481,7 +777,7 @@ VALUE_NULL: return s, getError(ErrorCodeUnexpectedToken, src, s) } index = len(src) - len(s) - t.buffer = append(t.buffer, Token{ + t.buffer = append(t.buffer, Token[S]{ Type: TokenTypeNull, Index: index, End: index + len("null"), @@ -496,7 +792,7 @@ VALUE_FALSE: return s, getError(ErrorCodeUnexpectedToken, src, s) } index = len(src) - len(s) - t.buffer = append(t.buffer, Token{ + t.buffer = append(t.buffer, Token[S]{ Type: TokenTypeFalse, Index: index, End: index + len("false"), @@ -511,7 +807,7 @@ VALUE_TRUE: return s, getError(ErrorCodeUnexpectedToken, src, s) } index = len(src) - len(s) - t.buffer = append(t.buffer, Token{ + t.buffer = append(t.buffer, Token[S]{ Type: TokenTypeTrue, Index: index, End: index + len("true"), @@ -642,7 +938,7 @@ OBJ_KEY: s = s[5:] case '"': s = s[1:] - t.buffer = append(t.buffer, Token{ + t.buffer = append(t.buffer, Token[S]{ Type: TokenTypeKey, Index: index, End: len(src) - len(s), @@ -716,7 +1012,7 @@ AFTER_VALUE: } t.buffer[t.stack[len(t.stack)-1]].End = len(t.buffer) // Link start token - t.buffer = append(t.buffer, Token{ + t.buffer = append(t.buffer, Token[S]{ Index: len(src) - len(s), End: t.stack[len(t.stack)-1], Type: TokenTypeObjectEnd, @@ -731,7 +1027,7 @@ AFTER_VALUE: } t.buffer[t.stack[len(t.stack)-1]].End = len(t.buffer) // Link start token - t.buffer = append(t.buffer, Token{ + t.buffer = append(t.buffer, Token[S]{ Index: len(src) - len(s), End: t.stack[len(t.stack)-1], Type: TokenTypeArrayEnd,