From 340cc1c6d71866ff60e4dc26073dfaab54ce10ad Mon Sep 17 00:00:00 2001 From: Stephane Moreau Date: Wed, 7 Oct 2020 13:29:20 +0200 Subject: [PATCH 1/4] - Adding ability to stop array processing by generating an error - Adding ability to guess datatype of the JSON data --- parser.go | 15 +++++++++++---- parser_test.go | 34 ++++++++++++++++++++++------------ 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/parser.go b/parser.go index 26d1cd8..52c1d8b 100644 --- a/parser.go +++ b/parser.go @@ -317,7 +317,7 @@ func searchKeys(data []byte, keys ...string) int { var valueFound []byte var valueOffset int var curI = i - ArrayEach(data[i:], func(value []byte, dataType ValueType, offset int, err error) { + ArrayEach(data[i:], func(value []byte, dataType ValueType, offset int, err error) error { if curIdx == aIdx { valueFound = value valueOffset = offset @@ -327,6 +327,7 @@ func searchKeys(data []byte, keys ...string) int { } } curIdx += 1 + return nil }) if valueFound == nil { @@ -508,7 +509,7 @@ func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]str level++ var curIdx int - arrOff, _ := ArrayEach(data[i:], func(value []byte, dataType ValueType, offset int, err error) { + arrOff, _ := ArrayEach(data[i:], func(value []byte, dataType ValueType, offset int, err error) error { if arrIdxFlags&bitwiseFlags[curIdx+1] != 0 { for pi, p := range paths { if pIdxFlags&bitwiseFlags[pi+1] != 0 { @@ -530,6 +531,7 @@ func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]str } curIdx += 1 + return nil }) if pathsMatched == len(paths) { @@ -555,6 +557,11 @@ func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]str return -1 } +func GuessValueType(data []byte, offset int) (ValueType, error) { + _, dataType, _, err := getType(data, offset) + return dataType, err +} + // Data types available in valid JSON data. type ValueType int @@ -920,7 +927,7 @@ func internalGet(data []byte, keys ...string) (value []byte, dataType ValueType, } // ArrayEach is used when iterating arrays, accepts a callback function with the same return arguments as `Get`. -func ArrayEach(data []byte, cb func(value []byte, dataType ValueType, offset int, err error), keys ...string) (offset int, err error) { +func ArrayEach(data []byte, cb func(value []byte, dataType ValueType, offset int, err error) error, keys ...string) (offset int, err error) { if len(data) == 0 { return -1, MalformedObjectError } @@ -930,7 +937,7 @@ func ArrayEach(data []byte, cb func(value []byte, dataType ValueType, offset int return -1, MalformedJsonError } - offset = nT+1 + offset = nT + 1 if len(keys) > 0 { if offset = searchKeys(data, keys...); offset == -1 { diff --git a/parser_test.go b/parser_test.go index b67a342..7308048 100644 --- a/parser_test.go +++ b/parser_test.go @@ -12,16 +12,18 @@ import ( var activeTest = "" func toArray(data []byte) (result [][]byte) { - ArrayEach(data, func(value []byte, dataType ValueType, offset int, err error) { + ArrayEach(data, func(value []byte, dataType ValueType, offset int, err error) error { result = append(result, value) + return nil }) return } func toStringArray(data []byte) (result []string) { - ArrayEach(data, func(value []byte, dataType ValueType, offset int, err error) { + ArrayEach(data, func(value []byte, dataType ValueType, offset int, err error) error { result = append(result, string(value)) + return nil }) return @@ -1363,7 +1365,7 @@ func TestArrayEach(t *testing.T) { mock := []byte(`{"a": { "b":[{"x": 1} ,{"x":2},{ "x":3}, {"x":4} ]}}`) count := 0 - ArrayEach(mock, func(value []byte, dataType ValueType, offset int, err error) { + ArrayEach(mock, func(value []byte, dataType ValueType, offset int, err error) error { count++ switch count { @@ -1386,14 +1388,18 @@ func TestArrayEach(t *testing.T) { default: t.Errorf("Should process only 4 items") } + return nil }, "a", "b") } func TestArrayEachWithWhiteSpace(t *testing.T) { //Issue #159 count := 0 - funcError := func([]byte, ValueType, int, error) { t.Errorf("Run func not allow") } - funcSuccess := func(value []byte, dataType ValueType, index int, err error) { + funcError := func([]byte, ValueType, int, error) error { + t.Errorf("Run func not allow") + return nil + } + funcSuccess := func(value []byte, dataType ValueType, index int, err error) error { count++ switch count { @@ -1412,17 +1418,18 @@ func TestArrayEachWithWhiteSpace(t *testing.T) { default: t.Errorf("Should process only 3 items") } + return nil } type args struct { data []byte - cb func(value []byte, dataType ValueType, offset int, err error) + cb func(value []byte, dataType ValueType, offset int, err error) error keys []string } tests := []struct { - name string - args args - wantErr bool + name string + args args + wantErr bool }{ {"Array with white space", args{[]byte(` ["AAA", "BBB", "CCC"]`), funcSuccess, []string{}}, false}, {"Array with only one character after white space", args{[]byte(` 1`), funcError, []string{}}, true}, @@ -1440,11 +1447,14 @@ func TestArrayEachWithWhiteSpace(t *testing.T) { } func TestArrayEachEmpty(t *testing.T) { - funcError := func([]byte, ValueType, int, error) { t.Errorf("Run func not allow") } + funcError := func([]byte, ValueType, int, error) error { + t.Errorf("Run func not allow") + return nil + } type args struct { data []byte - cb func(value []byte, dataType ValueType, offset int, err error) + cb func(value []byte, dataType ValueType, offset int, err error) error keys []string } tests := []struct { @@ -1675,7 +1685,7 @@ func TestEachKey(t *testing.T) { {"arrInt", "[3]"}, {"arrInt", "[5]"}, // Should not find last key {"nested"}, - {"arr", "["}, // issue#177 Invalid arguments + {"arr", "["}, // issue#177 Invalid arguments {"a\n", "b\n"}, // issue#165 } From ae12a8f05c0faf24e03811bb64a168cf70a5f5fd Mon Sep 17 00:00:00 2001 From: Stephane Moreau Date: Wed, 7 Oct 2020 14:32:00 +0200 Subject: [PATCH 2/4] Oups missing error check --- parser.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parser.go b/parser.go index 52c1d8b..24277e1 100644 --- a/parser.go +++ b/parser.go @@ -982,7 +982,7 @@ func ArrayEach(data []byte, cb func(value []byte, dataType ValueType, offset int } if t != NotExist { - cb(v, t, offset+o-len(v), e) + e = cb(v, t, offset+o-len(v), e) } if e != nil { From ef595a16905eab442e84af12ab882450f721847c Mon Sep 17 00:00:00 2001 From: Stephane Moreau Date: Wed, 7 Oct 2020 16:56:05 +0200 Subject: [PATCH 3/4] Make sure we can get the offset where array decoding stopped --- parser.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parser.go b/parser.go index 24277e1..d71e8f9 100644 --- a/parser.go +++ b/parser.go @@ -986,7 +986,7 @@ func ArrayEach(data []byte, cb func(value []byte, dataType ValueType, offset int } if e != nil { - break + return offset + o - len(v), e } offset += o From 3a7dadfebbbf28f9f0bdb54837df3a494bb3d22a Mon Sep 17 00:00:00 2001 From: Stephane Moreau Date: Thu, 8 Oct 2020 15:29:42 +0200 Subject: [PATCH 4/4] offset is automatically found while guessing type --- parser.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/parser.go b/parser.go index d71e8f9..c850179 100644 --- a/parser.go +++ b/parser.go @@ -557,7 +557,15 @@ func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]str return -1 } -func GuessValueType(data []byte, offset int) (ValueType, error) { +// GuessValueType : returns the type of the next JSON piece in data stream +// Does not ensure that the whole data is a valid JSON +func GuessValueType(data []byte) (ValueType, error) { + + offset := nextToken(data) + if offset == -1 { + return NotExist, MalformedJsonError + } + _, dataType, _, err := getType(data, offset) return dataType, err }