From f06c838c7dad2f87eb837321df9e881542e1c40a Mon Sep 17 00:00:00 2001 From: "Daniel G. Taylor" Date: Fri, 7 Oct 2022 21:59:31 -0700 Subject: [PATCH 1/6] feat: v2 rewrite --- .travis.yml | 9 - README.md | 318 ++- apply.go | 311 +++ apply_test.go | 302 +++ cmd/j/main.go | 63 +- document.go | 133 + document_test.go | 78 + error.go | 62 + generated.go | 2157 ----------------- get.go | 383 +++ get_test.go | 153 ++ go.mod | 20 +- go.sum | 24 +- parse.go | 584 +++++ parse_test.go | 224 ++ shorthand.go | 291 +-- shorthand.peg | 154 -- shorthand_test.go | 275 ++- testdata/binary | 1 + ...7850671bfee0b259ba88bd4e3bfd151e5feac1b8db | 2 + ...064bb81fff2541c71debe6a711781ef7a10ab00ed2 | 2 + ...b116a2b878980a3d9825af646dcbd9422b1f0a9903 | 2 + testdata/hello.cbor | Bin 0 -> 38 bytes testdata/hello.txt | 1 + 24 files changed, 2709 insertions(+), 2840 deletions(-) delete mode 100644 .travis.yml create mode 100644 apply.go create mode 100644 apply_test.go create mode 100644 document.go create mode 100644 document_test.go create mode 100644 error.go delete mode 100644 generated.go create mode 100644 get.go create mode 100644 get_test.go create mode 100644 parse.go create mode 100644 parse_test.go delete mode 100644 shorthand.peg create mode 100644 testdata/binary create mode 100644 testdata/fuzz/FuzzParser/12fdb5c4b8c383a42822597850671bfee0b259ba88bd4e3bfd151e5feac1b8db create mode 100644 testdata/fuzz/FuzzParser/bdced84807f06393f864fa064bb81fff2541c71debe6a711781ef7a10ab00ed2 create mode 100644 testdata/fuzz/FuzzParser/f2f1bb2b4123163cc61f3eb116a2b878980a3d9825af646dcbd9422b1f0a9903 create mode 100644 testdata/hello.cbor create mode 100644 testdata/hello.txt diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 18300df..0000000 --- a/.travis.yml +++ /dev/null @@ -1,9 +0,0 @@ -language: go -go: -- '1.11' -- '1.12' -before_install: -- go get -u github.com/shuLhan/go-bindata/... -- go get golang.org/x/tools/cmd/goimports -script: -- ./test.sh diff --git a/README.md b/README.md index c9b87c6..2f7f17a 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,40 @@ -# CLI Shorthand Syntax +# Structured Data Shorthand Syntax [![Docs](https://godoc.org/github.com/danielgtaylor/shorthand?status.svg)](https://pkg.go.dev/github.com/danielgtaylor/shorthand?tab=doc) [![Go Report Card](https://goreportcard.com/badge/github.com/danielgtaylor/shorthand)](https://goreportcard.com/report/github.com/danielgtaylor/shorthand) -CLI shorthand syntax is a contextual shorthand syntax for passing structured data into commands that require e.g. JSON/YAML. While you can always pass full JSON or other documents through `stdin`, you can also specify or modify them by hand as arguments to the command using this shorthand syntax. For example: +Shorthand is a superset and friendlier variant of JSON designed with several use-cases in mind: -```sh -$ my-cli do-something foo.bar[0].baz: 1, .hello: world -``` +| Use Case | Example | +| -------------------- | ------------------------------------------------ | +| CLI arguments/input | `my-cli post 'foo.bar[0]{baz: 1, hello: world}'` | +| Patch operations | `name: undefined, item.tags[]: appended` | +| Query language | `items[created before "2022-01-01"].{id, tags}` | +| Configuration format | `{json.save.autoFormat: true}` | + +The shorthand syntax supports the following features, described in more detail with examples below: -Would result in the following body contents being sent on the wire (assuming a JSON media type is specified in the OpenAPI spec): +- Superset of JSON (valid JSON is valid shorthand) +- Optional commas, quotes, and sometimes colons +- Automatic type coercion + - Support for bytes, dates, and maps with non-string keys +- Nested object & array creation +- Loading values from files +- Editing existing data + - Appending & inserting to arrays + - Unsetting properties + - Moving properties & items +- Querying, array filtering, and field selection + +The following are both completely valid shorthand and result in the same output: + +``` +{ + foo.bar[]{ + baz: 1 + hello: world + } +} +``` ```json { @@ -23,16 +49,7 @@ Would result in the following body contents being sent on the wire (assuming a J } ``` -The shorthand syntax supports the following features, described in more detail with examples below: - -- Automatic type coercion & forced strings -- Nested object creation -- Object property grouping -- Nested array creation -- Appending to arrays -- Both object and array backreferences -- Loading property values from files - - Supports structured, forced string, and base64 data +This library has excellent test coverage and is additionally fuzz tested to ensure correctness and prevent panics. ## Alternatives & Inspiration @@ -44,7 +61,7 @@ The CLI shorthand syntax is not the only one you can use to generate data for CL For example, the shorthand example given above could be rewritten as: ```sh -$ jo -p foo=$(jo -p bar=$(jo -a $(jo baz=1 hello=world))) | my-cli do-something +$ jo -p foo=$(jo -p bar=$(jo -a $(jo -p baz=1 hello=world))) ``` The shorthand syntax implementation described herein uses those and the following for inspiration: @@ -56,9 +73,10 @@ The shorthand syntax implementation described herein uses those and the followin It seems reasonable to ask, why create a new syntax? -1. Built-in. No extra executables required. Your CLI ships ready-to-go. +1. Built-in. No extra executables required. Your tool ships ready-to-go. 2. No need to use sub-shells to build complex structured data. 3. Syntax is closer to YAML & JSON and mimics how you do queries using tools like `jq` and `jmespath`. +4. It's _optional_, so you can use your favorite tool/language instead, while at the same time it provides a minimum feature set everyone will have in common. ## Features in Depth @@ -82,9 +100,24 @@ $ j hello: world, question: how are you? } ``` -### Types and Type Coercion +### Types -Well-known values like `null`, `true`, and `false` get converted to their respective types automatically. Numbers also get converted. Similar to YAML, anything that doesn't fit one of those is treated as a string. If needed, you can disable this automatic coercion by forcing a value to be treated as a string with the `~` operator. **Note**: the `~` modifier must come _directly after_ the colon. +Shorthand supports the standard JSON types, but adds some of its own as well to better support binary formats and its query features. + +| Type | Description | +| --------- | ---------------------------------------------------------------- | +| `null` | JSON `null` | +| `boolean` | Either `true` or `false` | +| `number` | JSON number, e.g. `1`, `2.5`, or `1.4e5` | +| `string` | Quoted or unquoted strings, e.g. `"hello"` | +| `bytes` | `%`-prefixed, unquoted, base64-encoded binary data, e.g. `%wg==` | +| `time` | Date/time in ISO8601, e.g. `2022-01-01T12:00:00Z` | +| `array` | JSON array, e.g. `[1, 2, 3]` | +| `object` | JSON object, e.g. `{"hello": "world"}` | + +### Type Coercion + +Well-known values like `null`, `true`, and `false` get converted to their respective types automatically. Numbers, bytes, and times also get converted. Similar to YAML, anything that doesn't fit one of those is treated as a string. This automatic coercion can be disabled by just wrapping your value in quotes. ```sh # With coercion @@ -97,7 +130,7 @@ $ j empty: null, bool: true, num: 1.5, string: hello } # As strings -$ j empty:~ null, bool:~ true, num:~ 1.5, string:~ hello +$ j empty: "null", bool: "true", num: "1.5", string: "hello" { "bool": "true", "empty": "null", @@ -106,21 +139,10 @@ $ j empty:~ null, bool:~ true, num:~ 1.5, string:~ hello } # Passing the empty string -$ j blank:~ -{ - "blank": "" -} - -# Passing a tilde using whitespace -$ j foo: ~/Documents +$ j blank1: , blank2: "" { - "foo": "~/Documents" -} - -# Passing a tilde using forced strings -$ j foo:~~/Documents -{ - "foo": "~/Documents" + "blank1": "", + "blank2": "" } ``` @@ -157,32 +179,32 @@ $ j foo.bar{id: 1, count.clicks: 5} ### Arrays -Simple arrays use a `,` between values. Nested arrays use square brackets `[` and `]` to specify the zero-based index to insert an item. Use a blank index to append to the array. +Arrays are surrounded by square brackets like in JSON: ```sh -# Array shorthand -$ j a: 1, 2, 3 -{ - "a": [ - 1, - 2, - 3 - ] -} +# Simple array +$ j [1, 2, 3] +[ + 1, + 2, + 3 +] +``` + +Array indexes use square brackets `[` and `]` to specify the zero-based index to set an item. If the index is out of bounds then `null` values are added as necessary to fill the array. Use an empty index `[]` to append to the an existing array. If the item is not an array, then a new one will be created. +```sh # Nested arrays -$ j a[0][2][0]: 1 -{ - "a": [ +$ j [0][2][0]: 1 +[ + [ + null, + null, [ - null, - null, - [ - 1 - ] + 1 ] ] -} +] # Appending arrays $ j a[]: 1, a[]: 2, a[]: 3 @@ -195,94 +217,151 @@ $ j a[]: 1, a[]: 2, a[]: 3 } ``` -### Backreferences +### Loading from Files -Since the shorthand syntax is context-aware, it is possible to use the current context to reference back to the most recently used object or array when creating new properties or items. +Sometimes a field makes more sense to load from a file than to be specified on the commandline. The `@` preprocessor lets you load structured data, text, and bytes depending on the file extension and whether all bytes are valid UTF-8: ```sh -# Backref with object properties -$ j foo.bar: 1, .baz: 2 +# Load a file's value as a parameter +$ j foo: @hello.txt { - "foo": { - "bar": 1, - "baz": 2 - } + "foo": "hello, world" } -# Backref with array appending -$ j foo.bar[]: 1, []: 2, []: 3 +# Load structured data +$ j foo: @hello.json { "foo": { - "bar": [ - 1, - 2, - 3 - ] + "hello": "world" } } +``` + +Remember, it's possible to disable this behavior with quotes: -# Easily build complex structures -$ j name: foo, tags[]{id: 1, count.clicks: 5, .sales: 1}, []{id: 2, count.clicks: 8, .sales: 2} +```sh +$ j 'twitter: "@user"' { - "name": "foo", - "tags": [ - { - "count": { - "clicks": 5, - "sales": 1 - }, - "id": 1 - }, - { - "count": { - "clicks": 8, - "sales": 2 - }, - "id": 2 - } - ] + "twitter": "@user" } ``` -### Loading from Files +### Patch (Partial Update) + +Partial updates are supported on existing data, which can be used to implement HTTP `PATCH`, templating, and other similar features. This feature combines the best of both: + +- [JSON Merge Patch](https://datatracker.ietf.org/doc/html/rfc7386) +- [JSON Patch](https://www.rfc-editor.org/rfc/rfc6902) -Sometimes a field makes more sense to load from a file than to be specified on the commandline. The `@` preprocessor and `~` & `%` modifiers let you load structured data, strings, and base64-encoded data into values. +Partial updates support: + +- Appending arrays via `[]` +- Inserting before via `[^index]` +- Removing fields or array items via `undefined` +- Moving/swapping fields or array items via `^` + - The right hand side is a path to the value to swap + +Some examples: ```sh -# Load a file's value as a parameter -$ j foo: @hello.txt +# First, let's create some data we'll modify later +$ j id: 1, tags: [a, b, c] >data.json + +# Now let's append to the tags array +$ j data.json + +# Query for each user's ID +$ j 0 { + // We have values, so try to parse it as an index! + s := d.buf.String() + if s[0] == '^' { + insert = true + s = s[1:] + } + index, err = strconv.Atoi(s) + if err != nil { + return nil, d.error(uint(len(s)), "Cannot convert index to number") + } + appnd = false + } + if d.options.DebugLogger != nil { + d.options.DebugLogger("Setting index %d insert:%v append:%v", index, insert, appnd) + } + + // Change the type to a slice if needed. + if !isArray(input) { + input = []any{} + } + + s := input.([]any) + if appnd { + // Append (i.e. index equals length). + index = len(s) + } else if index < 0 { + // Support negative indexes, e.g. `-1` is the last item. + index = len(s) + index + } + + // Grow by appending nil until the slice is the right length. + grew := false + for len(s) <= index { + grew = true + s = append(s, nil) + } + + // Handle insertion, i.e. shifting items if needed after appending a new + // item to shift them into (if the index was within the bounds of the + // original slice). + if insert { + if !grew { + s = append(s, nil) + } + for i := len(s) - 2; i >= index; i-- { + s[i+1] = s[i] + } + } + + // Depending on what the next character is, we either set the value or + // recurse with more indexes or path parts. + if p := d.peek(); p == -1 { + if op.Kind == OpDelete { + s = append(s[:index], s[index+1:]...) + } else { + s[index] = op.Value + } + } else if p == '[' { + d.next() + result, err := d.applyIndex(s[index], op) + if err != nil { + return nil, err + } + s[index] = result + } else if p == '.' { + d.next() + result, err := d.applyPathPart(s[index], op) + if err != nil { + return nil, err + } + s[index] = result + } else { + panic("unexpected char " + string(p)) + } + + return s, nil +} + +func (d *Document) applyPathPart(input any, op Operation) (any, Error) { + quoted := false + d.buf.Reset() + + for { + r := d.next() + + if r == '\\' { + if d.parseEscape(false, false) { + continue + } + } + + if r == '"' { + if err := d.parseQuoted(false); err != nil { + return nil, err + } + quoted = true + continue + } + + if r == '.' || r == '[' || r == -1 { + keystr := d.buf.String() + var key any = keystr + d.buf.Reset() + + if key == "" { + // Special case: raw value + if r == '[' { + // This raw value is an array. + return d.applyIndex(input, op) + } + return op.Value, nil + } + + if !d.options.ForceStringKeys && !quoted { + if tmp, ok := coerceValue(keystr); ok { + key = tmp + } + } + + if d.options.DebugLogger != nil { + d.options.DebugLogger("Setting key %v", key) + } + + wantMap := true + applyFunc := d.applyPathPart + if r == '[' { + wantMap = false + applyFunc = d.applyIndex + } else if r == -1 { + applyFunc = applyValue + } + + if !isMap(input) { + if def, ok := withDefault(true, key, nil); ok { + input = def + } + } + + if m, ok := input.(map[string]any); ok { + if s, ok := key.(string); ok { + if wantMap && op.Kind == OpDelete { + delete(m, s) + } else { + result, err := applyFunc(m[s], op) + if err != nil { + return nil, err + } + m[s] = result + } + break + } else { + // Key is not a string, so convert input into a generic + // map[any]any and process it further below. + input = makeGenericMap(m) + } + } + + if m, ok := input.(map[any]any); ok { + if wantMap && op.Kind == OpDelete { + delete(m, key) + } else { + v := m[key] + if def, ok := withDefault(wantMap, key, v); ok { + v = def + } + result, err := applyFunc(v, op) + if err != nil { + return nil, err + } + m[key] = result + } + break + } + + panic("can't get here") + } + + d.buf.WriteRune(r) + } + + return input, nil +} + +func (d *Document) applySwap(input any, op Operation) (any, Error) { + // First, get both left & right values from the input. + left, okl, err := GetPath(op.Path, input, GetOptions{DebugLogger: d.options.DebugLogger}) + if err != nil { + return nil, err + } + right, okr, err := GetPath(op.Value.(string), input, GetOptions{DebugLogger: d.options.DebugLogger}) + if err != nil { + return nil, err + } + + if d.options.DebugLogger != nil { + d.options.DebugLogger("Swapping (%v, %t) & (%v, %t)", left, okl, right, okr) + } + + // Set/unset left to value of right + kind := OpSet + if !okr { + kind = OpDelete + } + d.expression = op.Path + d.pos = 0 + input, err = d.applyPathPart(input, Operation{ + Kind: kind, + Path: op.Path, + Value: right, + }) + if err != nil { + return nil, err + } + + // Set/unset right to value of left + kind = OpSet + if !okl { + kind = OpDelete + } + d.expression = op.Value.(string) + d.pos = 0 + return d.applyPathPart(input, Operation{ + Kind: kind, + Path: op.Value.(string), + Value: left, + }) +} + +func (d *Document) applyOp(input any, op Operation) (any, Error) { + d.expression = op.Path + d.pos = 0 + d.buf.Reset() + + switch op.Kind { + case OpSet, OpDelete: + return d.applyPathPart(input, op) + case OpSwap: + return d.applySwap(input, op) + } + + return d.applyPathPart(input, op) +} diff --git a/apply_test.go b/apply_test.go new file mode 100644 index 0000000..03920e4 --- /dev/null +++ b/apply_test.go @@ -0,0 +1,302 @@ +package shorthand + +import ( + "encoding/json" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var dt, _ = time.Parse(time.RFC3339, "2020-01-01T12:00:00Z") + +var applyExamples = []struct { + Name string + Existing interface{} + Input string + Error string + Go interface{} + JSON string +}{ + { + Name: "Value", + Input: "true", + JSON: `true`, + }, + { + Name: "Coercion", + Input: "{n: null, b: true, i: 1, f: 1.0, dt: 2020-01-01T12:00:00Z, s: hello}", + Go: map[string]interface{}{ + "n": nil, + "b": true, + "i": 1, + "f": 1.0, + "dt": dt, + "s": "hello", + }, + JSON: `{"n": null, "b": true, "i": 1, "f": 1.0, "dt": "2020-01-01T12:00:00Z", "s": "hello"}`, + }, + { + Name: "Property nested", + Input: "{foo.bar.baz: hello}", + JSON: `{"foo": {"bar": {"baz": "hello"}}}`, + }, + { + Name: "Force strings quote", + Input: `{n: "null", b: "true", i: "1", f: "1.0", s: "hello"}`, + Go: map[string]interface{}{ + "n": "null", + "b": "true", + "i": "1", + "f": "1.0", + "s": "hello", + }, + JSON: `{"n": "null", "b": "true", "i": "1", "f": "1.0", "s": "hello"}`, + }, + { + Name: "Property new type", + Input: "{foo: [1, 2], foo: true}", + JSON: `{"foo": true}`, + }, + { + Name: "Ignore whitespace", + Input: "{foo : hello , bar:world }", + JSON: `{"foo": "hello", "bar": "world"}`, + }, + { + Name: "Allow quoted whitespace", + Input: `{"foo ": " hello ", " bar":"world "}`, + JSON: `{"foo ": " hello ", " bar": "world "}`, + }, + { + Name: "Coerce trailing space in object", + Input: "{foo{a: 1 }}", + JSON: `{"foo": {"a": 1}}`, + }, + { + Name: "Escape property", + Input: "{foo\\.bar: baz}", + JSON: `{"foo.bar": "baz"}`, + }, + { + Name: "Escape quoted property", + Input: `{"foo\"bar": baz}`, + JSON: `{"foo\"bar": "baz"}`, + }, + { + Name: "Quoted property special chars", + Input: `{"foo.bar": baz}`, + JSON: `{"foo.bar": "baz"}`, + }, + { + Name: "Array", + Input: "{foo: [1, 2, 3]}", + JSON: `{"foo": [1, 2, 3]}`, + }, + { + Name: "Array indexing", + Input: "{foo[3]: three, foo[5]: five, foo[0]: true}", + JSON: `{"foo": [true, null, null, "three", null, "five"]}`, + }, + { + Name: "Append", + Input: "{foo[]: 1, foo[]: 2, foo[]: 3}", + JSON: `{"foo": [1, 2, 3]}`, + }, + { + Name: "Insert prepend", + Input: "{foo: [1, 2], foo[^0]: 0}", + JSON: `{"foo": [0, 1, 2]}`, + }, + { + Name: "Insert middle", + Input: "{foo: [1, 2], foo[^1]: 0}", + JSON: `{"foo": [1, 0, 2]}`, + }, + { + Name: "Insert after", + Input: "{foo: [1, 2], foo[^3]: 0}", + JSON: `{"foo": [1, 2, null, 0]}`, + }, + { + Name: "Nested array", + Input: "{foo[][1][]: 1}", + JSON: `{"foo": [[null, [1]]]}`, + }, + { + Name: "Complex nested array", + Input: "{foo[][]: 1, foo[0][0][]: [2, 3], bar[]: true, bar[0]: false}", + JSON: `{"foo": [[[[2, 3]]]], "bar": [false]}`, + }, + { + Name: "List of objects", + Input: "{foo[]{id: 1, count: 1}, foo[]{id: 2, count: 2}}", + JSON: `{"foo": [{"id": 1, "count": 1}, {"id": 2, "count": 2}]}`, + }, + { + Name: "JSON input", + Input: `{"null": null, "bool": true, "num": 1.5, "str": "hello", "arr": ["tag1", "tag2"], "obj": {"id": [1]}}`, + JSON: `{"null": null, "bool": true, "num": 1.5, "str": "hello", "arr": ["tag1", "tag2"], "obj": {"id": [1]}}`, + }, + { + Name: "JSON naked escapes", + Input: `{foo\u000Abar: a\nb, baz\ta: a\nb}`, + JSON: `{"foo\nbar": "a\nb", "baz\ta": "a\nb"}`, + }, + { + Name: "JSON string escapes", + Input: `{"foo\u000Abar": "a\nb", "baz\ta": "a\nb"}`, + JSON: `{"foo\nbar": "a\nb", "baz\ta": "a\nb"}`, + }, + { + Name: "Top-level array", + Input: `[1, 2, "hello"]`, + JSON: `[1, 2, "hello"]`, + }, + { + Name: "Non-string keys", + Input: `{1: a, 2.3: b, bar.baz.4: c}`, + Go: map[interface{}]interface{}{ + 1: "a", + 2: map[interface{}]interface{}{ + 3: "b", + }, + "bar": map[string]interface{}{ + "baz": map[interface{}]interface{}{ + 4: "c", + }, + }, + }, + JSON: `{"1": "a", "2": {"3": "b"}, "bar": {"baz": {"4": "c"}}}`, + }, + { + Name: "Key type reset", + Input: `{foo: true, 2: false}`, + Go: map[interface{}]interface{}{ + "foo": true, + 2: false, + }, + JSON: `{"foo": true, "2": false}`, + }, + { + Name: "Nested key type reset", + Input: `{foo.bar: true, foo.2: false, foo.2.baz: hello, foo.2.3: false}`, + Go: map[string]interface{}{ + "foo": map[interface{}]interface{}{ + "bar": true, + 2: map[interface{}]interface{}{ + "baz": "hello", + 3: false, + }, + }, + }, + JSON: `{"foo": {"bar": true, "2": {"baz": "hello", "3": false}}}`, + }, + { + Name: "Existing", + Existing: map[string]interface{}{ + "foo": []interface{}{1, 2}, + "bar": []interface{}{[]interface{}{1}}, + "baz": map[string]interface{}{ + "id": 1, + }, + "hello": "world", + }, + Input: `{foo[]: 3, foo[]: 4, bar[0][]: 2, baz.another: test}`, + JSON: `{ + "foo": [1, 2, 3, 4], + "bar": [[1, 2]], + "baz": { + "id": 1, + "another": "test" + }, + "hello": "world" + }`, + }, + { + Name: "Unset property", + Existing: map[string]interface{}{ + "foo": true, + "bar": 1, + }, + Input: "{bar: undefined}", + JSON: `{"foo": true}`, + }, + { + Name: "Unset array item", + Existing: map[string]interface{}{ + "foo": []interface{}{1, 2, 3, 4}, + }, + Input: "{foo[1]: undefined}", + JSON: `{"foo": [1, 3, 4]}`, + }, + { + Name: "Move property", + Existing: map[string]interface{}{ + "foo": "hello", + }, + Input: "{bar ^ foo}", + JSON: `{"bar": "hello"}`, + }, + { + Name: "Swap property", + Existing: map[string]interface{}{ + "foo": "hello", + "bar": "world", + }, + Input: "{bar ^ foo}", + JSON: `{"bar": "hello", "foo": "world"}`, + }, + { + Name: "Swap index", + Existing: map[string]interface{}{ + "foo": []interface{}{1, 2, 3}, + }, + Input: "{bar ^ foo[0]}", + JSON: `{"bar": 1, "foo": [2, 3]}`, + }, +} + +func TestApply(t *testing.T) { + for _, example := range applyExamples { + t.Run(example.Name, func(t *testing.T) { + t.Logf("Input: %s", example.Input) + d := NewDocument( + ParseOptions{ + EnableFileInput: true, + DebugLogger: t.Logf, + }, + ) + err := d.Parse(example.Input) + if example.Error == "" { + require.NoError(t, err) + } else { + require.Error(t, err, "result is %v", d.Operations) + require.Contains(t, err.Error(), example.Error) + } + + ops := d.Marshal() + b, _ := json.Marshal(ops) + t.Log(string(b)) + + result, err := d.Apply(example.Existing) + if example.Error == "" { + require.NoError(t, err) + } else { + require.Error(t, err, "result is %v", d.Operations) + require.Contains(t, err.Error(), example.Error) + } + + if example.Go != nil { + assert.Equal(t, example.Go, result) + } + + if example.JSON != "" { + result = ConvertMapString(result) + b, _ := json.Marshal(result) + assert.JSONEq(t, example.JSON, string(b)) + } + }) + } +} diff --git a/cmd/j/main.go b/cmd/j/main.go index a07cb62..d39c5bb 100644 --- a/cmd/j/main.go +++ b/cmd/j/main.go @@ -4,8 +4,11 @@ import ( "encoding/json" "fmt" "os" + "reflect" + "strings" "github.com/danielgtaylor/shorthand" + "github.com/fxamacker/cbor/v2" toml "github.com/pelletier/go-toml" "github.com/spf13/cobra" yaml "gopkg.in/yaml.v3" @@ -13,16 +16,51 @@ import ( func main() { var format *string + var verbose *bool + var query *string + + var debugLog func(string, ...any) cmd := &cobra.Command{ Use: fmt.Sprintf("%s [flags] key1: value1, key2: value2, ...", os.Args[0]), Short: "Generate shorthand structured data", Example: fmt.Sprintf("%s foo.bar: 1, .baz: true", os.Args[0]), - Args: cobra.MinimumNArgs(1), Run: func(cmd *cobra.Command, args []string) { - result, err := shorthand.GetInput(args) + if len(args) == 0 && *query == "" { + fmt.Println("At least one arg or --query need to be passed") + os.Exit(1) + } + if *verbose { + debugLog = func(format string, a ...interface{}) { + fmt.Printf(format, a...) + fmt.Println() + } + fmt.Printf("Input: %s\n", strings.Join(args, " ")) + } + result, err := shorthand.GetInputWithOptions(args, shorthand.ParseOptions{ + EnableFileInput: true, + EnableObjectDetection: true, + ForceStringKeys: *format == "json", + DebugLogger: debugLog, + }) if err != nil { - panic(err) + if e, ok := err.(shorthand.Error); ok { + fmt.Println(e.Pretty()) + os.Exit(1) + } else { + panic(err) + } + } + + if *query != "" { + if selected, ok, err := shorthand.GetPath(*query, result, shorthand.GetOptions{DebugLogger: debugLog}); ok { + result = selected + } else if err != nil { + panic(err) + } else { + fmt.Println("No match") + return + } } var marshalled []byte @@ -30,15 +68,22 @@ func main() { switch *format { case "json": marshalled, err = json.MarshalIndent(result, "", " ") + case "cbor": + marshalled, err = cbor.Marshal(result) case "yaml": marshalled, err = yaml.Marshal(result) case "toml": - t, err := toml.TreeFromMap(result) - if err == nil { - marshalled = []byte(t.String()) + if k := reflect.TypeOf(result).Kind(); k != reflect.Map { + err = fmt.Errorf("TOML only supports maps but found %s", k.String()) + } else { + t, err := toml.TreeFromMap(result.(map[string]interface{})) + if err == nil { + marshalled = []byte(t.String()) + } } case "shorthand": - marshalled = []byte(shorthand.Get(result)) + // TODO: fix to support non-maps + marshalled = []byte(shorthand.Get(result.(map[string]interface{}))) } if err != nil { @@ -49,7 +94,9 @@ func main() { }, } - format = cmd.Flags().StringP("format", "f", "json", "Output format [json, yaml, toml, shorthand]") + format = cmd.Flags().StringP("format", "f", "json", "Output format [json, cbor, yaml, toml, shorthand]") + verbose = cmd.Flags().BoolP("verbose", "v", false, "Enable verbose output") + query = cmd.Flags().StringP("query", "q", "", "Path to query") cmd.Execute() } diff --git a/document.go b/document.go new file mode 100644 index 0000000..ad0ee86 --- /dev/null +++ b/document.go @@ -0,0 +1,133 @@ +package shorthand + +import ( + "bytes" +) + +type OpKind int + +const ( + OpSet OpKind = iota + OpDelete + OpSwap +) + +type ParseOptions struct { + // EnableFileInput turns on support for `@filename`` values which load from + // files rather than being treated as string input. + EnableFileInput bool + + // EnableObjectDetection will enable omitting the outer `{` and `}` for + // objects, which can be useful for some applications such as command line + // arguments. + EnableObjectDetection bool + + // ForceStringKeys forces all map keys to be treated as strings, resulting + // in all maps being of type `map[string]interface{}`. By default other types + // are allowed, which will result in the use of `map[interface{}]interface{}` + // for maps with non-string keys (`map[string]interface{}` is still the + // default even when non-string keys are allowed). + // If you know the output target will be JSON, you can enable this option + // to efficiently create a result that will `json.Marshal(...)` safely. + ForceStringKeys bool + + // DebugLogger sets a function to be used for printing out debug information. + DebugLogger func(format string, a ...interface{}) +} + +type Operation struct { + Kind OpKind + Path string + Value interface{} +} + +type Document struct { + Operations []Operation + + options ParseOptions + expression string + pos uint + lastWidth uint + buf bytes.Buffer +} + +func NewDocument(options ParseOptions) *Document { + return &Document{ + options: options, + } +} + +func (d *Document) String() string { + // TODO: serialize to text format + return "" +} + +func (d *Document) Unmarshal(data []byte) { + // TODO: load from JSON/CBOR representation +} + +func (d *Document) Marshal() interface{} { + ops := make([]interface{}, len(d.Operations)) + + for i, op := range d.Operations { + s := []interface{}{} + if op.Kind != OpSet { + s = append(s, op.Kind) + } + s = append(s, op.Path) + if op.Value != nil { + s = append(s, op.Value) + } + ops[i] = s + } + + return ops +} + +func (d *Document) Parse(input string) Error { + d.expression = input + d.pos = 0 + + if d.options.EnableObjectDetection { + // Try and determine if this is actually an object without the outer + // `{` and `}` surrounding it. We re-use `parseProp`` for this as it + // already handles things like quotes, escaping, etc. + for { + _, err := d.parseProp("", false) + if err != nil { + break + } + r := d.next() + if r == ':' || r == '^' { + // We have found an object! Wrap it and continue. + d.expression = "{" + input + "}" + if d.options.DebugLogger != nil { + d.options.DebugLogger("Detected object, wrapping in { and }") + } + } + } + d.pos = 0 + } + + err := d.parseValue("", true, false) + if err != nil { + return err + } + d.skipWhitespace() + if !d.expect(-1) { + return d.error(1, "Expected EOF but found additional input") + } + return nil +} + +func (d *Document) Apply(input interface{}) (interface{}, Error) { + var err Error + for _, op := range d.Operations { + input, err = d.applyOp(input, op) + if err != nil { + return nil, err + } + } + + return input, nil +} diff --git a/document_test.go b/document_test.go new file mode 100644 index 0000000..620524d --- /dev/null +++ b/document_test.go @@ -0,0 +1,78 @@ +package shorthand + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" +) + +func BenchmarkMinJSON(b *testing.B) { + b.ReportAllocs() + var v interface{} + + for n := 0; n < b.N; n++ { + assert.NoError(b, json.Unmarshal([]byte(`{"foo": {"bar": {"id": 1, "tags": ["one", "two"], "cost": 3.14}, "baz": {"id": 2}}}`), &v)) + } +} + +func BenchmarkFormattedJSON(b *testing.B) { + b.ReportAllocs() + var v interface{} + + large := []byte(`{ + "foo": { + "bar": { + "id": 1, + "tags": ["one", "two"], + "cost": 3.14 + }, + "baz": { + "id": 2 + } + } + }`) + + for n := 0; n < b.N; n++ { + assert.NoError(b, json.Unmarshal(large, &v)) + } +} + +func BenchmarkLatestFull(b *testing.B) { + b.ReportAllocs() + + d := NewDocument(ParseOptions{ + ForceStringKeys: true, + }) + + for n := 0; n < b.N; n++ { + d.Operations = d.Operations[:0] + d.Parse(`{foo{bar{id: 1, "tags": [one, two], cost: 3.14}, baz{id: 2}}}`) + d.Apply(nil) + } +} + +func BenchmarkLatestParse(b *testing.B) { + b.ReportAllocs() + + d := NewDocument(ParseOptions{ + ForceStringKeys: true, + }) + + for n := 0; n < b.N; n++ { + d.Operations = d.Operations[:0] + d.Parse(`{foo{bar{id: 1, "tags": [one, two], cost: 3.14}, baz{id: 2}}}`) + } +} + +func BenchmarkLatestApply(b *testing.B) { + b.ReportAllocs() + d := NewDocument(ParseOptions{ + ForceStringKeys: true, + }) + d.Parse(`{foo{bar{id: 1, "tags": [one, two], cost: 3.14}, baz{id: 2}}}`) + + for n := 0; n < b.N; n++ { + d.Apply(nil) + } +} diff --git a/error.go b/error.go new file mode 100644 index 0000000..a60d14f --- /dev/null +++ b/error.go @@ -0,0 +1,62 @@ +package shorthand + +import "fmt" + +// Error represents an error at a specific location. +type Error interface { + Error() string + + // Offset returns the character offset of the error within the experssion. + Offset() uint + + // Length returns the length in bytes after the offset where the error ends. + Length() uint + + // Pretty prints out a message with a pointer to the source location of the + // error. + Pretty() string +} + +type exprErr struct { + source *string + offset uint + length uint + message string +} + +func (e *exprErr) Error() string { + return e.message +} + +func (e *exprErr) Offset() uint { + return e.offset +} + +func (e *exprErr) Length() uint { + return e.length +} + +func (e *exprErr) Pretty() string { + // TODO: find previous line break if exists, also truncate to e.g. 80 chars, show one line only. Make it dead simple! + msg := e.Error() + "\n" + *e.source + "\n" + for i := uint(0); i < e.offset; i++ { + msg += "." + } + for i := uint(0); i < e.length; i++ { + msg += "^" + } + return msg +} + +// NewError creates a new error at a specific location. +func NewError(source *string, offset uint, length uint, format string, a ...interface{}) Error { + if length < 1 { + length = 1 + } + return &exprErr{ + source: source, + offset: offset, + length: length, + message: fmt.Sprintf(format, a...), + } +} diff --git a/generated.go b/generated.go deleted file mode 100644 index 1fa9213..0000000 --- a/generated.go +++ /dev/null @@ -1,2157 +0,0 @@ -// Code generated by pigeon; DO NOT EDIT. - -// Package shorthand provides a quick way to generate structured data via -// command line parameters. -package shorthand - -import ( - "bytes" - "errors" - "fmt" - "io" - "io/ioutil" - "math" - "os" - "sort" - "strconv" - "strings" - "sync" - "unicode" - "unicode/utf8" -) - -var g = &grammar{ - rules: []*rule{ - { - name: "ShortHand", - pos: position{line: 7, col: 1, offset: 131}, - expr: &actionExpr{ - pos: position{line: 7, col: 13, offset: 145}, - run: (*parser).callonShortHand1, - expr: &seqExpr{ - pos: position{line: 7, col: 13, offset: 145}, - exprs: []interface{}{ - &labeledExpr{ - pos: position{line: 7, col: 13, offset: 145}, - label: "val", - expr: &ruleRefExpr{ - pos: position{line: 7, col: 17, offset: 149}, - name: "Object", - }, - }, - &ruleRefExpr{ - pos: position{line: 7, col: 24, offset: 156}, - name: "EOF", - }, - }, - }, - }, - }, - { - name: "Object", - pos: position{line: 11, col: 1, offset: 183}, - expr: &actionExpr{ - pos: position{line: 11, col: 10, offset: 194}, - run: (*parser).callonObject1, - expr: &seqExpr{ - pos: position{line: 11, col: 10, offset: 194}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 11, col: 10, offset: 194}, - name: "_", - }, - &labeledExpr{ - pos: position{line: 11, col: 12, offset: 196}, - label: "vals", - expr: &seqExpr{ - pos: position{line: 11, col: 18, offset: 202}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 11, col: 18, offset: 202}, - name: "KeyValue", - }, - &zeroOrMoreExpr{ - pos: position{line: 11, col: 27, offset: 211}, - expr: &seqExpr{ - pos: position{line: 11, col: 28, offset: 212}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 11, col: 28, offset: 212}, - val: ",", - ignoreCase: false, - want: "\",\"", - }, - &ruleRefExpr{ - pos: position{line: 11, col: 32, offset: 216}, - name: "_", - }, - &ruleRefExpr{ - pos: position{line: 11, col: 34, offset: 218}, - name: "KeyValue", - }, - }, - }, - }, - }, - }, - }, - &ruleRefExpr{ - pos: position{line: 11, col: 46, offset: 230}, - name: "_", - }, - }, - }, - }, - }, - { - name: "KeyValue", - pos: position{line: 28, col: 1, offset: 547}, - expr: &actionExpr{ - pos: position{line: 28, col: 12, offset: 560}, - run: (*parser).callonKeyValue1, - expr: &seqExpr{ - pos: position{line: 28, col: 12, offset: 560}, - exprs: []interface{}{ - &labeledExpr{ - pos: position{line: 28, col: 12, offset: 560}, - label: "k", - expr: &ruleRefExpr{ - pos: position{line: 28, col: 14, offset: 562}, - name: "Key", - }, - }, - &labeledExpr{ - pos: position{line: 28, col: 18, offset: 566}, - label: "v", - expr: &choiceExpr{ - pos: position{line: 28, col: 21, offset: 569}, - alternatives: []interface{}{ - &seqExpr{ - pos: position{line: 28, col: 22, offset: 570}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 28, col: 22, offset: 570}, - val: ":", - ignoreCase: false, - want: "\":\"", - }, - &zeroOrOneExpr{ - pos: position{line: 28, col: 26, offset: 574}, - expr: &ruleRefExpr{ - pos: position{line: 28, col: 26, offset: 574}, - name: "Modifier", - }, - }, - &ruleRefExpr{ - pos: position{line: 28, col: 36, offset: 584}, - name: "_", - }, - &ruleRefExpr{ - pos: position{line: 28, col: 38, offset: 586}, - name: "Value", - }, - &zeroOrMoreExpr{ - pos: position{line: 28, col: 44, offset: 592}, - expr: &seqExpr{ - pos: position{line: 28, col: 45, offset: 593}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 28, col: 45, offset: 593}, - val: ",", - ignoreCase: false, - want: "\",\"", - }, - &ruleRefExpr{ - pos: position{line: 28, col: 49, offset: 597}, - name: "_", - }, - &ruleRefExpr{ - pos: position{line: 28, col: 51, offset: 599}, - name: "ArrayValue", - }, - &andExpr{ - pos: position{line: 28, col: 62, offset: 610}, - expr: &choiceExpr{ - pos: position{line: 28, col: 64, offset: 612}, - alternatives: []interface{}{ - &charClassMatcher{ - pos: position{line: 28, col: 64, offset: 612}, - val: "[^:]", - chars: []rune{':'}, - ignoreCase: false, - inverted: true, - }, - ¬Expr{ - pos: position{line: 28, col: 71, offset: 619}, - expr: &anyMatcher{ - line: 28, col: 72, offset: 620, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - &seqExpr{ - pos: position{line: 28, col: 81, offset: 629}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 28, col: 81, offset: 629}, - val: "{", - ignoreCase: false, - want: "\"{\"", - }, - &ruleRefExpr{ - pos: position{line: 28, col: 85, offset: 633}, - name: "Object", - }, - &litMatcher{ - pos: position{line: 28, col: 92, offset: 640}, - val: "}", - ignoreCase: false, - want: "\"}\"", - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - { - name: "Key", - pos: position{line: 77, col: 1, offset: 1625}, - expr: &actionExpr{ - pos: position{line: 77, col: 7, offset: 1633}, - run: (*parser).callonKey1, - expr: &labeledExpr{ - pos: position{line: 77, col: 7, offset: 1633}, - label: "parts", - expr: &seqExpr{ - pos: position{line: 77, col: 14, offset: 1640}, - exprs: []interface{}{ - &zeroOrOneExpr{ - pos: position{line: 77, col: 14, offset: 1640}, - expr: &litMatcher{ - pos: position{line: 77, col: 14, offset: 1640}, - val: ".", - ignoreCase: false, - want: "\".\"", - }, - }, - &zeroOrMoreExpr{ - pos: position{line: 77, col: 19, offset: 1645}, - expr: &seqExpr{ - pos: position{line: 77, col: 20, offset: 1646}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 77, col: 20, offset: 1646}, - name: "KeyPart", - }, - &litMatcher{ - pos: position{line: 77, col: 28, offset: 1654}, - val: ".", - ignoreCase: false, - want: "\".\"", - }, - }, - }, - }, - &ruleRefExpr{ - pos: position{line: 77, col: 34, offset: 1660}, - name: "KeyPart", - }, - }, - }, - }, - }, - }, - { - name: "KeyPart", - pos: position{line: 95, col: 1, offset: 1957}, - expr: &actionExpr{ - pos: position{line: 95, col: 11, offset: 1969}, - run: (*parser).callonKeyPart1, - expr: &seqExpr{ - pos: position{line: 95, col: 11, offset: 1969}, - exprs: []interface{}{ - &labeledExpr{ - pos: position{line: 95, col: 11, offset: 1969}, - label: "label", - expr: &ruleRefExpr{ - pos: position{line: 95, col: 17, offset: 1975}, - name: "KeyName", - }, - }, - &labeledExpr{ - pos: position{line: 95, col: 25, offset: 1983}, - label: "index", - expr: &zeroOrMoreExpr{ - pos: position{line: 95, col: 31, offset: 1989}, - expr: &seqExpr{ - pos: position{line: 95, col: 32, offset: 1990}, - exprs: []interface{}{ - &litMatcher{ - pos: position{line: 95, col: 32, offset: 1990}, - val: "[", - ignoreCase: false, - want: "\"[\"", - }, - &zeroOrOneExpr{ - pos: position{line: 95, col: 36, offset: 1994}, - expr: &ruleRefExpr{ - pos: position{line: 95, col: 36, offset: 1994}, - name: "Int", - }, - }, - &litMatcher{ - pos: position{line: 95, col: 41, offset: 1999}, - val: "]", - ignoreCase: false, - want: "\"]\"", - }, - }, - }, - }, - }, - }, - }, - }, - }, - { - name: "KeyName", - pos: position{line: 113, col: 1, offset: 2269}, - expr: &actionExpr{ - pos: position{line: 113, col: 11, offset: 2281}, - run: (*parser).callonKeyName1, - expr: &zeroOrMoreExpr{ - pos: position{line: 113, col: 11, offset: 2281}, - expr: &charClassMatcher{ - pos: position{line: 113, col: 11, offset: 2281}, - val: "[^.:[{]", - chars: []rune{'.', ':', '[', '{'}, - ignoreCase: false, - inverted: true, - }, - }, - }, - }, - { - name: "Modifier", - pos: position{line: 117, col: 1, offset: 2343}, - expr: &actionExpr{ - pos: position{line: 117, col: 12, offset: 2356}, - run: (*parser).callonModifier1, - expr: &charClassMatcher{ - pos: position{line: 117, col: 12, offset: 2356}, - val: "[~]", - chars: []rune{'~'}, - ignoreCase: false, - inverted: false, - }, - }, - }, - { - name: "ArrayValue", - pos: position{line: 126, col: 1, offset: 2505}, - expr: &choiceExpr{ - pos: position{line: 126, col: 14, offset: 2520}, - alternatives: []interface{}{ - &ruleRefExpr{ - pos: position{line: 126, col: 14, offset: 2520}, - name: "Null", - }, - &ruleRefExpr{ - pos: position{line: 126, col: 21, offset: 2527}, - name: "Bool", - }, - &ruleRefExpr{ - pos: position{line: 126, col: 28, offset: 2534}, - name: "Float", - }, - &ruleRefExpr{ - pos: position{line: 126, col: 36, offset: 2542}, - name: "Int", - }, - &ruleRefExpr{ - pos: position{line: 126, col: 42, offset: 2548}, - name: "ArrayString", - }, - }, - }, - }, - { - name: "ArrayString", - pos: position{line: 128, col: 1, offset: 2561}, - expr: &actionExpr{ - pos: position{line: 128, col: 15, offset: 2577}, - run: (*parser).callonArrayString1, - expr: &seqExpr{ - pos: position{line: 128, col: 15, offset: 2577}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 128, col: 15, offset: 2577}, - name: "_", - }, - &charClassMatcher{ - pos: position{line: 128, col: 17, offset: 2579}, - val: "[^[]", - chars: []rune{'['}, - ignoreCase: false, - inverted: true, - }, - &zeroOrMoreExpr{ - pos: position{line: 128, col: 22, offset: 2584}, - expr: &charClassMatcher{ - pos: position{line: 128, col: 22, offset: 2584}, - val: "[^,:}]", - chars: []rune{',', ':', '}'}, - ignoreCase: false, - inverted: true, - }, - }, - }, - }, - }, - }, - { - name: "Value", - pos: position{line: 132, col: 1, offset: 2645}, - expr: &choiceExpr{ - pos: position{line: 132, col: 9, offset: 2655}, - alternatives: []interface{}{ - &ruleRefExpr{ - pos: position{line: 132, col: 9, offset: 2655}, - name: "Null", - }, - &ruleRefExpr{ - pos: position{line: 132, col: 16, offset: 2662}, - name: "Bool", - }, - &ruleRefExpr{ - pos: position{line: 132, col: 23, offset: 2669}, - name: "Float", - }, - &ruleRefExpr{ - pos: position{line: 132, col: 31, offset: 2677}, - name: "Int", - }, - &ruleRefExpr{ - pos: position{line: 132, col: 37, offset: 2683}, - name: "String", - }, - }, - }, - }, - { - name: "ValueEnd", - pos: position{line: 134, col: 1, offset: 2691}, - expr: &choiceExpr{ - pos: position{line: 134, col: 12, offset: 2704}, - alternatives: []interface{}{ - &seqExpr{ - pos: position{line: 134, col: 12, offset: 2704}, - exprs: []interface{}{ - &ruleRefExpr{ - pos: position{line: 134, col: 12, offset: 2704}, - name: "_", - }, - &charClassMatcher{ - pos: position{line: 134, col: 14, offset: 2706}, - val: "[,}\\]]", - chars: []rune{',', '}', ']'}, - ignoreCase: false, - inverted: false, - }, - }, - }, - &ruleRefExpr{ - pos: position{line: 134, col: 23, offset: 2715}, - name: "EOF", - }, - }, - }, - }, - { - name: "Null", - pos: position{line: 136, col: 1, offset: 2720}, - expr: &actionExpr{ - pos: position{line: 136, col: 8, offset: 2729}, - run: (*parser).callonNull1, - expr: &litMatcher{ - pos: position{line: 136, col: 8, offset: 2729}, - val: "null", - ignoreCase: false, - want: "\"null\"", - }, - }, - }, - { - name: "Bool", - pos: position{line: 138, col: 1, offset: 2757}, - expr: &choiceExpr{ - pos: position{line: 138, col: 8, offset: 2766}, - alternatives: []interface{}{ - &actionExpr{ - pos: position{line: 138, col: 8, offset: 2766}, - run: (*parser).callonBool2, - expr: &litMatcher{ - pos: position{line: 138, col: 8, offset: 2766}, - val: "true", - ignoreCase: false, - want: "\"true\"", - }, - }, - &actionExpr{ - pos: position{line: 138, col: 38, offset: 2796}, - run: (*parser).callonBool4, - expr: &litMatcher{ - pos: position{line: 138, col: 38, offset: 2796}, - val: "false", - ignoreCase: false, - want: "\"false\"", - }, - }, - }, - }, - }, - { - name: "Float", - pos: position{line: 140, col: 1, offset: 2827}, - expr: &actionExpr{ - pos: position{line: 140, col: 9, offset: 2837}, - run: (*parser).callonFloat1, - expr: &seqExpr{ - pos: position{line: 140, col: 9, offset: 2837}, - exprs: []interface{}{ - &oneOrMoreExpr{ - pos: position{line: 140, col: 9, offset: 2837}, - expr: &charClassMatcher{ - pos: position{line: 140, col: 9, offset: 2837}, - val: "[0-9]", - ranges: []rune{'0', '9'}, - ignoreCase: false, - inverted: false, - }, - }, - &litMatcher{ - pos: position{line: 140, col: 16, offset: 2844}, - val: ".", - ignoreCase: false, - want: "\".\"", - }, - &oneOrMoreExpr{ - pos: position{line: 140, col: 20, offset: 2848}, - expr: &charClassMatcher{ - pos: position{line: 140, col: 20, offset: 2848}, - val: "[0-9]", - ranges: []rune{'0', '9'}, - ignoreCase: false, - inverted: false, - }, - }, - &andExpr{ - pos: position{line: 140, col: 27, offset: 2855}, - expr: &ruleRefExpr{ - pos: position{line: 140, col: 28, offset: 2856}, - name: "ValueEnd", - }, - }, - }, - }, - }, - }, - { - name: "Int", - pos: position{line: 144, col: 1, offset: 2918}, - expr: &actionExpr{ - pos: position{line: 144, col: 7, offset: 2926}, - run: (*parser).callonInt1, - expr: &seqExpr{ - pos: position{line: 144, col: 7, offset: 2926}, - exprs: []interface{}{ - &oneOrMoreExpr{ - pos: position{line: 144, col: 7, offset: 2926}, - expr: &charClassMatcher{ - pos: position{line: 144, col: 7, offset: 2926}, - val: "[0-9]", - ranges: []rune{'0', '9'}, - ignoreCase: false, - inverted: false, - }, - }, - &andExpr{ - pos: position{line: 144, col: 14, offset: 2933}, - expr: &ruleRefExpr{ - pos: position{line: 144, col: 15, offset: 2934}, - name: "ValueEnd", - }, - }, - }, - }, - }, - }, - { - name: "String", - pos: position{line: 148, col: 1, offset: 2986}, - expr: &actionExpr{ - pos: position{line: 148, col: 10, offset: 2997}, - run: (*parser).callonString1, - expr: &zeroOrMoreExpr{ - pos: position{line: 148, col: 10, offset: 2997}, - expr: &charClassMatcher{ - pos: position{line: 148, col: 10, offset: 2997}, - val: "[^,}]", - chars: []rune{',', '}'}, - ignoreCase: false, - inverted: true, - }, - }, - }, - }, - { - name: "_", - displayName: "\"whitespace\"", - pos: position{line: 152, col: 1, offset: 3057}, - expr: &zeroOrMoreExpr{ - pos: position{line: 152, col: 18, offset: 3076}, - expr: &charClassMatcher{ - pos: position{line: 152, col: 18, offset: 3076}, - val: "[ \\t\\r\\n]", - chars: []rune{' ', '\t', '\r', '\n'}, - ignoreCase: false, - inverted: false, - }, - }, - }, - { - name: "EOF", - pos: position{line: 154, col: 1, offset: 3088}, - expr: ¬Expr{ - pos: position{line: 154, col: 7, offset: 3096}, - expr: &anyMatcher{ - line: 154, col: 8, offset: 3097, - }, - }, - }, - }, -} - -func (c *current) onShortHand1(val interface{}) (interface{}, error) { - return val, nil -} - -func (p *parser) callonShortHand1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onShortHand1(stack["val"]) -} - -func (c *current) onObject1(vals interface{}) (interface{}, error) { - // Collect keys/values into an AST for processing. - ast := AST{} - - valsSl := toIfaceSlice(vals) - ast = append(ast, valsSl[0].(*KeyValue)) - - rest := toIfaceSlice(valsSl[1]) - for _, item := range rest { - itemSl := toIfaceSlice(item) - - ast = append(ast, itemSl[2].(*KeyValue)) - } - - return ast, nil -} - -func (p *parser) callonObject1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onObject1(stack["vals"]) -} - -func (c *current) onKeyValue1(k, v interface{}) (interface{}, error) { - vSl := toIfaceSlice(v) - var value interface{} - modifier := modifierNone - if vSl[0].([]byte)[0] == ':' { - if vSl[1] != nil { - modifier = vSl[1].(int) - } - - value = vSl[3] - - if modifier == modifierString { - if value == nil { - value = "null" - } else { - value = fmt.Sprintf("%v", value) - } - } - - extra := toIfaceSlice(vSl[4]) - if len(extra) > 0 { - values := []interface{}{value} - repeatedWithIndex(extra, 2, func(v interface{}) { - if modifier == modifierString { - v = fmt.Sprintf("%v", v) - } - values = append(values, v) - }) - value = values - } else { - if modifier == modifierString { - if value == nil { - value = "null" - } else { - value = fmt.Sprintf("%v", value) - } - } - } - } else { - value = vSl[1] - } - - return &KeyValue{ - PostProcess: modifier == modifierNone, - Key: k.(*Key), - Value: value, - }, nil -} - -func (p *parser) callonKeyValue1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onKeyValue1(stack["k"], stack["v"]) -} - -func (c *current) onKey1(parts interface{}) (interface{}, error) { - var kps []*KeyPart - - pSl := toIfaceSlice(parts) - for _, sub := range toIfaceSlice(pSl[1]) { - kps = append(kps, toIfaceSlice(sub)[0].(*KeyPart)) - } - - kps = append(kps, pSl[2].(*KeyPart)) - - key := &Key{ - ResetContext: pSl[0] == nil, - Parts: kps, - } - - return key, nil -} - -func (p *parser) callonKey1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onKey1(stack["parts"]) -} - -func (c *current) onKeyPart1(label, index interface{}) (interface{}, error) { - idx := make([]int, 0) - repeatedWithIndex(index, 1, func(v interface{}) { - if v == nil { - idx = append(idx, -1) - return - } - - idx = append(idx, v.(int)) - }) - - kp := &KeyPart{ - Key: label.(string), - Index: idx, - } - return kp, nil -} - -func (p *parser) callonKeyPart1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onKeyPart1(stack["label"], stack["index"]) -} - -func (c *current) onKeyName1() (interface{}, error) { - return strings.TrimSpace(string(c.text)), nil -} - -func (p *parser) callonKeyName1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onKeyName1() -} - -func (c *current) onModifier1() (interface{}, error) { - switch c.text[0] { - case '~': - return modifierString, nil - default: - return nil, fmt.Errorf("Unknown modifier %v", c.text[0]) - } -} - -func (p *parser) callonModifier1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onModifier1() -} - -func (c *current) onArrayString1() (interface{}, error) { - return strings.TrimSpace(string(c.text)), nil -} - -func (p *parser) callonArrayString1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onArrayString1() -} - -func (c *current) onNull1() (interface{}, error) { - return nil, nil -} - -func (p *parser) callonNull1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onNull1() -} - -func (c *current) onBool2() (interface{}, error) { - return true, nil -} - -func (p *parser) callonBool2() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onBool2() -} - -func (c *current) onBool4() (interface{}, error) { - return false, nil -} - -func (p *parser) callonBool4() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onBool4() -} - -func (c *current) onFloat1() (interface{}, error) { - return strconv.ParseFloat(string(c.text), 64) -} - -func (p *parser) callonFloat1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onFloat1() -} - -func (c *current) onInt1() (interface{}, error) { - return strconv.Atoi(string(c.text)) -} - -func (p *parser) callonInt1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onInt1() -} - -func (c *current) onString1() (interface{}, error) { - return strings.TrimSpace(string(c.text)), nil -} - -func (p *parser) callonString1() (interface{}, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onString1() -} - -var ( - // errNoRule is returned when the grammar to parse has no rule. - errNoRule = errors.New("grammar has no rule") - - // errInvalidEntrypoint is returned when the specified entrypoint rule - // does not exit. - errInvalidEntrypoint = errors.New("invalid entrypoint") - - // errInvalidEncoding is returned when the source is not properly - // utf8-encoded. - errInvalidEncoding = errors.New("invalid encoding") - - // errMaxExprCnt is used to signal that the maximum number of - // expressions have been parsed. - errMaxExprCnt = errors.New("max number of expresssions parsed") -) - -// Option is a function that can set an option on the parser. It returns -// the previous setting as an Option. -type Option func(*parser) Option - -// MaxExpressions creates an Option to stop parsing after the provided -// number of expressions have been parsed, if the value is 0 then the parser will -// parse for as many steps as needed (possibly an infinite number). -// -// The default for maxExprCnt is 0. -func MaxExpressions(maxExprCnt uint64) Option { - return func(p *parser) Option { - oldMaxExprCnt := p.maxExprCnt - p.maxExprCnt = maxExprCnt - return MaxExpressions(oldMaxExprCnt) - } -} - -// Entrypoint creates an Option to set the rule name to use as entrypoint. -// The rule name must have been specified in the -alternate-entrypoints -// if generating the parser with the -optimize-grammar flag, otherwise -// it may have been optimized out. Passing an empty string sets the -// entrypoint to the first rule in the grammar. -// -// The default is to start parsing at the first rule in the grammar. -func Entrypoint(ruleName string) Option { - return func(p *parser) Option { - oldEntrypoint := p.entrypoint - p.entrypoint = ruleName - if ruleName == "" { - p.entrypoint = g.rules[0].name - } - return Entrypoint(oldEntrypoint) - } -} - -// Statistics adds a user provided Stats struct to the parser to allow -// the user to process the results after the parsing has finished. -// Also the key for the "no match" counter is set. -// -// Example usage: -// -// input := "input" -// stats := Stats{} -// _, err := Parse("input-file", []byte(input), Statistics(&stats, "no match")) -// if err != nil { -// log.Panicln(err) -// } -// b, err := json.MarshalIndent(stats.ChoiceAltCnt, "", " ") -// if err != nil { -// log.Panicln(err) -// } -// fmt.Println(string(b)) -// -func Statistics(stats *Stats, choiceNoMatch string) Option { - return func(p *parser) Option { - oldStats := p.Stats - p.Stats = stats - oldChoiceNoMatch := p.choiceNoMatch - p.choiceNoMatch = choiceNoMatch - if p.Stats.ChoiceAltCnt == nil { - p.Stats.ChoiceAltCnt = make(map[string]map[string]int) - } - return Statistics(oldStats, oldChoiceNoMatch) - } -} - -// Debug creates an Option to set the debug flag to b. When set to true, -// debugging information is printed to stdout while parsing. -// -// The default is false. -func Debug(b bool) Option { - return func(p *parser) Option { - old := p.debug - p.debug = b - return Debug(old) - } -} - -// Memoize creates an Option to set the memoize flag to b. When set to true, -// the parser will cache all results so each expression is evaluated only -// once. This guarantees linear parsing time even for pathological cases, -// at the expense of more memory and slower times for typical cases. -// -// The default is false. -func Memoize(b bool) Option { - return func(p *parser) Option { - old := p.memoize - p.memoize = b - return Memoize(old) - } -} - -// AllowInvalidUTF8 creates an Option to allow invalid UTF-8 bytes. -// Every invalid UTF-8 byte is treated as a utf8.RuneError (U+FFFD) -// by character class matchers and is matched by the any matcher. -// The returned matched value, c.text and c.offset are NOT affected. -// -// The default is false. -func AllowInvalidUTF8(b bool) Option { - return func(p *parser) Option { - old := p.allowInvalidUTF8 - p.allowInvalidUTF8 = b - return AllowInvalidUTF8(old) - } -} - -// Recover creates an Option to set the recover flag to b. When set to -// true, this causes the parser to recover from panics and convert it -// to an error. Setting it to false can be useful while debugging to -// access the full stack trace. -// -// The default is true. -func Recover(b bool) Option { - return func(p *parser) Option { - old := p.recover - p.recover = b - return Recover(old) - } -} - -// GlobalStore creates an Option to set a key to a certain value in -// the globalStore. -func GlobalStore(key string, value interface{}) Option { - return func(p *parser) Option { - old := p.cur.globalStore[key] - p.cur.globalStore[key] = value - return GlobalStore(key, old) - } -} - -// InitState creates an Option to set a key to a certain value in -// the global "state" store. -func InitState(key string, value interface{}) Option { - return func(p *parser) Option { - old := p.cur.state[key] - p.cur.state[key] = value - return InitState(key, old) - } -} - -// ParseFile parses the file identified by filename. -func ParseFile(filename string, opts ...Option) (i interface{}, err error) { - f, err := os.Open(filename) - if err != nil { - return nil, err - } - defer func() { - if closeErr := f.Close(); closeErr != nil { - err = closeErr - } - }() - return ParseReader(filename, f, opts...) -} - -// ParseReader parses the data from r using filename as information in the -// error messages. -func ParseReader(filename string, r io.Reader, opts ...Option) (interface{}, error) { - b, err := ioutil.ReadAll(r) - if err != nil { - return nil, err - } - - return Parse(filename, b, opts...) -} - -// Parse parses the data from b using filename as information in the -// error messages. -func Parse(filename string, b []byte, opts ...Option) (interface{}, error) { - return newParser(filename, b, opts...).parse(g) -} - -// position records a position in the text. -type position struct { - line, col, offset int -} - -func (p position) String() string { - return strconv.Itoa(p.line) + ":" + strconv.Itoa(p.col) + " [" + strconv.Itoa(p.offset) + "]" -} - -// savepoint stores all state required to go back to this point in the -// parser. -type savepoint struct { - position - rn rune - w int -} - -type current struct { - pos position // start position of the match - text []byte // raw text of the match - - // state is a store for arbitrary key,value pairs that the user wants to be - // tied to the backtracking of the parser. - // This is always rolled back if a parsing rule fails. - state storeDict - - // globalStore is a general store for the user to store arbitrary key-value - // pairs that they need to manage and that they do not want tied to the - // backtracking of the parser. This is only modified by the user and never - // rolled back by the parser. It is always up to the user to keep this in a - // consistent state. - globalStore storeDict -} - -type storeDict map[string]interface{} - -// the AST types... - -type grammar struct { - pos position - rules []*rule -} - -type rule struct { - pos position - name string - displayName string - expr interface{} -} - -type choiceExpr struct { - pos position - alternatives []interface{} -} - -type actionExpr struct { - pos position - expr interface{} - run func(*parser) (interface{}, error) -} - -type recoveryExpr struct { - pos position - expr interface{} - recoverExpr interface{} - failureLabel []string -} - -type seqExpr struct { - pos position - exprs []interface{} -} - -type throwExpr struct { - pos position - label string -} - -type labeledExpr struct { - pos position - label string - expr interface{} -} - -type expr struct { - pos position - expr interface{} -} - -type andExpr expr -type notExpr expr -type zeroOrOneExpr expr -type zeroOrMoreExpr expr -type oneOrMoreExpr expr - -type ruleRefExpr struct { - pos position - name string -} - -type stateCodeExpr struct { - pos position - run func(*parser) error -} - -type andCodeExpr struct { - pos position - run func(*parser) (bool, error) -} - -type notCodeExpr struct { - pos position - run func(*parser) (bool, error) -} - -type litMatcher struct { - pos position - val string - ignoreCase bool - want string -} - -type charClassMatcher struct { - pos position - val string - basicLatinChars [128]bool - chars []rune - ranges []rune - classes []*unicode.RangeTable - ignoreCase bool - inverted bool -} - -type anyMatcher position - -// errList cumulates the errors found by the parser. -type errList []error - -func (e *errList) add(err error) { - *e = append(*e, err) -} - -func (e errList) err() error { - if len(e) == 0 { - return nil - } - e.dedupe() - return e -} - -func (e *errList) dedupe() { - var cleaned []error - set := make(map[string]bool) - for _, err := range *e { - if msg := err.Error(); !set[msg] { - set[msg] = true - cleaned = append(cleaned, err) - } - } - *e = cleaned -} - -func (e errList) Error() string { - switch len(e) { - case 0: - return "" - case 1: - return e[0].Error() - default: - var buf bytes.Buffer - - for i, err := range e { - if i > 0 { - buf.WriteRune('\n') - } - buf.WriteString(err.Error()) - } - return buf.String() - } -} - -// parserError wraps an error with a prefix indicating the rule in which -// the error occurred. The original error is stored in the Inner field. -type parserError struct { - Inner error - pos position - prefix string - expected []string -} - -// Error returns the error message. -func (p *parserError) Error() string { - return p.prefix + ": " + p.Inner.Error() -} - -// newParser creates a parser with the specified input source and options. -func newParser(filename string, b []byte, opts ...Option) *parser { - stats := Stats{ - ChoiceAltCnt: make(map[string]map[string]int), - } - - p := &parser{ - filename: filename, - errs: new(errList), - data: b, - pt: savepoint{position: position{line: 1}}, - recover: true, - cur: current{ - state: make(storeDict), - globalStore: make(storeDict), - }, - maxFailPos: position{col: 1, line: 1}, - maxFailExpected: make([]string, 0, 20), - Stats: &stats, - // start rule is rule [0] unless an alternate entrypoint is specified - entrypoint: g.rules[0].name, - } - p.setOptions(opts) - - if p.maxExprCnt == 0 { - p.maxExprCnt = math.MaxUint64 - } - - return p -} - -// setOptions applies the options to the parser. -func (p *parser) setOptions(opts []Option) { - for _, opt := range opts { - opt(p) - } -} - -type resultTuple struct { - v interface{} - b bool - end savepoint -} - -const choiceNoMatch = -1 - -// Stats stores some statistics, gathered during parsing -type Stats struct { - // ExprCnt counts the number of expressions processed during parsing - // This value is compared to the maximum number of expressions allowed - // (set by the MaxExpressions option). - ExprCnt uint64 - - // ChoiceAltCnt is used to count for each ordered choice expression, - // which alternative is used how may times. - // These numbers allow to optimize the order of the ordered choice expression - // to increase the performance of the parser - // - // The outer key of ChoiceAltCnt is composed of the name of the rule as well - // as the line and the column of the ordered choice. - // The inner key of ChoiceAltCnt is the number (one-based) of the matching alternative. - // For each alternative the number of matches are counted. If an ordered choice does not - // match, a special counter is incremented. The name of this counter is set with - // the parser option Statistics. - // For an alternative to be included in ChoiceAltCnt, it has to match at least once. - ChoiceAltCnt map[string]map[string]int -} - -type parser struct { - filename string - pt savepoint - cur current - - data []byte - errs *errList - - depth int - recover bool - debug bool - - memoize bool - // memoization table for the packrat algorithm: - // map[offset in source] map[expression or rule] {value, match} - memo map[int]map[interface{}]resultTuple - - // rules table, maps the rule identifier to the rule node - rules map[string]*rule - // variables stack, map of label to value - vstack []map[string]interface{} - // rule stack, allows identification of the current rule in errors - rstack []*rule - - // parse fail - maxFailPos position - maxFailExpected []string - maxFailInvertExpected bool - - // max number of expressions to be parsed - maxExprCnt uint64 - // entrypoint for the parser - entrypoint string - - allowInvalidUTF8 bool - - *Stats - - choiceNoMatch string - // recovery expression stack, keeps track of the currently available recovery expression, these are traversed in reverse - recoveryStack []map[string]interface{} -} - -// push a variable set on the vstack. -func (p *parser) pushV() { - if cap(p.vstack) == len(p.vstack) { - // create new empty slot in the stack - p.vstack = append(p.vstack, nil) - } else { - // slice to 1 more - p.vstack = p.vstack[:len(p.vstack)+1] - } - - // get the last args set - m := p.vstack[len(p.vstack)-1] - if m != nil && len(m) == 0 { - // empty map, all good - return - } - - m = make(map[string]interface{}) - p.vstack[len(p.vstack)-1] = m -} - -// pop a variable set from the vstack. -func (p *parser) popV() { - // if the map is not empty, clear it - m := p.vstack[len(p.vstack)-1] - if len(m) > 0 { - // GC that map - p.vstack[len(p.vstack)-1] = nil - } - p.vstack = p.vstack[:len(p.vstack)-1] -} - -// push a recovery expression with its labels to the recoveryStack -func (p *parser) pushRecovery(labels []string, expr interface{}) { - if cap(p.recoveryStack) == len(p.recoveryStack) { - // create new empty slot in the stack - p.recoveryStack = append(p.recoveryStack, nil) - } else { - // slice to 1 more - p.recoveryStack = p.recoveryStack[:len(p.recoveryStack)+1] - } - - m := make(map[string]interface{}, len(labels)) - for _, fl := range labels { - m[fl] = expr - } - p.recoveryStack[len(p.recoveryStack)-1] = m -} - -// pop a recovery expression from the recoveryStack -func (p *parser) popRecovery() { - // GC that map - p.recoveryStack[len(p.recoveryStack)-1] = nil - - p.recoveryStack = p.recoveryStack[:len(p.recoveryStack)-1] -} - -func (p *parser) print(prefix, s string) string { - if !p.debug { - return s - } - - fmt.Printf("%s %d:%d:%d: %s [%#U]\n", - prefix, p.pt.line, p.pt.col, p.pt.offset, s, p.pt.rn) - return s -} - -func (p *parser) in(s string) string { - p.depth++ - return p.print(strings.Repeat(" ", p.depth)+">", s) -} - -func (p *parser) out(s string) string { - p.depth-- - return p.print(strings.Repeat(" ", p.depth)+"<", s) -} - -func (p *parser) addErr(err error) { - p.addErrAt(err, p.pt.position, []string{}) -} - -func (p *parser) addErrAt(err error, pos position, expected []string) { - var buf bytes.Buffer - if p.filename != "" { - buf.WriteString(p.filename) - } - if buf.Len() > 0 { - buf.WriteString(":") - } - buf.WriteString(fmt.Sprintf("%d:%d (%d)", pos.line, pos.col, pos.offset)) - if len(p.rstack) > 0 { - if buf.Len() > 0 { - buf.WriteString(": ") - } - rule := p.rstack[len(p.rstack)-1] - if rule.displayName != "" { - buf.WriteString("rule " + rule.displayName) - } else { - buf.WriteString("rule " + rule.name) - } - } - pe := &parserError{Inner: err, pos: pos, prefix: buf.String(), expected: expected} - p.errs.add(pe) -} - -func (p *parser) failAt(fail bool, pos position, want string) { - // process fail if parsing fails and not inverted or parsing succeeds and invert is set - if fail == p.maxFailInvertExpected { - if pos.offset < p.maxFailPos.offset { - return - } - - if pos.offset > p.maxFailPos.offset { - p.maxFailPos = pos - p.maxFailExpected = p.maxFailExpected[:0] - } - - if p.maxFailInvertExpected { - want = "!" + want - } - p.maxFailExpected = append(p.maxFailExpected, want) - } -} - -// read advances the parser to the next rune. -func (p *parser) read() { - p.pt.offset += p.pt.w - rn, n := utf8.DecodeRune(p.data[p.pt.offset:]) - p.pt.rn = rn - p.pt.w = n - p.pt.col++ - if rn == '\n' { - p.pt.line++ - p.pt.col = 0 - } - - if rn == utf8.RuneError && n == 1 { // see utf8.DecodeRune - if !p.allowInvalidUTF8 { - p.addErr(errInvalidEncoding) - } - } -} - -// restore parser position to the savepoint pt. -func (p *parser) restore(pt savepoint) { - if p.debug { - defer p.out(p.in("restore")) - } - if pt.offset == p.pt.offset { - return - } - p.pt = pt -} - -// Cloner is implemented by any value that has a Clone method, which returns a -// copy of the value. This is mainly used for types which are not passed by -// value (e.g map, slice, chan) or structs that contain such types. -// -// This is used in conjunction with the global state feature to create proper -// copies of the state to allow the parser to properly restore the state in -// the case of backtracking. -type Cloner interface { - Clone() interface{} -} - -var statePool = &sync.Pool{ - New: func() interface{} { return make(storeDict) }, -} - -func (sd storeDict) Discard() { - for k := range sd { - delete(sd, k) - } - statePool.Put(sd) -} - -// clone and return parser current state. -func (p *parser) cloneState() storeDict { - if p.debug { - defer p.out(p.in("cloneState")) - } - - state := statePool.Get().(storeDict) - for k, v := range p.cur.state { - if c, ok := v.(Cloner); ok { - state[k] = c.Clone() - } else { - state[k] = v - } - } - return state -} - -// restore parser current state to the state storeDict. -// every restoreState should applied only one time for every cloned state -func (p *parser) restoreState(state storeDict) { - if p.debug { - defer p.out(p.in("restoreState")) - } - p.cur.state.Discard() - p.cur.state = state -} - -// get the slice of bytes from the savepoint start to the current position. -func (p *parser) sliceFrom(start savepoint) []byte { - return p.data[start.position.offset:p.pt.position.offset] -} - -func (p *parser) getMemoized(node interface{}) (resultTuple, bool) { - if len(p.memo) == 0 { - return resultTuple{}, false - } - m := p.memo[p.pt.offset] - if len(m) == 0 { - return resultTuple{}, false - } - res, ok := m[node] - return res, ok -} - -func (p *parser) setMemoized(pt savepoint, node interface{}, tuple resultTuple) { - if p.memo == nil { - p.memo = make(map[int]map[interface{}]resultTuple) - } - m := p.memo[pt.offset] - if m == nil { - m = make(map[interface{}]resultTuple) - p.memo[pt.offset] = m - } - m[node] = tuple -} - -func (p *parser) buildRulesTable(g *grammar) { - p.rules = make(map[string]*rule, len(g.rules)) - for _, r := range g.rules { - p.rules[r.name] = r - } -} - -func (p *parser) parse(g *grammar) (val interface{}, err error) { - if len(g.rules) == 0 { - p.addErr(errNoRule) - return nil, p.errs.err() - } - - // TODO : not super critical but this could be generated - p.buildRulesTable(g) - - if p.recover { - // panic can be used in action code to stop parsing immediately - // and return the panic as an error. - defer func() { - if e := recover(); e != nil { - if p.debug { - defer p.out(p.in("panic handler")) - } - val = nil - switch e := e.(type) { - case error: - p.addErr(e) - default: - p.addErr(fmt.Errorf("%v", e)) - } - err = p.errs.err() - } - }() - } - - startRule, ok := p.rules[p.entrypoint] - if !ok { - p.addErr(errInvalidEntrypoint) - return nil, p.errs.err() - } - - p.read() // advance to first rune - val, ok = p.parseRule(startRule) - if !ok { - if len(*p.errs) == 0 { - // If parsing fails, but no errors have been recorded, the expected values - // for the farthest parser position are returned as error. - maxFailExpectedMap := make(map[string]struct{}, len(p.maxFailExpected)) - for _, v := range p.maxFailExpected { - maxFailExpectedMap[v] = struct{}{} - } - expected := make([]string, 0, len(maxFailExpectedMap)) - eof := false - if _, ok := maxFailExpectedMap["!."]; ok { - delete(maxFailExpectedMap, "!.") - eof = true - } - for k := range maxFailExpectedMap { - expected = append(expected, k) - } - sort.Strings(expected) - if eof { - expected = append(expected, "EOF") - } - p.addErrAt(errors.New("no match found, expected: "+listJoin(expected, ", ", "or")), p.maxFailPos, expected) - } - - return nil, p.errs.err() - } - return val, p.errs.err() -} - -func listJoin(list []string, sep string, lastSep string) string { - switch len(list) { - case 0: - return "" - case 1: - return list[0] - default: - return strings.Join(list[:len(list)-1], sep) + " " + lastSep + " " + list[len(list)-1] - } -} - -func (p *parser) parseRule(rule *rule) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseRule " + rule.name)) - } - - if p.memoize { - res, ok := p.getMemoized(rule) - if ok { - p.restore(res.end) - return res.v, res.b - } - } - - start := p.pt - p.rstack = append(p.rstack, rule) - p.pushV() - val, ok := p.parseExpr(rule.expr) - p.popV() - p.rstack = p.rstack[:len(p.rstack)-1] - if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) - } - - if p.memoize { - p.setMemoized(start, rule, resultTuple{val, ok, p.pt}) - } - return val, ok -} - -func (p *parser) parseExpr(expr interface{}) (interface{}, bool) { - var pt savepoint - - if p.memoize { - res, ok := p.getMemoized(expr) - if ok { - p.restore(res.end) - return res.v, res.b - } - pt = p.pt - } - - p.ExprCnt++ - if p.ExprCnt > p.maxExprCnt { - panic(errMaxExprCnt) - } - - var val interface{} - var ok bool - switch expr := expr.(type) { - case *actionExpr: - val, ok = p.parseActionExpr(expr) - case *andCodeExpr: - val, ok = p.parseAndCodeExpr(expr) - case *andExpr: - val, ok = p.parseAndExpr(expr) - case *anyMatcher: - val, ok = p.parseAnyMatcher(expr) - case *charClassMatcher: - val, ok = p.parseCharClassMatcher(expr) - case *choiceExpr: - val, ok = p.parseChoiceExpr(expr) - case *labeledExpr: - val, ok = p.parseLabeledExpr(expr) - case *litMatcher: - val, ok = p.parseLitMatcher(expr) - case *notCodeExpr: - val, ok = p.parseNotCodeExpr(expr) - case *notExpr: - val, ok = p.parseNotExpr(expr) - case *oneOrMoreExpr: - val, ok = p.parseOneOrMoreExpr(expr) - case *recoveryExpr: - val, ok = p.parseRecoveryExpr(expr) - case *ruleRefExpr: - val, ok = p.parseRuleRefExpr(expr) - case *seqExpr: - val, ok = p.parseSeqExpr(expr) - case *stateCodeExpr: - val, ok = p.parseStateCodeExpr(expr) - case *throwExpr: - val, ok = p.parseThrowExpr(expr) - case *zeroOrMoreExpr: - val, ok = p.parseZeroOrMoreExpr(expr) - case *zeroOrOneExpr: - val, ok = p.parseZeroOrOneExpr(expr) - default: - panic(fmt.Sprintf("unknown expression type %T", expr)) - } - if p.memoize { - p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) - } - return val, ok -} - -func (p *parser) parseActionExpr(act *actionExpr) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseActionExpr")) - } - - start := p.pt - val, ok := p.parseExpr(act.expr) - if ok { - p.cur.pos = start.position - p.cur.text = p.sliceFrom(start) - state := p.cloneState() - actVal, err := act.run(p) - if err != nil { - p.addErrAt(err, start.position, []string{}) - } - p.restoreState(state) - - val = actVal - } - if ok && p.debug { - p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) - } - return val, ok -} - -func (p *parser) parseAndCodeExpr(and *andCodeExpr) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseAndCodeExpr")) - } - - state := p.cloneState() - - ok, err := and.run(p) - if err != nil { - p.addErr(err) - } - p.restoreState(state) - - return nil, ok -} - -func (p *parser) parseAndExpr(and *andExpr) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseAndExpr")) - } - - pt := p.pt - state := p.cloneState() - p.pushV() - _, ok := p.parseExpr(and.expr) - p.popV() - p.restoreState(state) - p.restore(pt) - - return nil, ok -} - -func (p *parser) parseAnyMatcher(any *anyMatcher) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseAnyMatcher")) - } - - if p.pt.rn == utf8.RuneError && p.pt.w == 0 { - // EOF - see utf8.DecodeRune - p.failAt(false, p.pt.position, ".") - return nil, false - } - start := p.pt - p.read() - p.failAt(true, start.position, ".") - return p.sliceFrom(start), true -} - -func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseCharClassMatcher")) - } - - cur := p.pt.rn - start := p.pt - - // can't match EOF - if cur == utf8.RuneError && p.pt.w == 0 { // see utf8.DecodeRune - p.failAt(false, start.position, chr.val) - return nil, false - } - - if chr.ignoreCase { - cur = unicode.ToLower(cur) - } - - // try to match in the list of available chars - for _, rn := range chr.chars { - if rn == cur { - if chr.inverted { - p.failAt(false, start.position, chr.val) - return nil, false - } - p.read() - p.failAt(true, start.position, chr.val) - return p.sliceFrom(start), true - } - } - - // try to match in the list of ranges - for i := 0; i < len(chr.ranges); i += 2 { - if cur >= chr.ranges[i] && cur <= chr.ranges[i+1] { - if chr.inverted { - p.failAt(false, start.position, chr.val) - return nil, false - } - p.read() - p.failAt(true, start.position, chr.val) - return p.sliceFrom(start), true - } - } - - // try to match in the list of Unicode classes - for _, cl := range chr.classes { - if unicode.Is(cl, cur) { - if chr.inverted { - p.failAt(false, start.position, chr.val) - return nil, false - } - p.read() - p.failAt(true, start.position, chr.val) - return p.sliceFrom(start), true - } - } - - if chr.inverted { - p.read() - p.failAt(true, start.position, chr.val) - return p.sliceFrom(start), true - } - p.failAt(false, start.position, chr.val) - return nil, false -} - -func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { - choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) - m := p.ChoiceAltCnt[choiceIdent] - if m == nil { - m = make(map[string]int) - p.ChoiceAltCnt[choiceIdent] = m - } - // We increment altI by 1, so the keys do not start at 0 - alt := strconv.Itoa(altI + 1) - if altI == choiceNoMatch { - alt = p.choiceNoMatch - } - m[alt]++ -} - -func (p *parser) parseChoiceExpr(ch *choiceExpr) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseChoiceExpr")) - } - - for altI, alt := range ch.alternatives { - // dummy assignment to prevent compile error if optimized - _ = altI - - state := p.cloneState() - - p.pushV() - val, ok := p.parseExpr(alt) - p.popV() - if ok { - p.incChoiceAltCnt(ch, altI) - return val, ok - } - p.restoreState(state) - } - p.incChoiceAltCnt(ch, choiceNoMatch) - return nil, false -} - -func (p *parser) parseLabeledExpr(lab *labeledExpr) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseLabeledExpr")) - } - - p.pushV() - val, ok := p.parseExpr(lab.expr) - p.popV() - if ok && lab.label != "" { - m := p.vstack[len(p.vstack)-1] - m[lab.label] = val - } - return val, ok -} - -func (p *parser) parseLitMatcher(lit *litMatcher) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseLitMatcher")) - } - - start := p.pt - for _, want := range lit.val { - cur := p.pt.rn - if lit.ignoreCase { - cur = unicode.ToLower(cur) - } - if cur != want { - p.failAt(false, start.position, lit.want) - p.restore(start) - return nil, false - } - p.read() - } - p.failAt(true, start.position, lit.want) - return p.sliceFrom(start), true -} - -func (p *parser) parseNotCodeExpr(not *notCodeExpr) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseNotCodeExpr")) - } - - state := p.cloneState() - - ok, err := not.run(p) - if err != nil { - p.addErr(err) - } - p.restoreState(state) - - return nil, !ok -} - -func (p *parser) parseNotExpr(not *notExpr) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseNotExpr")) - } - - pt := p.pt - state := p.cloneState() - p.pushV() - p.maxFailInvertExpected = !p.maxFailInvertExpected - _, ok := p.parseExpr(not.expr) - p.maxFailInvertExpected = !p.maxFailInvertExpected - p.popV() - p.restoreState(state) - p.restore(pt) - - return nil, !ok -} - -func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseOneOrMoreExpr")) - } - - var vals []interface{} - - for { - p.pushV() - val, ok := p.parseExpr(expr.expr) - p.popV() - if !ok { - if len(vals) == 0 { - // did not match once, no match - return nil, false - } - return vals, true - } - vals = append(vals, val) - } -} - -func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseRecoveryExpr (" + strings.Join(recover.failureLabel, ",") + ")")) - } - - p.pushRecovery(recover.failureLabel, recover.recoverExpr) - val, ok := p.parseExpr(recover.expr) - p.popRecovery() - - return val, ok -} - -func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseRuleRefExpr " + ref.name)) - } - - if ref.name == "" { - panic(fmt.Sprintf("%s: invalid rule: missing name", ref.pos)) - } - - rule := p.rules[ref.name] - if rule == nil { - p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) - return nil, false - } - return p.parseRule(rule) -} - -func (p *parser) parseSeqExpr(seq *seqExpr) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseSeqExpr")) - } - - vals := make([]interface{}, 0, len(seq.exprs)) - - pt := p.pt - state := p.cloneState() - for _, expr := range seq.exprs { - val, ok := p.parseExpr(expr) - if !ok { - p.restoreState(state) - p.restore(pt) - return nil, false - } - vals = append(vals, val) - } - return vals, true -} - -func (p *parser) parseStateCodeExpr(state *stateCodeExpr) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseStateCodeExpr")) - } - - err := state.run(p) - if err != nil { - p.addErr(err) - } - return nil, true -} - -func (p *parser) parseThrowExpr(expr *throwExpr) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseThrowExpr")) - } - - for i := len(p.recoveryStack) - 1; i >= 0; i-- { - if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { - if val, ok := p.parseExpr(recoverExpr); ok { - return val, ok - } - } - } - - return nil, false -} - -func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseZeroOrMoreExpr")) - } - - var vals []interface{} - - for { - p.pushV() - val, ok := p.parseExpr(expr.expr) - p.popV() - if !ok { - return vals, true - } - vals = append(vals, val) - } -} - -func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (interface{}, bool) { - if p.debug { - defer p.out(p.in("parseZeroOrOneExpr")) - } - - p.pushV() - val, _ := p.parseExpr(expr.expr) - p.popV() - // whether it matched or not, consider it a match - return val, true -} diff --git a/get.go b/get.go new file mode 100644 index 0000000..f001405 --- /dev/null +++ b/get.go @@ -0,0 +1,383 @@ +package shorthand + +import ( + "strconv" + "strings" + + "github.com/danielgtaylor/mexpr" +) + +type GetOptions struct { + // DebugLogger sets a function to be used for printing out debug information. + DebugLogger func(format string, a ...interface{}) +} + +func GetPath(path string, input any, options GetOptions) (any, bool, Error) { + d := Document{ + expression: path, + options: ParseOptions{ + DebugLogger: options.DebugLogger, + }, + } + result := input + var ok bool + var err Error + for d.pos < uint(len(d.expression)) { + result, ok, err = d.getPath(result) + if err != nil { + return result, ok, err + } + if d.peek() == '|' { + d.next() + } + } + return result, ok, nil +} + +func (d *Document) parseUntil(open int, terminators ...rune) (quoted bool, canSlice bool, err Error) { + canSlice = true + d.buf.Reset() + +outer: + for { + p := d.peek() + if p == -1 { + break outer + } + + if p == '[' || p == '{' { + open++ + } else if p == ']' || p == '}' { + open-- + if open == 0 { + break outer + } + } + + for _, t := range terminators { + if p == t { + break outer + } + } + + r := d.next() + + if r == '\\' { + if d.parseEscape(false, false) { + canSlice = false + continue + } + } + + if r == '"' { + if err = d.parseQuoted(false); err != nil { + return + } + quoted = true + continue + } + + d.buf.WriteRune(r) + } + + return +} + +func (d *Document) parsePathIndex() (bool, int, int, string, Error) { + d.skipWhitespace() + start := d.pos + _, canSlice, err := d.parseUntil(1, '|') + if err != nil { + return false, 0, 0, "", err + } + + var value string + if canSlice { + value = d.expression[start:d.pos] + } else { + value = d.buf.String() + } + + if !d.expect(']') { + return false, 0, 0, "", d.error(d.pos-start, "expected ']' after index or filter") + } + + if len(value) > 0 { + indexes := strings.Split(value, ":") + if len(indexes) == 1 { + if index, err := strconv.Atoi(value); err == nil { + return false, index, index, "", nil + } + } else { + if startIndex, err := strconv.Atoi(indexes[0]); err == nil { + if stopIndex, err := strconv.Atoi(indexes[1]); err == nil { + return true, startIndex, stopIndex, "", nil + } + } + } + } + + return false, 0, 0, value, nil +} + +func (d *Document) getFiltered(expr string, input any) (any, Error) { + ast, err := mexpr.Parse(expr, nil) + if err != nil { + return nil, NewError(&d.expression, d.pos+uint(err.Offset()), uint(err.Length()), err.Error()) + } + interpreter := mexpr.NewInterpreter(ast, mexpr.UnquotedStrings) + savedPos := d.pos + + if s, ok := input.([]any); ok { + results := []any{} + for _, item := range s { + result, err := interpreter.Run(item) + if err != nil { + continue + } + if b, ok := result.(bool); ok && b { + out := item + + var err Error + d.pos = savedPos + out, _, err = d.getPath(item) + if err != nil { + return nil, err + } + results = append(results, out) + } + } + return results, nil + } + return nil, nil +} + +func (d *Document) getIndex2(input any) (any, Error) { + isSlice, startIndex, stopIndex, expr, err := d.parsePathIndex() + if err != nil { + return nil, err + } + + if expr != "" { + return d.getFiltered(expr, input) + } + + if s, ok := input.([]any); ok { + if startIndex > len(s)-1 || stopIndex > len(s)-1 { + return nil, nil + } + for startIndex < 0 { + startIndex += len(s) + } + for stopIndex < 0 { + stopIndex += len(s) + } + + if !isSlice { + return s[startIndex], nil + } + + return s[startIndex : stopIndex+1], nil + } + + return nil, nil +} + +func (d *Document) parseProp2() (any, Error) { + d.skipWhitespace() + start := d.pos + quoted, canSlice, err := d.parseUntil(0, '.', '[', '|', ',', '}') + if err != nil { + return nil, err + } + + var key string + if canSlice { + key = d.expression[start:d.pos] + } else { + key = d.buf.String() + } + + if !d.options.ForceStringKeys && !quoted { + if v, ok := coerceValue(key); ok { + return v, nil + } + } + + return key, nil +} + +func (d *Document) getProp(input any) (any, bool, Error) { + if s, ok := input.([]any); ok { + var err Error + savedPos := d.pos + out := make([]any, len(s)) + + for i := range s { + d.pos = savedPos + out[i], _, err = d.getPath(s[i]) + if err != nil { + return nil, false, err + } + } + + return out, true, nil + } + + key, err := d.parseProp2() + if err != nil { + return nil, false, err + } + + if d.options.DebugLogger != nil { + d.options.DebugLogger("Getting key '%v'", key) + } + + if m, ok := input.(map[string]any); ok { + if s, ok := key.(string); ok { + v, ok := m[s] + return v, ok, nil + } + } else if m, ok := input.(map[any]any); ok { + v, ok := m[key] + return v, ok, nil + } + + return nil, false, nil +} + +func (d *Document) flatten(input any) (any, Error) { + if s, ok := input.([]any); ok { + out := make([]any, 0, len(s)) + + for _, item := range s { + if !isArray(item) { + out = append(out, item) + continue + } + + out = append(out, item.([]any)...) + } + return out, nil + } + return nil, nil +} + +func (d *Document) getPath(input any) (any, bool, Error) { + var err Error + found := false + +outer: + for { + switch d.peek() { + case -1, '|': + break outer + case '[': + d.next() + if d.peek() == ']' { + // Special case: flatten one level + // [[1, 2], 3, [[4]]] => [1, 2, 3, [4]] + d.next() + input, err = d.flatten(input) + if err != nil { + return nil, false, err + } + found = true + continue + } + input, err = d.getIndex2(input) + if err != nil { + return nil, false, err + } + found = true + case '.': + d.next() + continue + case '{': + d.next() + input, err = d.getFields(input) + if err != nil { + return nil, false, err + } + found = true + continue + default: + input, found, err = d.getProp(input) + if err != nil { + return nil, false, err + } + } + } + + return input, found, nil +} + +func (d *Document) getFields(input any) (any, Error) { + d.buf.Reset() + if !isMap(input) { + return nil, d.error(1, "field selection requires a map, but found %v", input) + } + result := map[string]any{} + key := "" + open := 1 + var r rune + d.skipWhitespace() + for { + r = d.next() + if r == '"' { + d.buf.WriteRune('"') + if err := d.parseQuoted(true); err != nil { + return nil, err + } + d.buf.WriteRune('"') + continue + } + if r == '\\' { + if d.parseEscape(false, true) { + continue + } + } + if r == -1 { + break + } + if r == ':' { + key = d.buf.String() + d.buf.Reset() + d.skipWhitespace() + continue + } + if r == '{' { + open++ + } + if r == '}' { + open-- + } + if r == ',' || open == 0 { + path := d.buf.String() + if m, ok := input.(map[string]any); ok { + if key == "" { + result[path] = m[path] + } else { + expr, pos := d.expression, d.pos + tmp, _, err := GetPath(path, input, GetOptions{ + DebugLogger: d.options.DebugLogger, + }) + d.expression, d.pos = expr, pos + if err != nil { + return nil, err + } + result[key] = tmp + } + } + if r == '}' { + break + } + key = "" + d.buf.Reset() + d.skipWhitespace() + continue + } + d.buf.WriteRune(r) + } + return result, nil +} diff --git a/get_test.go b/get_test.go new file mode 100644 index 0000000..dd0cee8 --- /dev/null +++ b/get_test.go @@ -0,0 +1,153 @@ +package shorthand + +import ( + "encoding/json" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var getExamples = []struct { + Name string + Input interface{} + Query string + Error string + Go interface{} + JSON string +}{ + { + Name: "Field", + Input: `{"field": "value"}`, + Query: "field", + Go: "value", + }, + { + Name: "Nested fields", + Input: `{"f1": {"f2": {"f3": true}}}`, + Query: `f1.f2.f3`, + Go: true, + }, + { + Name: "Array index", + Input: `{"field": [1, 2, 3]}`, + Query: `field[0]`, + Go: 1.0, + }, + { + Name: "Array index out of bounds", + Input: `{"field": [1, 2, 3]}`, + Query: `field[5]`, + Go: nil, + }, + { + Name: "Array index nested", + Input: `{"field": [null, [[1]]]}`, + Query: `field[1][0][0]`, + Go: 1.0, + }, + { + Name: "Array item fields", + Input: `{"items": [{"f1": {"f2": 1}}, {"f1": {"f2": 2}}, {"other": 3}]}`, + Query: `items.f1.f2`, + Go: []any{1.0, 2.0, nil}, + }, + { + Name: "Array item fields empty index", + Input: `{"items": [{"f1": {"f2": 1}}, {"f1": {"f2": 2}}, {"other": 3}]}`, + Query: `items[].f1.f2`, + Go: []any{1.0, 2.0, nil}, + }, + { + Name: "Array item scalar filtering", + Input: `{"items": ["a", "b", "c"]}`, + Query: `items[@ startsWith a]`, + Go: []any{"a"}, + }, + { + Name: "Array item filtering", + Input: `{"items": [{"f1": {"f2": 1}}, {"f1": {"f2": 2}}, {"other": 3}]}`, + Query: `items[f1 and f1.f2 > 1].f1.f2`, + Go: []any{2.0}, + }, + { + Name: "Array filtering first match", + Input: `{"items": ["a", "b", "c"]}`, + Query: `items[@ startsWith a]|[0]`, + Go: "a", + }, + { + Name: "Field selection", + Input: `{"link": {"id": 1, "verified": true, "tags": ["a", "b"]}}`, + Query: `link.{id, tags}`, + Go: map[string]any{"id": 1.0, "tags": []any{"a", "b"}}, + }, + { + Name: "Field expression", + Input: `{"foo": "bar", "link": {"id": 1, "verified": true, "tags": ["a", "b"]}}`, + Query: `{foo, id: link.id, tags: link.tags[@ startsWith a]}`, + Go: map[string]any{"foo": "bar", "id": 1.0, "tags": []any{"a"}}, + }, + { + Name: "Field expression with pipe", + Input: `{"foo": "bar", "link": {"id": 1, "verified": true, "tags": ["a", "b"]}}`, + Query: `{foo, tags: link.tags[@ startsWith a]|[0], id: link.id}`, + Go: map[string]any{"foo": "bar", "id": 1.0, "tags": "a"}, + }, +} + +func TestGet(t *testing.T) { + for _, example := range getExamples { + t.Run(example.Name, func(t *testing.T) { + t.Logf("Input: %s", example.Input) + input := example.Input + if s, ok := input.(string); ok { + require.NoError(t, json.Unmarshal([]byte(s), &input)) + } + result, _, err := GetPath(example.Query, input, GetOptions{DebugLogger: t.Logf}) + + if example.Error == "" { + msg := "" + if err != nil { + msg = err.Pretty() + } + require.NoError(t, err, msg) + } else { + require.Error(t, err, "result is %v", result) + require.Contains(t, err.Error(), example.Error) + } + + if example.Go != nil { + assert.Equal(t, example.Go, result) + } + + if example.JSON != "" { + result = ConvertMapString(result) + b, _ := json.Marshal(result) + assert.JSONEq(t, example.JSON, string(b)) + } + }) + } +} + +func FuzzGet(f *testing.F) { + f.Fuzz(func(t *testing.T, s string) { + data := map[string]any{ + "n": nil, + "b": true, + "i": 123, + "f": 4.5, + "s": "hello", + "b2": []byte{0, 1, 2}, + "d": time.Now(), + "a": []any{1, 2.5, "foo"}, + "aa": []any{[]any{[]any{1, 2, 3}}}, + "am": []any{map[string]any{"a": []any{1, 2, 3}}}, + "m": map[any]any{ + 1: true, + }, + } + GetPath(s, data, GetOptions{DebugLogger: t.Logf}) + }) +} diff --git a/go.mod b/go.mod index 881bb1b..3cd10be 100644 --- a/go.mod +++ b/go.mod @@ -1,14 +1,24 @@ module github.com/danielgtaylor/shorthand -go 1.16 +go 1.18 + +replace github.com/danielgtaylor/mexpr => ../../mexpr require ( - github.com/mna/pigeon v1.1.0 // indirect + github.com/fxamacker/cbor/v2 v2.4.0 github.com/pelletier/go-toml v1.9.4 github.com/spf13/cobra v1.2.1 github.com/stretchr/testify v1.7.0 - golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5 // indirect - golang.org/x/tools v0.1.9 // indirect - gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b ) + +require ( + github.com/danielgtaylor/mexpr v1.7.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/x448/float16 v0.8.4 // indirect + golang.org/x/exp v0.0.0-20220518171630-0b5c67f07fdf // indirect + gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect +) diff --git a/go.sum b/go.sum index 8ca554f..68f0622 100644 --- a/go.sum +++ b/go.sum @@ -56,6 +56,8 @@ github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnht github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/danielgtaylor/mexpr v1.5.1 h1:sSlycueushuMlcb/8bB3fTcMRhLWmPa9XMKyr5WQDvU= +github.com/danielgtaylor/mexpr v1.5.1/go.mod h1:xQ64V12CB+4K0wb1na+MtWSNQsPgjAyyHT4wTM7mM0I= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -68,6 +70,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fxamacker/cbor/v2 v2.4.0 h1:ri0ArlOR+5XunOP8CRUowT0pSJOwhW098ZCUyskZD88= +github.com/fxamacker/cbor/v2 v2.4.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -185,8 +189,6 @@ github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0Qu github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mna/pigeon v1.1.0 h1:EjlvVbkGnNGemf8OrjeJX0nH8orujY/HkJgzJtd7kxc= -github.com/mna/pigeon v1.1.0/go.mod h1:rkFeDZ0gc+YbnrXPw0q2RlI0QRuKBBPu67fgYIyGRNg= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -225,12 +227,13 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= @@ -261,6 +264,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20220518171630-0b5c67f07fdf h1:oXVg4h2qJDd9htKxb5SCpFBHLipW6hXmL3qpUixS2jw= +golang.org/x/exp v0.0.0-20220518171630-0b5c67f07fdf/go.mod h1:yh0Ynu2b5ZUe3MQfp2nM0ecK7wsgouWTDN0FNeJuIys= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -286,8 +291,6 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38= -golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -324,7 +327,6 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -388,11 +390,7 @@ golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5 h1:y/woIyUBFbpQGKS0u1aHF/40WUDnek3fPOyD08H5Vng= -golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -401,8 +399,6 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -420,7 +416,6 @@ golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190830223141-573d9926052a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -458,12 +453,9 @@ golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.9 h1:j9KsMiaP1c3B0OTQGth0/k+miLGTgLsAFUCrF2vLcF8= -golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= diff --git a/parse.go b/parse.go new file mode 100644 index 0000000..0bcbc74 --- /dev/null +++ b/parse.go @@ -0,0 +1,584 @@ +package shorthand + +import ( + "encoding/base64" + "encoding/json" + "io/ioutil" + "strconv" + "strings" + "time" + "unicode" + "unicode/utf8" + + "github.com/fxamacker/cbor/v2" +) + +var JSONReplacements = map[rune]rune{ + '"': '"', + '\\': '\\', + '/': '/', + 'b': '\b', + 'f': '\f', + 'n': '\n', + 'r': '\r', + 't': '\t', +} + +// runeStr returns a rune as a string, taking care to handle -1 as end-of-file. +func runeStr(r rune) string { + if r == -1 { + return "EOF" + } + return string(r) +} + +func canCoerce(value string) bool { + if value == "null" { + return true + } else if value == "true" { + return true + } else if value == "false" { + return true + } else if len(value) >= 10 && value[0] >= '0' && value[0] <= '9' && value[3] >= '0' && value[3] <= '9' && value[4] == '-' && value[7] == '-' { + return true + } else if len(value) > 0 && value[0] >= '0' && value[0] <= '9' { + return true + } + return false +} + +func coerceValue(value string) (any, bool) { + if value == "null" { + return nil, true + } else if value == "true" { + return true, true + } else if value == "false" { + return false, true + } else if len(value) >= 10 && value[0] >= '0' && value[0] <= '9' && value[3] >= '0' && value[3] <= '9' && value[4] == '-' && value[7] == '-' { + // This looks date or time-like. + if t, err := time.Parse(time.RFC3339Nano, value); err == nil { + return t, true + } + } else if len(value) > 0 && value[0] >= '0' && value[0] <= '9' { + // This looks like a number. + isFloat := false + for _, r := range value { + if r == '.' || r == 'e' || r == 'E' { + isFloat = true + break + } + } + if isFloat { + if f, err := strconv.ParseFloat(value, 64); err == nil { + return f, true + } + } else if i, err := strconv.Atoi(value); err == nil { + return i, true + } + } + return nil, false +} + +// next returns the next rune in the expression at the current position. +func (d *Document) next() rune { + if d.pos >= uint(len(d.expression)) { + d.lastWidth = 0 + return -1 + } + + var r rune + if d.expression[d.pos] < utf8.RuneSelf { + // Optimization for a simple ASCII character + r = rune(d.expression[d.pos]) + d.pos += 1 + d.lastWidth = 1 + } else { + var w int + r, w = utf8.DecodeRuneInString(d.expression[d.pos:]) + d.pos += uint(w) + d.lastWidth = uint(w) + } + + return r +} + +// Back moves back one rune. +func (d *Document) back() { + d.pos -= d.lastWidth +} + +// peek returns the next rune without moving the position forward. +func (d *Document) peek() rune { + r := d.next() + d.back() + return r +} + +// expect returns true if the next value is the given value, otherwise false. +// ignores whitespace. +func (d *Document) expect(value rune) bool { + d.skipWhitespace() + peek := d.peek() + if peek == value { + d.next() + return true + } + return false +} + +func (d *Document) error(length uint, format string, a ...any) Error { + return NewError(&d.expression, d.pos, length, format, a...) +} + +func (d *Document) skipWhitespace() { + for { + peek := d.peek() + if unicode.IsSpace(peek) { + d.next() + continue + } + break + } +} + +func (d *Document) parseEscape(quoted bool, includeEscape bool) bool { + peek := d.peek() + if !quoted { + if peek == '.' || peek == '{' || peek == '[' || peek == ':' || peek == '^' || peek == ']' || peek == ',' { + d.next() + if includeEscape { + d.buf.WriteRune('\\') + } + d.buf.WriteRune(peek) + return true + } + } else { + if peek == '"' { + d.next() + if includeEscape { + d.buf.WriteRune('\\') + } + d.buf.WriteRune(peek) + return true + } + } + + if replace, ok := JSONReplacements[peek]; ok { + d.next() + d.buf.WriteRune(replace) + return true + } + if peek == 'u' && len(d.expression) >= int(d.pos)+5 { + if s, err := strconv.Unquote(`"` + d.expression[d.pos-1:d.pos+5] + `"`); err == nil { + d.buf.WriteString(s) + d.next() + d.next() + d.next() + d.next() + d.next() + return true + } + } + + return false +} + +func (d *Document) parseQuoted(escapeProp bool) Error { + if d.options.DebugLogger != nil { + d.options.DebugLogger("Parsing quoted string") + } + start := d.pos + for { + r := d.next() + if r == '\\' { + if d.parseEscape(true, escapeProp) { + continue + } + } + + if escapeProp { + if r == '.' || r == '{' || r == '[' || r == ':' || r == '^' { + d.buf.WriteRune('\\') + } + } + + if r == -1 { + return NewError(&d.expression, start, d.pos-start, "Expected quote but found EOF") + } else if r == '"' { + break + } else { + d.buf.WriteRune(r) + } + } + return nil +} + +func (d *Document) parseIndex() Error { + for { + r := d.next() + + if (r >= '0' && r <= '9') || r == '.' || r == '-' || r == '^' { + d.buf.WriteRune(r) + continue + } + + d.back() + break + } + + if d.expect(']') { + d.buf.WriteRune(']') + } else { + return d.error(1, "Expected ']' but found %s", runeStr(d.next())) + } + + return nil +} + +func (d *Document) parseProp(path string, commaStop bool) (string, Error) { + start := d.pos + d.skipWhitespace() + d.buf.Reset() + + for { + r := d.next() + + if r == '[' { + d.buf.WriteRune(r) + if err := d.parseIndex(); err != nil { + return "", err + } + continue + } + + if r == -1 || r == ':' || r == '{' || r == '}' || r == '^' || (commaStop && r == ',') { + d.back() + break + } + + if r == '"' { + if err := d.parseQuoted(true); err != nil { + return "", err + } + prop := d.buf.String() + + if canCoerce(prop) { + // This could be coerced into another type, so let's keep it wrapped + // in quotes to ensure it is treated properly. + prop = `"` + prop + `"` + } + + if path != "" { + return path + "." + prop, nil + } + return prop, nil + } + + if r == '\\' { + if d.parseEscape(false, true) { + continue + } + } + + d.buf.WriteRune(r) + } + + var prop string + if path != "" { + prop = path + "." + strings.TrimSpace(d.buf.String()) + } else { + prop = strings.TrimSpace(d.buf.String()) + } + + if d.options.DebugLogger != nil { + d.options.DebugLogger("Setting key %s", prop) + } + + if prop == "" { + return "", d.error(d.pos-start, "expected at least one property name") + } + + return prop, nil +} + +func (d *Document) parseObject(path string) Error { + // Special case: empty object + d.skipWhitespace() + if d.peek() == '}' { + d.Operations = append(d.Operations, Operation{ + Kind: OpSet, + Path: path, + Value: map[string]any{}, + }) + } + + for { + d.skipWhitespace() + r := d.peek() + + if r == -1 || r == '}' { + break + } + + if r == ',' { + d.next() + } + + prop, err := d.parseProp(path, false) + if err != nil { + return err + } + r = d.next() + if r == '{' { + // a{b: 1} is equivalent to a: {b: 1}, so we just send this to be parsed + // as a value. + d.back() + } else if r == '^' { + // a ^ b is a swap operation which takes a fully-qualified path as its + // value. The result of the paths are swapped in the resulting structure. + v, err := d.parseProp("", true) + if err != nil { + return err + } + d.Operations = append(d.Operations, Operation{ + Kind: OpSwap, + Path: prop, + Value: v, + }) + continue + } else { + if r != ':' { + return d.error(1, "Expected colon but got %v", runeStr(r)) + } + } + if err := d.parseValue(prop, true, true); err != nil { + return err + } + if strings.Contains(path, "[]") { + // Subsequent paths should not append additional values. + path = strings.ReplaceAll(path, "[]", "[-1]") + } + } + return nil +} + +func (d *Document) parseValue(path string, coerce bool, terminateComma bool) Error { + d.skipWhitespace() + d.buf.Reset() + start := d.pos + canSlice := true + first := true + + for { + r := d.next() + + if r == '\\' { + if d.parseEscape(false, false) { + canSlice = false + first = false + continue + } + } + + if first { + if r == '{' { + if d.options.DebugLogger != nil { + d.options.DebugLogger("Parsing sub-object") + } + start = d.pos + if err := d.parseObject(path); err != nil { + return err + } + if d.options.DebugLogger != nil { + d.options.DebugLogger("Sub-object done") + } + if !d.expect('}') { + return d.error(d.pos-start, "Expected '}' but found %s", runeStr(r)) + } + break + } else if r == '[' { + if d.options.DebugLogger != nil { + d.options.DebugLogger("Parsing sub-array") + } + // Special case: empty array + d.skipWhitespace() + if d.peek() == ']' { + if d.options.DebugLogger != nil { + d.options.DebugLogger("Parse value: []") + } + d.Operations = append(d.Operations, Operation{ + Kind: OpSet, + Path: path, + Value: []any{}, + }) + d.next() + break + } + + idx := 0 + for { + if idx > 0 && strings.Contains(path, "[]") { + path = strings.ReplaceAll(path, "[]", "[-1]") + } + d.parseValue(path+"["+strconv.Itoa(idx)+"]", true, true) + + d.skipWhitespace() + peek := d.peek() + if peek == ']' { + d.next() + break + } else if peek == ',' { + d.next() + } else { + return d.error(1, "Expected ',' or ']' but found '%s'", runeStr(peek)) + } + + idx++ + } + if d.options.DebugLogger != nil { + d.options.DebugLogger("Sub-array done") + } + break + } else if r == '"' { + if err := d.parseQuoted(false); err != nil { + return err + } + if d.options.DebugLogger != nil { + d.options.DebugLogger("Parse value: %v", d.buf.String()) + } + d.Operations = append(d.Operations, Operation{ + Kind: OpSet, + Path: path, + Value: d.buf.String(), + }) + break + } + } + first = false + + if r == -1 || r == '\n' || r == '}' || r == ']' || (terminateComma && r == ',') { + if r == '\n' { + d.skipWhitespace() + } else { + d.back() + } + var value string + if canSlice { + value = strings.TrimSpace(d.expression[start:d.pos]) + } else { + value = strings.TrimSpace(d.buf.String()) + } + + if coerce && len(value) > 0 { + if d.options.EnableFileInput && strings.HasPrefix(value, "@") && len(value) > 1 { + filename := value[1:] + + if d.options.DebugLogger != nil { + d.options.DebugLogger("Found file %s", filename) + } + + data, err := ioutil.ReadFile(filename) + if err != nil { + return d.error(uint(len(value)), "Unable to read file: %v", err) + } + + if strings.HasSuffix(filename, ".json") { + var structured any + if err := json.Unmarshal(data, &structured); err != nil { + return d.error(uint(len(value)), "Unable to unmarshal JSON: %v", err) + } + if d.options.DebugLogger != nil { + d.options.DebugLogger("Parse value: %v", structured) + } + d.Operations = append(d.Operations, Operation{ + Kind: OpSet, + Path: path, + Value: structured, + }) + break + } else if strings.HasSuffix(filename, ".cbor") { + var structured any + if err := cbor.Unmarshal(data, &structured); err != nil { + return d.error(uint(len(value)), "Unable to unmarshal CBOR: %v", err) + } + + if d.options.ForceStringKeys { + structured = ConvertMapString(structured) + } + if d.options.DebugLogger != nil { + d.options.DebugLogger("Parse value: %v", structured) + } + d.Operations = append(d.Operations, Operation{ + Kind: OpSet, + Path: path, + Value: structured, + }) + break + } else if utf8.Valid(data) { + value = string(data) + } else { + if d.options.DebugLogger != nil { + d.options.DebugLogger("Parse value: %v", data) + } + d.Operations = append(d.Operations, Operation{ + Kind: OpSet, + Path: path, + Value: data, + }) + break + } + } else if strings.HasPrefix(value, "%") { + binary, err := base64.StdEncoding.DecodeString(value[1:]) + if err != nil { + return d.error(uint(len(value)), "Unable to Base64 decode: %v", err) + } + if d.options.DebugLogger != nil { + d.options.DebugLogger("Parse value: %v", binary) + } + d.Operations = append(d.Operations, Operation{ + Kind: OpSet, + Path: path, + Value: binary, + }) + break + } else { + if value == "undefined" { + if d.options.DebugLogger != nil { + d.options.DebugLogger("Unsetting value") + } + d.Operations = append(d.Operations, Operation{ + Kind: OpDelete, + Path: path, + }) + break + } + + if coerced, ok := coerceValue(value); ok { + if d.options.DebugLogger != nil { + d.options.DebugLogger("Parse value: %v", coerced) + } + d.Operations = append(d.Operations, Operation{ + Kind: OpSet, + Path: path, + Value: coerced, + }) + break + } + } + } + + if d.options.DebugLogger != nil { + d.options.DebugLogger("Parse value: " + value) + } + d.Operations = append(d.Operations, Operation{ + Kind: OpSet, + Path: path, + Value: value, + }) + break + } + + d.buf.WriteRune(r) + } + return nil +} diff --git a/parse_test.go b/parse_test.go new file mode 100644 index 0000000..994a186 --- /dev/null +++ b/parse_test.go @@ -0,0 +1,224 @@ +package shorthand + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type l = []interface{} + +var parseExamples = []struct { + Name string + Existing interface{} + Input string + Error string + ForceStringKeys bool + Go interface{} + JSON string +}{ + { + Name: "Value", + Input: "true", + JSON: `[["", true]]`, + }, + { + Name: "Empty array", + Input: "[]", + JSON: `[["", []]]`, + }, + { + Name: "Empty object", + Input: "{}", + JSON: `[["", {}]]`, + }, + { + Name: "UTF-8 characters", + Input: "รค", + JSON: `[["", "รค"]]`, + }, + { + Name: "Escape property unquoted", + Input: `a\:\{b: c`, + JSON: `[["a\\:\\{b", "c"]]`, + }, + { + Name: "Coercion", + Input: "{n: null, b: true, i: 1, f: 1.0, fe: 1e+4, dt: 2020-01-01T12:00:00Z, s: hello, b: %wg==}", + Go: l{ + l{"n"}, + l{"b", true}, + l{"i", 1}, + l{"f", 1.0}, + l{"fe", 10000.0}, + l{"dt", dt}, + l{"s", "hello"}, + l{"b", []byte{0xc2}}, + }, + JSON: `[["n"], ["b", true], ["i", 1], ["f", 1.0], ["fe", 10000], ["dt", "2020-01-01T12:00:00Z"], ["s", "hello"], ["b", "wg=="]]`, + }, + { + Name: "Quoted Coerceable Keys", + Input: `{"null": 0, "true": 1, "false": 2, "2020-01-01T12:00:00Z": 3, "4": 5}`, + JSON: `[["\"null\"", 0], ["\"true\"", 1], ["\"false\"", 2], ["\"2020-01-01T12\\:00\\:00Z\"", 3], ["\"4\"", 5]]`, + }, + { + Name: "Guess object", + Input: `a: 1`, + JSON: `[["a", 1]]`, + }, + { + Name: "Nesting", + Input: `{a: [[{b: [[1], [{c: [2]}]]}]]}`, + JSON: `[["a[0][0].b[0][0]", 1], ["a[0][0].b[1][0].c[0]", 2]]`, + }, + { + Name: "Multiline", + Input: `{ + a: 1 + b{ + c: 2 + } + }`, + JSON: `[["a", 1], ["b.c", 2]]`, + }, + { + Name: "Spacing weirdness", + Input: ` { + a :1 + +b { + c: string value }} `, + JSON: `[["a", 1], ["b.c", "string value"]]`, + }, + { + Name: "File include JSON", + Input: `a: @testdata/hello.json`, + JSON: `[["a", {"hello": "world"}]]`, + }, + { + Name: "File include CBOR", + Input: `a: @testdata/hello.cbor`, + Go: l{ + l{"a", map[any]any{ + "hello": "world", + "ints": map[any]any{ + uint64(1): "hello", + uint64(2): true, + uint64(3): 4.5, + }, + }}, + }, + }, + { + Name: "File include CBOR string keys", + Input: `a: @testdata/hello.cbor`, + ForceStringKeys: true, + Go: l{ + l{"a", map[string]any{ + "hello": "world", + "ints": map[string]any{ + "1": "hello", + "2": true, + "3": 4.5, + }, + }}, + }, + }, + { + Name: "File include unstructured text", + Input: `a: @testdata/hello.txt`, + Go: l{ + l{"a", "hello\n"}, + }, + }, + { + Name: "File include unstructured binary", + Input: `a: @testdata/binary`, + Go: l{ + l{"a", []byte{0xc2}}, + }, + }, + { + Name: "Unclosed quoted string", + Input: `"hello`, + Error: "Expected quote", + }, + { + Name: "Unclosed index EOF", + Input: `{a[1`, + Error: "Expected ']'", + }, + { + Name: "Unclosed index other char", + Input: `{a[1b: 1}`, + Error: "Expected ']'", + }, + { + Name: "Invalid filename", + Input: `a: @invalid`, + Error: "Unable to read file", + }, +} + +func TestParser(t *testing.T) { + for _, example := range parseExamples { + t.Run(example.Name, func(t *testing.T) { + t.Logf("Input: %s", example.Input) + d := NewDocument( + ParseOptions{ + ForceStringKeys: example.ForceStringKeys, + EnableFileInput: true, + EnableObjectDetection: true, + DebugLogger: func(format string, a ...interface{}) { + t.Logf(format, a...) + }, + }, + ) + err := d.Parse(example.Input) + result := d.Marshal() + + if example.Error == "" { + require.NoError(t, err) + } else { + require.Error(t, err, "result is %v", d.Operations) + require.Contains(t, err.Error(), example.Error) + } + + if example.Go != nil { + assert.Equal(t, example.Go, result) + } + + if example.JSON != "" { + result = ConvertMapString(result) + b, _ := json.Marshal(result) + assert.JSONEq(t, example.JSON, string(b)) + } + }) + } +} + +func FuzzParser(f *testing.F) { + f.Add("{") + f.Add("}") + f.Add("[") + f.Add("]") + f.Add("null") + f.Add("true") + f.Add("0") + f.Add(`"hello"`) + f.Add(`"\u0020"`) + f.Fuzz(func(t *testing.T, s string) { + d := NewDocument( + ParseOptions{ + EnableFileInput: true, + DebugLogger: func(format string, a ...interface{}) { + t.Logf(format, a...) + }, + }, + ) + d.Parse(s) + }) +} diff --git a/shorthand.go b/shorthand.go index c4e4363..7de06fe 100644 --- a/shorthand.go +++ b/shorthand.go @@ -1,8 +1,6 @@ package shorthand import ( - "encoding/base64" - "encoding/json" "fmt" "io" "io/fs" @@ -15,48 +13,11 @@ import ( "gopkg.in/yaml.v3" ) -//go:generate pigeon -o generated.go shorthand.peg - const ( modifierNone = iota modifierString ) -func toIfaceSlice(v interface{}) []interface{} { - if v == nil { - return nil - } - return v.([]interface{}) -} - -func repeatedWithIndex(v interface{}, index int, cb func(v interface{})) { - for _, i := range v.([]interface{}) { - cb(i.([]interface{})[index]) - } -} - -// AST contains all of the key-value pairs in the document. -type AST []*KeyValue - -// KeyValue groups a Key with the key's associated value. -type KeyValue struct { - PostProcess bool - Key *Key - Value interface{} -} - -// Key contains parts and key-specific configuration. -type Key struct { - ResetContext bool - Parts []*KeyPart -} - -// KeyPart has a name and optional indices. -type KeyPart struct { - Key string - Index []int -} - // DeepAssign recursively merges a source map into the target. func DeepAssign(target, source map[string]interface{}) { for k, v := range source { @@ -76,15 +37,44 @@ func DeepAssign(target, source map[string]interface{}) { } } +func ConvertMapString(value interface{}) interface{} { + switch tmp := value.(type) { + case map[interface{}]interface{}: + m := make(map[string]interface{}, len(tmp)) + for k, v := range tmp { + m[fmt.Sprintf("%v", k)] = ConvertMapString(v) + } + return m + case map[string]interface{}: + for k, v := range tmp { + tmp[k] = ConvertMapString(v) + } + case []interface{}: + for i, v := range tmp { + tmp[i] = ConvertMapString(v) + } + } + + return value +} + // GetInput loads data from stdin (if present) and from the passed arguments, // returning the final structure. -func GetInput(args []string) (map[string]interface{}, error) { +func GetInput(args []string) (interface{}, error) { + stat, _ := os.Stdin.Stat() + return getInput(stat.Mode(), os.Stdin, args, ParseOptions{ + EnableFileInput: true, + EnableObjectDetection: true, + }) +} + +func GetInputWithOptions(args []string, options ParseOptions) (interface{}, error) { stat, _ := os.Stdin.Stat() - return getInput(stat.Mode(), os.Stdin, args) + return getInput(stat.Mode(), os.Stdin, args, options) } -func getInput(mode fs.FileMode, stdinFile io.Reader, args []string) (map[string]interface{}, error) { - var stdin map[string]interface{} +func getInput(mode fs.FileMode, stdinFile io.Reader, args []string, options ParseOptions) (interface{}, error) { + var stdin interface{} if (mode & os.ModeCharDevice) == 0 { d, err := ioutil.ReadAll(stdinFile) @@ -93,6 +83,9 @@ func getInput(mode fs.FileMode, stdinFile io.Reader, args []string) (map[string] } if err := yaml.Unmarshal(d, &stdin); err != nil { + if len(args) > 0 { + return nil, err + } return nil, err } } @@ -101,213 +94,27 @@ func getInput(mode fs.FileMode, stdinFile io.Reader, args []string) (map[string] return stdin, nil } - parsed, err := ParseAndBuild("args", strings.Join(args, " "), stdin) - if err != nil { - return nil, err + d := Document{ + options: options, } - - return parsed, nil -} - -// sliceRef represents a reference to a slice on a parent object (map or slice) -// that can be modified and have values set on it. -type sliceRef struct { - Base interface{} - Index int - Key string -} - -func (c *sliceRef) GetList() []interface{} { - switch b := c.Base.(type) { - case map[string]interface{}: - if l, ok := b[c.Key].([]interface{}); ok { - return l - } - case []interface{}: - if l, ok := b[c.Index].([]interface{}); ok { - return l - } - } - return nil -} - -func (c *sliceRef) Length() int { - return len(c.GetList()) -} - -func (c *sliceRef) Grow(length int) { - l := c.GetList() - if l == nil { - // This was not a list... might have been another type which we are - // now going to overwrite. - l = []interface{}{} - } - - for len(l) < length+1 { - l = append(l, nil) - } - - switch b := c.Base.(type) { - case map[string]interface{}: - b[c.Key] = l - case []interface{}: - b[c.Index] = l + if err := d.Parse(strings.Join(args, " ")); err != nil { + return nil, err } -} -func (c *sliceRef) GetValue(index int) interface{} { - return c.GetList()[index] -} - -func (c *sliceRef) SetValue(index int, value interface{}) { - c.GetList()[index] = value -} - -// ParseAndBuild takes a string and returns the structured data it represents. -func ParseAndBuild(filename, input string, existing ...map[string]interface{}) (map[string]interface{}, error) { - parsed, err := Parse(filename, []byte(input)) + data, err := d.Apply(stdin) if err != nil { return nil, err } - return Build(parsed.(AST), existing...) + return data, nil } -// Build an AST of key-value pairs into structured data. -func Build(ast AST, existing ...map[string]interface{}) (map[string]interface{}, error) { - result := map[string]interface{}{} - for _, e := range existing { - DeepAssign(result, e) - } - ctx := result - var ctxSlice sliceRef - - for _, kv := range ast { - k := kv.Key - v := kv.Value - - if subAST, ok := v.(AST); ok { - // If the value is itself an AST, then recursively process it! - parsed, err := Build(subAST) - if err != nil { - return result, err - } - v = parsed - } else if vStr, ok := v.(string); ok && kv.PostProcess { - // If the value is a string, then handle special cases here. - if len(vStr) > 1 && strings.HasPrefix(vStr, "@") { - filename := vStr[1:] - - forceString := false - useBase64 := false - if filename[0] == '~' { - forceString = true - filename = filename[1:] - } else if filename[0] == '%' { - forceString = true - filename = filename[1:] - useBase64 = true - } - data, err := ioutil.ReadFile(filename) - if err != nil { - return result, err - } - - if !forceString && strings.HasSuffix(vStr, ".json") { - // Try to load data from JSON file. - var unmarshalled interface{} - - if err := json.Unmarshal(data, &unmarshalled); err != nil { - return result, err - } - - v = unmarshalled - } else { - if useBase64 { - v = base64.StdEncoding.EncodeToString(data) - } else { - v = string(data) - } - } - } - } - - // Reset context to the root or keep going from where we left off. - if k.ResetContext { - ctx = result - } - - for ki, kp := range k.Parts { - // If there is a key, and the key is not in the current context, then it - // must be created as either a list or map depending on whether there - // are index items for one or more lists. - if kp.Key != "" && (ki < len(k.Parts)-1 || len(kp.Index) > 0) { - if len(kp.Index) > 0 { - if ctx[kp.Key] == nil { - ctx[kp.Key] = []interface{}{} - } - ctxSlice.Base = ctx - ctxSlice.Key = kp.Key - } else { - if ctx[kp.Key] == nil { - ctx[kp.Key] = make(map[string]interface{}) - } - if _, ok := ctx[kp.Key].(map[string]interface{}); !ok { - ctx[kp.Key] = make(map[string]interface{}) - } - ctx = ctx[kp.Key].(map[string]interface{}) - } - } - - // For each index item, create the associated list item and update the - // context. - for i, index := range kp.Index { - if index == -1 { - if ctxSlice.Base != nil { - index = ctxSlice.Length() - } else { - index = 0 - } - } - - ctxSlice.Grow(index) - - if i < len(kp.Index)-1 { - newBase := ctxSlice.GetList() - if len(newBase) < index+1 || newBase[index] == nil { - newList := []interface{}{} - ctxSlice.SetValue(index, newList) - } - ctxSlice.Index = index - ctxSlice.Base = newBase - } else { - // This is the last index item. If it is also the last key part, then - // set the value. Otherwise, create a map for the next key part to - // use and update the context. - if ki < len(k.Parts)-1 { - if ctxSlice.GetValue(index) == nil { - ctxSlice.SetValue(index, map[string]interface{}{}) - } - ctx = ctxSlice.GetValue(index).(map[string]interface{}) - } else { - ctxSlice.SetValue(index, v) - } - } - } - - // If this is the last key part and has no list indexes, then just set - // the value on the current context. - if ki == len(k.Parts)-1 && len(kp.Index) == 0 { - ctx[kp.Key] = v - if _, ok := v.([]interface{}); ok { - ctxSlice.Base = v - ctxSlice.Index = ki - } - } - } +func Parse(input string, options ParseOptions, existing interface{}) (interface{}, Error) { + d := Document{options: options} + if err := d.Parse(input); err != nil { + return nil, err } - - return result, nil + return d.Apply(existing) } // Get the shorthand representation of an input map. @@ -349,7 +156,7 @@ func renderValue(start bool, value interface{}) string { case []interface{}: var items []string - // Special case: foo: 1, 2, 3 + // Special case: foo: [1, 2, 3] scalars := true for _, item := range v { switch item.(type) { @@ -365,7 +172,7 @@ func renderValue(start bool, value interface{}) string { items = append(items, fmt.Sprintf("%v", item)) } - return ": " + strings.Join(items, ", ") + return ": [" + strings.Join(items, ", ") + "]" } // Normal case: foo[]: 1, []{id: 1, count: 2} diff --git a/shorthand.peg b/shorthand.peg deleted file mode 100644 index e489b93..0000000 --- a/shorthand.peg +++ /dev/null @@ -1,154 +0,0 @@ -{ - // Package shorthand provides a quick way to generate structured data via - // command line parameters. - package shorthand -} - -ShortHand โ† val:Object EOF { - return val, nil -} - -Object โ† _ vals:(KeyValue (',' _ KeyValue)*) _ { - // Collect keys/values into an AST for processing. - ast := AST{} - - valsSl := toIfaceSlice(vals) - ast = append(ast, valsSl[0].(*KeyValue)) - - rest := toIfaceSlice(valsSl[1]) - for _, item := range rest { - itemSl := toIfaceSlice(item) - - ast = append(ast, itemSl[2].(*KeyValue)) - } - - return ast, nil -} - -KeyValue โ† k:Key v:((':' Modifier? _ Value (',' _ ArrayValue &([^:] / !.))*) / ('{' Object '}')) { - vSl := toIfaceSlice(v) - var value interface{} - modifier := modifierNone - if vSl[0].([]byte)[0] == ':' { - if vSl[1] != nil { - modifier = vSl[1].(int) - } - - value = vSl[3] - - if modifier == modifierString { - if value == nil { - value = "null" - } else { - value = fmt.Sprintf("%v", value) - } - } - - extra := toIfaceSlice(vSl[4]) - if len(extra) > 0 { - values := []interface{}{value} - repeatedWithIndex(extra, 2, func(v interface{}) { - if modifier == modifierString { - v = fmt.Sprintf("%v", v) - } - values = append(values, v) - }) - value = values - } else { - if modifier == modifierString { - if value == nil { - value = "null" - } else { - value = fmt.Sprintf("%v", value) - } - } - } - } else { - value = vSl[1] - } - - return &KeyValue{ - PostProcess: modifier == modifierNone, - Key: k.(*Key), - Value: value, - }, nil -} - -Key โ† parts:('.'? (KeyPart '.')* KeyPart) { - var kps []*KeyPart - - pSl := toIfaceSlice(parts) - for _, sub := range toIfaceSlice(pSl[1]) { - kps = append(kps, toIfaceSlice(sub)[0].(*KeyPart)) - } - - kps = append(kps, pSl[2].(*KeyPart)) - - key := &Key{ - ResetContext: pSl[0] == nil, - Parts: kps, - } - - return key, nil -} - -KeyPart โ† label:KeyName index:('[' Int? ']')* { - idx := make([]int, 0) - repeatedWithIndex(index, 1, func(v interface{}) { - if v == nil { - idx = append(idx, -1) - return - } - - idx = append(idx, v.(int)) - }) - - kp := &KeyPart{ - Key: label.(string), - Index: idx, - } - return kp, nil -} - -KeyName โ† [^.:[{]* { - return strings.TrimSpace(string(c.text)), nil -} - -Modifier โ† [~] { - switch c.text[0] { - case '~': - return modifierString, nil - default: - return nil, fmt.Errorf("Unknown modifier %v", c.text[0]) - } -} - -ArrayValue โ† Null / Bool / Float / Int / ArrayString - -ArrayString โ† _ [^[] [^,:}]* { - return strings.TrimSpace(string(c.text)), nil -} - -Value โ† Null / Bool / Float / Int / String - -ValueEnd โ† _ [,}\]] / EOF - -Null โ† "null" { return nil, nil } - -Bool โ† "true" { return true, nil } / "false" { return false, nil } - -Float โ† [0-9]+ '.' [0-9]+ &ValueEnd { - return strconv.ParseFloat(string(c.text), 64) -} - -Int โ† [0-9]+ &ValueEnd { - return strconv.Atoi(string(c.text)) -} - -String โ† [^,}]* { - return strings.TrimSpace(string(c.text)), nil -} - -_ "whitespace" โ† [ \t\r\n]* - -EOF โ† !. diff --git a/shorthand_test.go b/shorthand_test.go index 0646412..9712f08 100644 --- a/shorthand_test.go +++ b/shorthand_test.go @@ -2,14 +2,13 @@ package shorthand import ( "encoding/json" - "strings" "testing" "github.com/stretchr/testify/assert" ) func parsed(input string, existing ...map[string]interface{}) string { - result, err := ParseAndBuild("stdin", input, existing...) + result, err := Parse(input, ParseOptions{}, existing) if err != nil { panic(err) } @@ -18,140 +17,140 @@ func parsed(input string, existing ...map[string]interface{}) string { return string(j) } -func TestGetInput(t *testing.T) { - file := strings.NewReader(`{ - "foo": [1], - "bar": { - "baz": true - }, - "existing": [1, 2, 3] - }`) - - result, err := getInput(0, file, []string{"foo[]: 2, bar.another: false, existing: null, existing[]: 1"}) - assert.NoError(t, err) - - j, _ := json.Marshal(result) - assert.JSONEq(t, `{ - "foo": [1, 2], - "bar": { - "another": false, - "baz": true - }, - "existing": [1] - }`, string(j)) -} - -func TestParseCoerce(t *testing.T) { - result := parsed(`n: null, b: true, i: 1, f: 1.0, s: hello`) - assert.JSONEq(t, `{"n": null, "b": true, "i": 1, "f": 1.0, "s": "hello"}`, result) -} - -func TestParseWhitespace(t *testing.T) { - assert.JSONEq(t, `{"foo": "hello", "bar": "world"}`, parsed(`foo : hello , bar:world `)) -} - -func TestParseIP(t *testing.T) { - assert.JSONEq(t, `{"foo": "1.2.3.4"}`, parsed(`foo: 1.2.3.4`)) -} - -func TestParseTrailingSpace(t *testing.T) { - assert.JSONEq(t, `{"foo": {"a": 1}}`, parsed(`foo{a: 1 }`)) -} - -func TestParseForceString(t *testing.T) { - assert.JSONEq(t, `{"foo": "1"}`, parsed(`foo:~ 1`)) -} - -func TestParseMultipleProperties(t *testing.T) { - assert.JSONEq(t, `{"foo": {"bar": {"baz": 1}}}`, parsed(`foo.bar.baz: 1`)) -} - -func TestParseContext(t *testing.T) { - result := parsed(`foo.bar: 1, .baz: 2, qux: 3`) - assert.JSONEq(t, `{"foo": {"bar": 1, "baz": 2}, "qux": 3}`, result) -} - -func TestParsePropertyGrouping(t *testing.T) { - assert.JSONEq(t, `{"foo": {"bar": 1, "baz": 2}}`, parsed(`foo{bar: 1, baz: 2}`)) -} - -func TestParserShortList(t *testing.T) { - assert.JSONEq(t, `{"foo": [1, 2, 3]}`, parsed(`foo: 1, 2, 3`)) -} - -func TestParserShortStringList(t *testing.T) { - assert.JSONEq(t, `{"foo": ["1", "2", "3"]}`, parsed(`foo:~ 1, 2, 3`)) -} - -func TestParserListOfList(t *testing.T) { - assert.JSONEq(t, `{"foo": [[null, [1]]]}`, parsed(`foo[][1][]: 1`)) -} - -func TestParserAppendList(t *testing.T) { - assert.JSONEq(t, `{"foo": [1, 2, 3]}`, parsed(`foo[]: 1, []: 2, []: 3`)) -} - -func TestParserAppendNestedList(t *testing.T) { - assert.JSONEq(t, `{"foo": [[[[2, 3]]]], "bar": [false]}`, parsed(`foo[][]: 1, [0][]: 2, 3, bar[]: true, [0]: false`)) -} - -func TestParserListIndex(t *testing.T) { - assert.JSONEq(t, `{"foo": [true, null, null, "three", null, "five"]}`, - parsed(`foo[3]: three, foo[5]: five, foo[0]: true`)) -} - -func TestParserListIndexNested(t *testing.T) { - assert.JSONEq(t, `{"foo": [null, null, null, [null, ["three"], true]]}`, - parsed(`foo[3][1][]: three, foo[3][2]: true`)) -} - -func TestParserListIndexObject(t *testing.T) { - result := parsed(`foo[0].bar: 1, foo[0].baz: 2`) - assert.JSONEq(t, `{"foo": [{"bar": 1, "baz": 2}]}`, result) -} - -func TestParserAppendBackRef(t *testing.T) { - assert.JSONEq(t, `{"foo": [1, 3], "bar": 2}`, parsed(`foo[]: 1, bar: 2, []: 3`)) -} - -func TestParserListObjects(t *testing.T) { - result := parsed(`foo[].id: 1, .count: 1, [].id: 2, .count: 2`) - assert.JSONEq(t, `{"foo": [{"id": 1, "count": 1}, {"id": 2, "count": 2}]}`, result) -} - -func TestParserListInlineObjects(t *testing.T) { - result := parsed(`foo[]{id: 1, count: 1}, []{id: 2, count: 2}`) - assert.JSONEq(t, `{"foo": [{"id": 1, "count": 1}, {"id": 2, "count": 2}]}`, result) -} - -func TestParserNonFile(t *testing.T) { - assert.JSONEq(t, `{"foo": "@user"}`, parsed(`foo:~ @user`)) -} - -func TestParserFileStructured(t *testing.T) { - result := parsed(`foo: @testdata/hello.json`) - assert.JSONEq(t, `{"foo": {"hello": "world"}}`, result) -} - -func TestParserFileForceString(t *testing.T) { - result := parsed(`foo: @~testdata/hello.json`) - assert.JSONEq(t, `{"foo": "{\n \"hello\": \"world\"\n}\n"}`, result) -} - -func TestExistingInput(t *testing.T) { - result := parsed(`foo[]: 3, foo[]: 4, bar[0][]: 2, baz.another: test`, map[string]interface{}{ - "foo": []interface{}{1, 2}, - "bar": []interface{}{[]interface{}{1}}, - "baz": true, - }) - assert.JSONEq(t, `{ - "foo": [1, 2, 3, 4], - "bar": [[1, 2]], - "baz": { - "another": "test" - } - }`, result) -} +// func TestGetInput(t *testing.T) { +// file := strings.NewReader(`{ +// "foo": [1], +// "bar": { +// "baz": true +// }, +// "existing": [1, 2, 3] +// }`) + +// result, err := getInput(0, file, []string{"foo[]: 2, bar.another: false, existing: null, existing[]: 1"}, ParseOptions{}) +// assert.NoError(t, err) + +// j, _ := json.Marshal(result) +// assert.JSONEq(t, `{ +// "foo": [1, 2], +// "bar": { +// "another": false, +// "baz": true +// }, +// "existing": [1] +// }`, string(j)) +// } + +// func TestParseCoerce(t *testing.T) { +// result := parsed(`n: null, b: true, i: 1, f: 1.0, s: hello`) +// assert.JSONEq(t, `{"n": null, "b": true, "i": 1, "f": 1.0, "s": "hello"}`, result) +// } + +// func TestParseWhitespace(t *testing.T) { +// assert.JSONEq(t, `{"foo": "hello", "bar": "world"}`, parsed(`foo : hello , bar:world `)) +// } + +// func TestParseIP(t *testing.T) { +// assert.JSONEq(t, `{"foo": "1.2.3.4"}`, parsed(`foo: 1.2.3.4`)) +// } + +// func TestParseTrailingSpace(t *testing.T) { +// assert.JSONEq(t, `{"foo": {"a": 1}}`, parsed(`foo{a: 1 }`)) +// } + +// func TestParseForceString(t *testing.T) { +// assert.JSONEq(t, `{"foo": "1"}`, parsed(`foo:~ 1`)) +// } + +// func TestParseMultipleProperties(t *testing.T) { +// assert.JSONEq(t, `{"foo": {"bar": {"baz": 1}}}`, parsed(`foo.bar.baz: 1`)) +// } + +// func TestParseContext(t *testing.T) { +// result := parsed(`foo.bar: 1, .baz: 2, qux: 3`) +// assert.JSONEq(t, `{"foo": {"bar": 1, "baz": 2}, "qux": 3}`, result) +// } + +// func TestParsePropertyGrouping(t *testing.T) { +// assert.JSONEq(t, `{"foo": {"bar": 1, "baz": 2}}`, parsed(`foo{bar: 1, baz: 2}`)) +// } + +// func TestParserShortList(t *testing.T) { +// assert.JSONEq(t, `{"foo": [1, 2, 3]}`, parsed(`foo: 1, 2, 3`)) +// } + +// func TestParserShortStringList(t *testing.T) { +// assert.JSONEq(t, `{"foo": ["1", "2", "3"]}`, parsed(`foo:~ 1, 2, 3`)) +// } + +// func TestParserListOfList(t *testing.T) { +// assert.JSONEq(t, `{"foo": [[null, [1]]]}`, parsed(`foo[][1][]: 1`)) +// } + +// func TestParserAppendList(t *testing.T) { +// assert.JSONEq(t, `{"foo": [1, 2, 3]}`, parsed(`foo[]: 1, []: 2, []: 3`)) +// } + +// func TestParserAppendNestedList(t *testing.T) { +// assert.JSONEq(t, `{"foo": [[[[2, 3]]]], "bar": [false]}`, parsed(`foo[][]: 1, [0][]: 2, 3, bar[]: true, [0]: false`)) +// } + +// func TestParserListIndex(t *testing.T) { +// assert.JSONEq(t, `{"foo": [true, null, null, "three", null, "five"]}`, +// parsed(`foo[3]: three, foo[5]: five, foo[0]: true`)) +// } + +// func TestParserListIndexNested(t *testing.T) { +// assert.JSONEq(t, `{"foo": [null, null, null, [null, ["three"], true]]}`, +// parsed(`foo[3][1][]: three, foo[3][2]: true`)) +// } + +// func TestParserListIndexObject(t *testing.T) { +// result := parsed(`foo[0].bar: 1, foo[0].baz: 2`) +// assert.JSONEq(t, `{"foo": [{"bar": 1, "baz": 2}]}`, result) +// } + +// func TestParserAppendBackRef(t *testing.T) { +// assert.JSONEq(t, `{"foo": [1, 3], "bar": 2}`, parsed(`foo[]: 1, bar: 2, []: 3`)) +// } + +// func TestParserListObjects(t *testing.T) { +// result := parsed(`foo[].id: 1, .count: 1, [].id: 2, .count: 2`) +// assert.JSONEq(t, `{"foo": [{"id": 1, "count": 1}, {"id": 2, "count": 2}]}`, result) +// } + +// func TestParserListInlineObjects(t *testing.T) { +// result := parsed(`foo[]{id: 1, count: 1}, []{id: 2, count: 2}`) +// assert.JSONEq(t, `{"foo": [{"id": 1, "count": 1}, {"id": 2, "count": 2}]}`, result) +// } + +// func TestParserNonFile(t *testing.T) { +// assert.JSONEq(t, `{"foo": "@user"}`, parsed(`foo:~ @user`)) +// } + +// func TestParserFileStructured(t *testing.T) { +// result := parsed(`foo: @testdata/hello.json`) +// assert.JSONEq(t, `{"foo": {"hello": "world"}}`, result) +// } + +// func TestParserFileForceString(t *testing.T) { +// result := parsed(`foo: @~testdata/hello.json`) +// assert.JSONEq(t, `{"foo": "{\n \"hello\": \"world\"\n}\n"}`, result) +// } + +// func TestExistingInput(t *testing.T) { +// result := parsed(`foo[]: 3, foo[]: 4, bar[0][]: 2, baz.another: test`, map[string]interface{}{ +// "foo": []interface{}{1, 2}, +// "bar": []interface{}{[]interface{}{1}}, +// "baz": true, +// }) +// assert.JSONEq(t, `{ +// "foo": [1, 2, 3, 4], +// "bar": [[1, 2]], +// "baz": { +// "another": "test" +// } +// }`, result) +// } func TestGetShorthandSimple(t *testing.T) { result := Get(map[string]interface{}{ @@ -194,7 +193,7 @@ func TestGetShorthandListSimple(t *testing.T) { result := Get(map[string]interface{}{ "foo": []interface{}{1, 2, 3}, }) - assert.Equal(t, "foo: 1, 2, 3", result) + assert.Equal(t, "foo: [1, 2, 3]", result) } func TestGetShorthandListOfListScalar(t *testing.T) { @@ -203,7 +202,7 @@ func TestGetShorthandListOfListScalar(t *testing.T) { []interface{}{1, 2, 3}, }, }) - assert.Equal(t, "foo[]: 1, 2, 3", result) + assert.Equal(t, "foo[]: [1, 2, 3]", result) } func TestGetShorthandListOfObjects(t *testing.T) { diff --git a/testdata/binary b/testdata/binary new file mode 100644 index 0000000..5277188 --- /dev/null +++ b/testdata/binary @@ -0,0 +1 @@ +ย \ No newline at end of file diff --git a/testdata/fuzz/FuzzParser/12fdb5c4b8c383a42822597850671bfee0b259ba88bd4e3bfd151e5feac1b8db b/testdata/fuzz/FuzzParser/12fdb5c4b8c383a42822597850671bfee0b259ba88bd4e3bfd151e5feac1b8db new file mode 100644 index 0000000..4722a86 --- /dev/null +++ b/testdata/fuzz/FuzzParser/12fdb5c4b8c383a42822597850671bfee0b259ba88bd4e3bfd151e5feac1b8db @@ -0,0 +1,2 @@ +go test fuzz v1 +string("{\"\"") diff --git a/testdata/fuzz/FuzzParser/bdced84807f06393f864fa064bb81fff2541c71debe6a711781ef7a10ab00ed2 b/testdata/fuzz/FuzzParser/bdced84807f06393f864fa064bb81fff2541c71debe6a711781ef7a10ab00ed2 new file mode 100644 index 0000000..e4ed566 --- /dev/null +++ b/testdata/fuzz/FuzzParser/bdced84807f06393f864fa064bb81fff2541c71debe6a711781ef7a10ab00ed2 @@ -0,0 +1,2 @@ +go test fuzz v1 +string("\\u") diff --git a/testdata/fuzz/FuzzParser/f2f1bb2b4123163cc61f3eb116a2b878980a3d9825af646dcbd9422b1f0a9903 b/testdata/fuzz/FuzzParser/f2f1bb2b4123163cc61f3eb116a2b878980a3d9825af646dcbd9422b1f0a9903 new file mode 100644 index 0000000..3db2ef9 --- /dev/null +++ b/testdata/fuzz/FuzzParser/f2f1bb2b4123163cc61f3eb116a2b878980a3d9825af646dcbd9422b1f0a9903 @@ -0,0 +1,2 @@ +go test fuzz v1 +string("@") diff --git a/testdata/hello.cbor b/testdata/hello.cbor new file mode 100644 index 0000000000000000000000000000000000000000..0f0efef7e1b8deebcd99a0cb8f8b7135ee37c51c GIT binary patch literal 38 mcmZ3Knvt55lb>3iUzC%Ql9^XhyqFOp#PpTYB literal 0 HcmV?d00001 diff --git a/testdata/hello.txt b/testdata/hello.txt new file mode 100644 index 0000000..ce01362 --- /dev/null +++ b/testdata/hello.txt @@ -0,0 +1 @@ +hello From 466392a147897b23627db6dee5f86901e073a749 Mon Sep 17 00:00:00 2001 From: "Daniel G. Taylor" Date: Sun, 23 Oct 2022 23:34:56 -0700 Subject: [PATCH 2/6] feat: tons of improvements & new path features --- README.md | 93 +++-- apply_test.go | 2 +- cmd/j/main.go | 6 +- document.go | 34 +- document_test.go | 57 ++- get.go | 226 ++++++++--- get_test.go | 166 +++++++- go.mod | 22 +- go.sum | 24 ++ parse.go | 4 +- parse_test.go | 22 +- shorthand.go | 202 +++++----- shorthand_test.go | 362 +++++++----------- ...0ff51642701a828ed8c9a233c067e94f248dc1a5de | 2 + ...fc2ea6b48ffc6130af63659b91cd8457c14857a454 | 2 + ...f007927bb0bc4f8669bfc58ea14c33f3346f9fdc05 | 2 + ...bbd732b2f241677f215c6ac174dc06e68b420b678b | 2 + ...172dbab5c265208e1fd478da18ae6575fc345e4c1b | 2 + ...a1aff81415c353f82ef83ff6958348d599633c504c | 2 + ...85ed84a72303c0ab410960f6028d032ecb5028db24 | 2 + ...3e0515cd74438d1bf1e9c636776ce14c6091a71fe7 | 2 + ...b3aa1067211e877fd21f4678868a9aef65b173d392 | 2 + ...adab1792fb7b0eaa63ce1a3352d3f4486b763f4747 | 2 + 23 files changed, 779 insertions(+), 461 deletions(-) create mode 100644 testdata/fuzz/FuzzGet/2f79470e0d08980f7b9b0f0ff51642701a828ed8c9a233c067e94f248dc1a5de create mode 100644 testdata/fuzz/FuzzGet/406e598a104f397fe8b6ccfc2ea6b48ffc6130af63659b91cd8457c14857a454 create mode 100644 testdata/fuzz/FuzzGet/40930d75c6901bdbf2e65cf007927bb0bc4f8669bfc58ea14c33f3346f9fdc05 create mode 100644 testdata/fuzz/FuzzGet/43aa7239034153dc0487b2bbd732b2f241677f215c6ac174dc06e68b420b678b create mode 100644 testdata/fuzz/FuzzGet/4741428453e25583ed8911172dbab5c265208e1fd478da18ae6575fc345e4c1b create mode 100644 testdata/fuzz/FuzzGet/4946d5954441694fefe136a1aff81415c353f82ef83ff6958348d599633c504c create mode 100644 testdata/fuzz/FuzzGet/a7443f5e6edd0ba18f227c85ed84a72303c0ab410960f6028d032ecb5028db24 create mode 100644 testdata/fuzz/FuzzGet/d36cd3eed329c48f98dc153e0515cd74438d1bf1e9c636776ce14c6091a71fe7 create mode 100644 testdata/fuzz/FuzzGet/da25bd636594308d5ef6cbb3aa1067211e877fd21f4678868a9aef65b173d392 create mode 100644 testdata/fuzz/FuzzGet/ec08094fe77963efb1dc4badab1792fb7b0eaa63ce1a3352d3f4486b763f4747 diff --git a/README.md b/README.md index 2f7f17a..ddea169 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Shorthand is a superset and friendlier variant of JSON designed with several use | -------------------- | ------------------------------------------------ | | CLI arguments/input | `my-cli post 'foo.bar[0]{baz: 1, hello: world}'` | | Patch operations | `name: undefined, item.tags[]: appended` | -| Query language | `items[created before "2022-01-01"].{id, tags}` | +| Query language | `items[created before 2022-01-01].{id, tags}` | | Configuration format | `{json.save.autoFormat: true}` | The shorthand syntax supports the following features, described in more detail with examples below: @@ -25,7 +25,11 @@ The shorthand syntax supports the following features, described in more detail w - Moving properties & items - Querying, array filtering, and field selection -The following are both completely valid shorthand and result in the same output: +The following are all completely valid shorthand and result in the same output: + +``` +foo.bar[]{baz: 1, hello: world} +``` ``` { @@ -49,7 +53,7 @@ The following are both completely valid shorthand and result in the same output: } ``` -This library has excellent test coverage and is additionally fuzz tested to ensure correctness and prevent panics. +This library has excellent test coverage to ensure correctness and is additionally fuzz tested to prevent panics. ## Alternatives & Inspiration @@ -161,7 +165,7 @@ $ j foo.bar.baz: 1 } ``` -Properties of nested objects can be grouped by placing them inside `{` and `}`. +Properties of nested objects can be grouped by placing them inside `{` and `}`. The `:` becomes optional for nested objects, so `foo.bar: {...}` is equivalent to `foo.bar{...}`. ```sh $ j foo.bar{id: 1, count.clicks: 5} @@ -259,7 +263,7 @@ Partial updates support: - Inserting before via `[^index]` - Removing fields or array items via `undefined` - Moving/swapping fields or array items via `^` - - The right hand side is a path to the value to swap + - The right hand side is a path to the value to swap. See Querying below for the path syntax. Some examples: @@ -279,7 +283,7 @@ $ j +No special steps are necessary to test local changes to the grammar. You can just run the included `j` utility to test: + +```sh +$ go run ./cmd/j your: new feature here ``` diff --git a/apply_test.go b/apply_test.go index 03920e4..8bcba51 100644 --- a/apply_test.go +++ b/apply_test.go @@ -276,7 +276,7 @@ func TestApply(t *testing.T) { require.Contains(t, err.Error(), example.Error) } - ops := d.Marshal() + ops := d.marshalOps() b, _ := json.Marshal(ops) t.Log(string(b)) diff --git a/cmd/j/main.go b/cmd/j/main.go index d39c5bb..160eea3 100644 --- a/cmd/j/main.go +++ b/cmd/j/main.go @@ -56,7 +56,8 @@ func main() { if selected, ok, err := shorthand.GetPath(*query, result, shorthand.GetOptions{DebugLogger: debugLog}); ok { result = selected } else if err != nil { - panic(err) + fmt.Println(err.Pretty()) + os.Exit(1) } else { fmt.Println("No match") return @@ -82,8 +83,7 @@ func main() { } } case "shorthand": - // TODO: fix to support non-maps - marshalled = []byte(shorthand.Get(result.(map[string]interface{}))) + marshalled = []byte(shorthand.MarshalPretty(result)) } if err != nil { diff --git a/document.go b/document.go index ad0ee86..403c319 100644 --- a/document.go +++ b/document.go @@ -57,33 +57,6 @@ func NewDocument(options ParseOptions) *Document { } } -func (d *Document) String() string { - // TODO: serialize to text format - return "" -} - -func (d *Document) Unmarshal(data []byte) { - // TODO: load from JSON/CBOR representation -} - -func (d *Document) Marshal() interface{} { - ops := make([]interface{}, len(d.Operations)) - - for i, op := range d.Operations { - s := []interface{}{} - if op.Kind != OpSet { - s = append(s, op.Kind) - } - s = append(s, op.Path) - if op.Value != nil { - s = append(s, op.Value) - } - ops[i] = s - } - - return ops -} - func (d *Document) Parse(input string) Error { d.expression = input d.pos = 0 @@ -131,3 +104,10 @@ func (d *Document) Apply(input interface{}) (interface{}, Error) { return input, nil } + +func (d *Document) Unmarshal(input string, existing any) (any, Error) { + if err := d.Parse(input); err != nil { + return nil, err + } + return d.Apply(existing) +} diff --git a/document_test.go b/document_test.go index 620524d..5f6ba2b 100644 --- a/document_test.go +++ b/document_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v3" ) func BenchmarkMinJSON(b *testing.B) { @@ -38,7 +39,26 @@ func BenchmarkFormattedJSON(b *testing.B) { } } -func BenchmarkLatestFull(b *testing.B) { +func BenchmarkYAML(b *testing.B) { + b.ReportAllocs() + var v interface{} + + large := []byte(` + foo: + bar: + id: 1 + tags: [one, two] + cost: 3.14 + baz: + id: 2 +`) + + for n := 0; n < b.N; n++ { + assert.NoError(b, yaml.Unmarshal(large, &v)) + } +} + +func BenchmarkShorthand(b *testing.B) { b.ReportAllocs() d := NewDocument(ParseOptions{ @@ -47,12 +67,37 @@ func BenchmarkLatestFull(b *testing.B) { for n := 0; n < b.N; n++ { d.Operations = d.Operations[:0] - d.Parse(`{foo{bar{id: 1, "tags": [one, two], cost: 3.14}, baz{id: 2}}}`) + d.Parse(`{foo{bar{id: 1, tags: [one, two], cost: 3.14}, baz{id: 2}}}`) + d.Apply(nil) + } +} + +func BenchmarkPretty(b *testing.B) { + b.ReportAllocs() + + d := NewDocument(ParseOptions{ + ForceStringKeys: true, + }) + + for n := 0; n < b.N; n++ { + d.Operations = d.Operations[:0] + d.Parse(`{ + foo{ + bar{ + id: 1 + tags: [one, two] + cost: 3.14 + } + baz{ + id: 2 + } + } + }`) d.Apply(nil) } } -func BenchmarkLatestParse(b *testing.B) { +func BenchmarkParse(b *testing.B) { b.ReportAllocs() d := NewDocument(ParseOptions{ @@ -61,16 +106,16 @@ func BenchmarkLatestParse(b *testing.B) { for n := 0; n < b.N; n++ { d.Operations = d.Operations[:0] - d.Parse(`{foo{bar{id: 1, "tags": [one, two], cost: 3.14}, baz{id: 2}}}`) + d.Parse(`{foo{bar{id: 1, tags: [one, two], cost: 3.14}, baz{id: 2}}}`) } } -func BenchmarkLatestApply(b *testing.B) { +func BenchmarkApply(b *testing.B) { b.ReportAllocs() d := NewDocument(ParseOptions{ ForceStringKeys: true, }) - d.Parse(`{foo{bar{id: 1, "tags": [one, two], cost: 3.14}, baz{id: 2}}}`) + d.Parse(`{foo{bar{id: 1, tags: [one, two], cost: 3.14}, baz{id: 2}}}`) for n := 0; n < b.N; n++ { d.Apply(nil) diff --git a/get.go b/get.go index f001405..315ac6f 100644 --- a/get.go +++ b/get.go @@ -1,6 +1,7 @@ package shorthand import ( + "sort" "strconv" "strings" @@ -109,12 +110,22 @@ func (d *Document) parsePathIndex() (bool, int, int, string, Error) { return false, index, index, "", nil } } else { + if indexes[0] == "" { + indexes[0] = "0" + } if startIndex, err := strconv.Atoi(indexes[0]); err == nil { + if indexes[1] == "" { + indexes[1] = "-1" + } if stopIndex, err := strconv.Atoi(indexes[1]); err == nil { return true, startIndex, stopIndex, "", nil } } } + + if value[0] == '?' { + value = value[1:] + } } return false, 0, 0, value, nil @@ -123,7 +134,10 @@ func (d *Document) parsePathIndex() (bool, int, int, string, Error) { func (d *Document) getFiltered(expr string, input any) (any, Error) { ast, err := mexpr.Parse(expr, nil) if err != nil { - return nil, NewError(&d.expression, d.pos+uint(err.Offset()), uint(err.Length()), err.Error()) + // Pos = current - expression - 1 for bracket + error offset. + // a.b.c[filter is here] + // current position....^ + return nil, NewError(&d.expression, d.pos-uint(len(expr)+1)+uint(err.Offset()), uint(err.Length()), err.Error()) } interpreter := mexpr.NewInterpreter(ast, mexpr.UnquotedStrings) savedPos := d.pos @@ -152,41 +166,65 @@ func (d *Document) getFiltered(expr string, input any) (any, Error) { return nil, nil } -func (d *Document) getIndex2(input any) (any, Error) { +func (d *Document) getPathIndex(input any) (any, Error) { isSlice, startIndex, stopIndex, expr, err := d.parsePathIndex() if err != nil { return nil, err } + if d.options.DebugLogger != nil { + d.options.DebugLogger("Getting index %v:%v %v", startIndex, stopIndex, expr) + } + if expr != "" { return d.getFiltered(expr, input) } - if s, ok := input.([]any); ok { - if startIndex > len(s)-1 || stopIndex > len(s)-1 { - return nil, nil - } - for startIndex < 0 { - startIndex += len(s) + l := 0 + switch t := input.(type) { + case string: + l = len(t) + case []byte: + l = len(t) + case []any: + l = len(t) + } + + if startIndex < 0 { + startIndex += l + } + if stopIndex < 0 { + stopIndex += l + } + if startIndex < 0 || startIndex > l-1 || stopIndex < 0 || stopIndex > l-1 || startIndex > stopIndex { + return nil, nil + } + + switch t := input.(type) { + case string: + if !isSlice { + return string(t[startIndex]), nil } - for stopIndex < 0 { - stopIndex += len(s) + return t[startIndex : stopIndex+1], nil + case []byte: + if !isSlice { + return t[startIndex], nil } - + return t[startIndex : stopIndex+1], nil + case []any: if !isSlice { - return s[startIndex], nil + return t[startIndex], nil } - - return s[startIndex : stopIndex+1], nil + return t[startIndex : stopIndex+1], nil } return nil, nil } -func (d *Document) parseProp2() (any, Error) { +func (d *Document) parseGetProp() (any, Error) { d.skipWhitespace() start := d.pos - quoted, canSlice, err := d.parseUntil(0, '.', '[', '|', ',', '}') + quoted, canSlice, err := d.parseUntil(0, '.', '[', '|', ',', '}', ']') if err != nil { return nil, err } @@ -208,23 +246,7 @@ func (d *Document) parseProp2() (any, Error) { } func (d *Document) getProp(input any) (any, bool, Error) { - if s, ok := input.([]any); ok { - var err Error - savedPos := d.pos - out := make([]any, len(s)) - - for i := range s { - d.pos = savedPos - out[i], _, err = d.getPath(s[i]) - if err != nil { - return nil, false, err - } - } - - return out, true, nil - } - - key, err := d.parseProp2() + key, err := d.parseGetProp() if err != nil { return nil, false, err } @@ -241,12 +263,78 @@ func (d *Document) getProp(input any) (any, bool, Error) { } else if m, ok := input.(map[any]any); ok { v, ok := m[key] return v, ok, nil + } else { + if d.options.DebugLogger != nil { + d.options.DebugLogger("Cannot get key %v from input %v", key, input) + } } return nil, false, nil } +func (d *Document) getPropRecursive(input any) ([]any, Error) { + key, err := d.parseGetProp() + if err != nil { + return nil, err + } + + if d.options.DebugLogger != nil { + d.options.DebugLogger("Recursive getting key '%v'", key) + } + + return d.findPropRecursive(key, input) +} + +func (d *Document) findPropRecursive(key, input any) ([]any, Error) { + results := []any{} + + savedPos := d.pos + if m, ok := input.(map[string]any); ok { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + v := m[k] + if s, ok := key.(string); ok { + if k == s { + results = append(results, v) + } + } + d.pos = savedPos + if tmp, err := d.findPropRecursive(key, v); err == nil { + results = append(results, tmp...) + } + } + } + if m, ok := input.(map[any]any); ok { + for k, v := range m { + if k == key { + results = append(results, v) + } + d.pos = savedPos + if tmp, err := d.findPropRecursive(key, v); err == nil { + results = append(results, tmp...) + } + } + } + if s, ok := input.([]any); ok { + for _, v := range s { + d.pos = savedPos + if tmp, err := d.findPropRecursive(key, v); err == nil { + results = append(results, tmp...) + } + } + } + + return results, nil +} + func (d *Document) flatten(input any) (any, Error) { + if d.options.DebugLogger != nil { + d.options.DebugLogger("Flattening %v", input) + } if s, ok := input.([]any); ok { out := make([]any, 0, len(s)) @@ -285,13 +373,49 @@ outer: found = true continue } - input, err = d.getIndex2(input) + input, err = d.getPathIndex(input) if err != nil { return nil, false, err } found = true case '.': d.next() + if d.peek() == '.' { + // Special case: recursive descent property query + // i.e. find this key recursively through everything + d.next() + input, err = d.getPropRecursive(input) + if err != nil { + return nil, false, err + } + } + if s, ok := input.([]any); ok { + // Special case: input is an slice of items, so run the right hand + // side on every item in the slice. + var err Error + var result any + savedPos := d.pos + out := make([]any, 0, len(s)) + + if len(s) == 0 { + // We will get `nil`, but still need to move the read head forwards to + // prevent an infinite loop here. + d.getPath(nil) + } + + for i := range s { + d.pos = savedPos + result, _, err = d.getPath(s[i]) + if err != nil { + return nil, false, err + } + if result != nil { + out = append(out, result) + } + } + + return out, true, nil + } continue case '{': d.next() @@ -301,6 +425,8 @@ outer: } found = true continue + case ',', ']', '}': + d.next() default: input, found, err = d.getProp(input) if err != nil { @@ -352,23 +478,27 @@ func (d *Document) getFields(input any) (any, Error) { if r == '}' { open-- } - if r == ',' || open == 0 { + if open == 0 || (open == 1 && r == ',') { path := d.buf.String() - if m, ok := input.(map[string]any); ok { - if key == "" { - result[path] = m[path] - } else { - expr, pos := d.expression, d.pos - tmp, _, err := GetPath(path, input, GetOptions{ - DebugLogger: d.options.DebugLogger, - }) - d.expression, d.pos = expr, pos - if err != nil { - return nil, err - } - result[key] = tmp + var value any + if key == "" { + key = path + if m, ok := input.(map[any]any); ok { + value = m[key] + } + if m, ok := input.(map[string]any); ok { + value = m[key] + } + } else { + var err Error + value, _, err = GetPath(path, input, GetOptions{ + DebugLogger: d.options.DebugLogger, + }) + if err != nil { + return nil, err } } + result[key] = value if r == '}' { break } diff --git a/get_test.go b/get_test.go index dd0cee8..ebda5f8 100644 --- a/get_test.go +++ b/get_test.go @@ -5,6 +5,7 @@ import ( "testing" "time" + "github.com/jmespath/go-jmespath" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -29,6 +30,12 @@ var getExamples = []struct { Query: `f1.f2.f3`, Go: true, }, + { + Name: "Recursive fields", + Input: `{"a": [{"id": 1}, {"b": {"id": 2}}], "c": {"d": {"id": 3}}}`, + Query: `..id`, + Go: []any{1.0, 2.0, 3.0}, + }, { Name: "Array index", Input: `{"field": [1, 2, 3]}`, @@ -51,13 +58,13 @@ var getExamples = []struct { Name: "Array item fields", Input: `{"items": [{"f1": {"f2": 1}}, {"f1": {"f2": 2}}, {"other": 3}]}`, Query: `items.f1.f2`, - Go: []any{1.0, 2.0, nil}, + Go: []any{1.0, 2.0}, }, { Name: "Array item fields empty index", Input: `{"items": [{"f1": {"f2": 1}}, {"f1": {"f2": 2}}, {"other": 3}]}`, Query: `items[].f1.f2`, - Go: []any{1.0, 2.0, nil}, + Go: []any{1.0, 2.0}, }, { Name: "Array item scalar filtering", @@ -83,12 +90,24 @@ var getExamples = []struct { Query: `link.{id, tags}`, Go: map[string]any{"id": 1.0, "tags": []any{"a", "b"}}, }, + { + Name: "Array field selection", + Input: `{"links": [{"rel": "next", "href": "..."}, {"rel": "prev", "href": "..."}]}`, + Query: `links.{rel}`, + Go: []any{map[string]any{"rel": "next"}, map[string]any{"rel": "prev"}}, + }, { Name: "Field expression", Input: `{"foo": "bar", "link": {"id": 1, "verified": true, "tags": ["a", "b"]}}`, Query: `{foo, id: link.id, tags: link.tags[@ startsWith a]}`, Go: map[string]any{"foo": "bar", "id": 1.0, "tags": []any{"a"}}, }, + { + Name: "Field expression nested multi", + Input: `{"body": [{"id": "a", "created": "2022", "link": "..."}], "headers": {"one": 1, "two": 2}}`, + Query: `{body: body.{id, created}, one: headers.one}`, + Go: map[string]any{"body": []any{map[string]any{"id": "a", "created": "2022"}}, "one": 1.0}, + }, { Name: "Field expression with pipe", Input: `{"foo": "bar", "link": {"id": 1, "verified": true, "tags": ["a", "b"]}}`, @@ -131,23 +150,136 @@ func TestGet(t *testing.T) { } } +var getBenchInput = map[string]any{ + "items": []any{ + 0, + map[string]any{ + "id": 1, + "name": "Item 1", + "desc": "...", + "price": 4.99, + "tags": []any{"one", "two", "three"}, + }, + map[string]any{ + "id": 2, + "name": "Item 2", + "desc": "...", + "price": 1.50, + "tags": []any{"four", "five", "six"}, + }, + }, +} + +func BenchmarkGetJMESPathSimple(b *testing.B) { + b.ReportAllocs() + + query := "items[1].name" + + out, err := jmespath.Search(query, getBenchInput) + require.NoError(b, err) + require.Equal(b, "Item 1", out) + + for n := 0; n < b.N; n++ { + jmespath.Search(query, getBenchInput) + } +} + +func BenchmarkGetJMESPath(b *testing.B) { + b.ReportAllocs() + + query := "items[-1].{name: name, price: price, f: tags[?starts_with(@, `\"f\"`)]}" + + out, err := jmespath.Search(query, getBenchInput) + require.NoError(b, err) + require.Equal(b, map[string]any{ + "name": "Item 2", + "price": 1.50, + "f": []any{"four", "five"}, + }, out) + + for n := 0; n < b.N; n++ { + jmespath.Search(query, getBenchInput) + } +} + +func BenchmarkGetJMESPathFlatten(b *testing.B) { + b.ReportAllocs() + + query := "items[].tags|[]" + + out, err := jmespath.Search(query, getBenchInput) + require.NoError(b, err) + require.Equal(b, []any{"one", "two", "three", "four", "five", "six"}, out) + + for n := 0; n < b.N; n++ { + GetPath(query, getBenchInput, GetOptions{}) + } +} + +func BenchmarkGetPathSimple(b *testing.B) { + b.ReportAllocs() + + query := "items[1].name" + + out, _, err := GetPath(query, getBenchInput, GetOptions{}) + require.NoError(b, err) + require.Equal(b, "Item 1", out) + + for n := 0; n < b.N; n++ { + GetPath(query, getBenchInput, GetOptions{}) + } +} + +func BenchmarkGetPath(b *testing.B) { + b.ReportAllocs() + + query := "items[-1].{name, price, f: tags[@ startsWith f]}" + + out, _, err := GetPath(query, getBenchInput, GetOptions{}) + require.NoError(b, err) + require.Equal(b, map[string]any{ + "name": "Item 2", + "price": 1.50, + "f": []any{"four", "five"}, + }, out) + + for n := 0; n < b.N; n++ { + GetPath(query, getBenchInput, GetOptions{}) + } +} + +func BenchmarkGetPathFlatten(b *testing.B) { + b.ReportAllocs() + + query := "items.tags|[]" + + out, _, err := GetPath(query, getBenchInput, GetOptions{}) + require.NoError(b, err) + require.Equal(b, []any{"one", "two", "three", "four", "five", "six"}, out) + + for n := 0; n < b.N; n++ { + GetPath(query, getBenchInput, GetOptions{}) + } +} + func FuzzGet(f *testing.F) { + data := map[string]any{ + "n": nil, + "b": true, + "i": 123, + "f": 4.5, + "s": "hello", + "b2": []byte{0, 1, 2}, + "d": time.Now(), + "a": []any{1, 2.5, "foo"}, + "aa": []any{[]any{[]any{1, 2, 3}}}, + "am": []any{map[string]any{"a": []any{1, 2, 3}}}, + "m": map[any]any{ + 1: true, + }, + } + f.Fuzz(func(t *testing.T, s string) { - data := map[string]any{ - "n": nil, - "b": true, - "i": 123, - "f": 4.5, - "s": "hello", - "b2": []byte{0, 1, 2}, - "d": time.Now(), - "a": []any{1, 2.5, "foo"}, - "aa": []any{[]any{[]any{1, 2, 3}}}, - "am": []any{map[string]any{"a": []any{1, 2, 3}}}, - "m": map[any]any{ - 1: true, - }, - } GetPath(s, data, GetOptions{DebugLogger: t.Logf}) }) } diff --git a/go.mod b/go.mod index 3cd10be..264c292 100644 --- a/go.mod +++ b/go.mod @@ -2,23 +2,27 @@ module github.com/danielgtaylor/shorthand go 1.18 -replace github.com/danielgtaylor/mexpr => ../../mexpr +replace github.com/danielgtaylor/mexpr => ../mexpr require ( + github.com/danielgtaylor/mexpr v1.7.1 github.com/fxamacker/cbor/v2 v2.4.0 - github.com/pelletier/go-toml v1.9.4 - github.com/spf13/cobra v1.2.1 - github.com/stretchr/testify v1.7.0 - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b + github.com/jmespath/go-jmespath v0.4.0 + github.com/pelletier/go-toml v1.9.5 + github.com/spf13/cobra v1.6.0 + github.com/stretchr/testify v1.8.0 + gopkg.in/yaml.v3 v3.0.1 ) require ( - github.com/danielgtaylor/mexpr v1.7.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/inconshreveable/mousetrap v1.0.1 // indirect + github.com/kr/pretty v0.2.1 // indirect + github.com/kr/text v0.1.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/stretchr/objx v0.4.0 // indirect github.com/x448/float16 v0.8.4 // indirect - golang.org/x/exp v0.0.0-20220518171630-0b5c67f07fdf // indirect - gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect + golang.org/x/exp v0.0.0-20221019170559-20944726eadf // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) diff --git a/go.sum b/go.sum index 68f0622..71a9d72 100644 --- a/go.sum +++ b/go.sum @@ -56,6 +56,7 @@ github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnht github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/danielgtaylor/mexpr v1.5.1 h1:sSlycueushuMlcb/8bB3fTcMRhLWmPa9XMKyr5WQDvU= github.com/danielgtaylor/mexpr v1.5.1/go.mod h1:xQ64V12CB+4K0wb1na+MtWSNQsPgjAyyHT4wTM7mM0I= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -165,6 +166,11 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= +github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= @@ -174,6 +180,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -196,6 +204,8 @@ github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FI github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= +github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -205,6 +215,7 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= @@ -214,11 +225,15 @@ github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw= github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= +github.com/spf13/cobra v1.6.0 h1:42a0n6jwCot1pUmomAp4T7DeMD+20LFv4Q54pxLf2LI= +github.com/spf13/cobra v1.6.0/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -226,6 +241,9 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= @@ -266,6 +284,8 @@ golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EH golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20220518171630-0b5c67f07fdf h1:oXVg4h2qJDd9htKxb5SCpFBHLipW6hXmL3qpUixS2jw= golang.org/x/exp v0.0.0-20220518171630-0b5c67f07fdf/go.mod h1:yh0Ynu2b5ZUe3MQfp2nM0ecK7wsgouWTDN0FNeJuIys= +golang.org/x/exp v0.0.0-20221019170559-20944726eadf h1:nFVjjKDgNY37+ZSYCJmtYf7tOlfQswHqplG2eosjOMg= +golang.org/x/exp v0.0.0-20221019170559-20944726eadf/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -563,6 +583,8 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -572,6 +594,8 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/parse.go b/parse.go index 0bcbc74..b094435 100644 --- a/parse.go +++ b/parse.go @@ -3,7 +3,7 @@ package shorthand import ( "encoding/base64" "encoding/json" - "io/ioutil" + "os" "strconv" "strings" "time" @@ -477,7 +477,7 @@ func (d *Document) parseValue(path string, coerce bool, terminateComma bool) Err d.options.DebugLogger("Found file %s", filename) } - data, err := ioutil.ReadFile(filename) + data, err := os.ReadFile(filename) if err != nil { return d.error(uint(len(value)), "Unable to read file: %v", err) } diff --git a/parse_test.go b/parse_test.go index 994a186..2002443 100644 --- a/parse_test.go +++ b/parse_test.go @@ -10,6 +10,24 @@ import ( type l = []interface{} +func (d *Document) marshalOps() interface{} { + ops := make([]interface{}, len(d.Operations)) + + for i, op := range d.Operations { + s := []interface{}{} + if op.Kind != OpSet { + s = append(s, op.Kind) + } + s = append(s, op.Path) + if op.Value != nil { + s = append(s, op.Value) + } + ops[i] = s + } + + return ops +} + var parseExamples = []struct { Name string Existing interface{} @@ -75,7 +93,7 @@ var parseExamples = []struct { JSON: `[["a[0][0].b[0][0]", 1], ["a[0][0].b[1][0].c[0]", 2]]`, }, { - Name: "Multiline", + Name: "Multiline optional commas", Input: `{ a: 1 b{ @@ -178,7 +196,7 @@ func TestParser(t *testing.T) { }, ) err := d.Parse(example.Input) - result := d.Marshal() + result := d.marshalOps() if example.Error == "" { require.NoError(t, err) diff --git a/shorthand.go b/shorthand.go index 7de06fe..a77739a 100644 --- a/shorthand.go +++ b/shorthand.go @@ -4,52 +4,26 @@ import ( "fmt" "io" "io/fs" - "io/ioutil" "os" "sort" - "strconv" "strings" "gopkg.in/yaml.v3" ) -const ( - modifierNone = iota - modifierString -) - -// DeepAssign recursively merges a source map into the target. -func DeepAssign(target, source map[string]interface{}) { - for k, v := range source { - if vm, ok := v.(map[string]interface{}); ok { - if _, ok := target[k]; ok { - if tkm, ok := target[k].(map[string]interface{}); ok { - DeepAssign(tkm, vm) - } else { - target[k] = vm - } - } else { - target[k] = vm - } - } else { - target[k] = v - } - } -} - -func ConvertMapString(value interface{}) interface{} { +func ConvertMapString(value any) any { switch tmp := value.(type) { - case map[interface{}]interface{}: - m := make(map[string]interface{}, len(tmp)) + case map[any]any: + m := make(map[string]any, len(tmp)) for k, v := range tmp { m[fmt.Sprintf("%v", k)] = ConvertMapString(v) } return m - case map[string]interface{}: + case map[string]any: for k, v := range tmp { tmp[k] = ConvertMapString(v) } - case []interface{}: + case []any: for i, v := range tmp { tmp[i] = ConvertMapString(v) } @@ -60,24 +34,23 @@ func ConvertMapString(value interface{}) interface{} { // GetInput loads data from stdin (if present) and from the passed arguments, // returning the final structure. -func GetInput(args []string) (interface{}, error) { - stat, _ := os.Stdin.Stat() - return getInput(stat.Mode(), os.Stdin, args, ParseOptions{ +func GetInput(args []string) (any, error) { + return GetInputWithOptions(args, ParseOptions{ EnableFileInput: true, EnableObjectDetection: true, }) } -func GetInputWithOptions(args []string, options ParseOptions) (interface{}, error) { +func GetInputWithOptions(args []string, options ParseOptions) (any, error) { stat, _ := os.Stdin.Stat() return getInput(stat.Mode(), os.Stdin, args, options) } -func getInput(mode fs.FileMode, stdinFile io.Reader, args []string, options ParseOptions) (interface{}, error) { - var stdin interface{} +func getInput(mode fs.FileMode, stdinFile io.Reader, args []string, options ParseOptions) (any, error) { + var stdin any if (mode & os.ModeCharDevice) == 0 { - d, err := ioutil.ReadAll(stdinFile) + d, err := io.ReadAll(stdinFile) if err != nil { return nil, err } @@ -94,108 +67,153 @@ func getInput(mode fs.FileMode, stdinFile io.Reader, args []string, options Pars return stdin, nil } - d := Document{ - options: options, + return Unmarshal(strings.Join(args, " "), options, stdin) +} + +func Unmarshal(input string, options ParseOptions, existing any) (any, Error) { + d := Document{options: options} + return d.Unmarshal(input, existing) +} + +type MarshalOptions struct { + Indent string + Spacer string + UseFile bool +} + +func (o MarshalOptions) GetIndent(level int) string { + if o.Indent == "" { + return "" } - if err := d.Parse(strings.Join(args, " ")); err != nil { - return nil, err + result := "\n" + for i := 0; i < level; i++ { + result += o.Indent } + return result +} - data, err := d.Apply(stdin) - if err != nil { - return nil, err +func (o MarshalOptions) GetSeparator(level int) string { + if o.Indent != "" { + return o.GetIndent(level) } - return data, nil + return "," + o.Spacer } -func Parse(input string, options ParseOptions, existing interface{}) (interface{}, Error) { - d := Document{options: options} - if err := d.Parse(input); err != nil { - return nil, err +func Marshal(input any, options ...MarshalOptions) string { + if len(options) == 0 { + options = []MarshalOptions{{}} + } + return renderValue(options[0], 0, false, input) +} + +func MarshalCLI(input any) string { + result := Marshal(input, MarshalOptions{Spacer: " ", UseFile: true}) + if strings.HasPrefix(result, "{") { + result = result[1 : len(result)-1] } - return d.Apply(existing) + return result } -// Get the shorthand representation of an input map. -func Get(input map[string]interface{}) string { - result := renderValue(true, input) - return result[1 : len(result)-1] +func MarshalPretty(input any) string { + return Marshal(input, MarshalOptions{Spacer: " ", Indent: " "}) } -func renderValue(start bool, value interface{}) string { - // Go uses `` so here we hard-code `null` to match JSON/YAML. +func renderValue(options MarshalOptions, level int, fromKey bool, value any) string { + prefix := "" + if fromKey { + prefix = ":" + options.Spacer + } + + // Go uses `nil` so here we hard-code `null` to match JSON/YAML. if value == nil { - return ": null" + return prefix + "null" } switch v := value.(type) { - case map[string]interface{}: + case map[any]any: // Special case: foo.bar: 1 - if !start && len(v) == 1 { + if len(v) == 1 { + dot := "" + if fromKey { + dot = "." + } for k := range v { - return "." + k + renderValue(false, v[k]) + return dot + fmt.Sprintf("%v", k) + renderValue(options, level, true, v[k]) } } // Normal case: foo{a: 1, b: 2} - var keys []string + var keys []any for k := range v { keys = append(keys, k) } - sort.Strings(keys) + sort.Slice(keys, func(i, j int) bool { + return fmt.Sprintf("%v", keys[i]) < fmt.Sprintf("%v", keys[j]) + }) var fields []string for _, k := range keys { - fields = append(fields, k+renderValue(false, v[k])) + fields = append(fields, fmt.Sprintf("%v", k)+renderValue(options, level+1, true, v[k])) } - return "{" + strings.Join(fields, ", ") + "}" - case []interface{}: - var items []string - - // Special case: foo: [1, 2, 3] - scalars := true - for _, item := range v { - switch item.(type) { - case map[string]interface{}: - scalars = false - case []interface{}: - scalars = false + return "{" + options.GetIndent(level+1) + strings.Join(fields, options.GetSeparator(level+1)) + options.GetIndent(level) + "}" + case map[string]any: + // Special case: foo.bar: 1 + if len(v) == 1 { + dot := "" + if fromKey { + dot = "." + } + for k := range v { + return dot + k + renderValue(options, level, true, v[k]) } } - if scalars { - for _, item := range v { - items = append(items, fmt.Sprintf("%v", item)) - } + // Normal case: foo{a: 1, b: 2} + var keys []string - return ": [" + strings.Join(items, ", ") + "]" + for k := range v { + keys = append(keys, k) } - // Normal case: foo[]: 1, []{id: 1, count: 2} + sort.Strings(keys) + + var fields []string + for _, k := range keys { + kStr := k + if canCoerce(k) { + kStr = `"` + k + `"` + } + fields = append(fields, kStr+renderValue(options, level+1, true, v[k])) + } + + return "{" + options.GetIndent(level+1) + strings.Join(fields, options.GetSeparator(level+1)) + options.GetIndent(level) + "}" + case []any: + var items []string + + // Normal case: foo: [1, true, {id: 1, count: 2}] for _, item := range v { - items = append(items, "[]"+renderValue(false, item)) + items = append(items, renderValue(options, level+1, false, item)) } - return strings.Join(items, ", ") + return prefix + "[" + options.GetIndent(level+1) + strings.Join(items, options.GetSeparator(level+1)) + options.GetIndent(level) + "]" default: - modifier := "" - if s, ok := v.(string); ok { - _, err := strconv.ParseFloat(s, 64) - - if err == nil || s == "null" || s == "true" || s == "false" { - modifier = "~" + if canCoerce(s) { + // This is a string but needs to be quoted so it doesn't get coerced + // into some other type when parsed. + v = `"` + strings.Replace(s, `"`, `\"`, -1) + `"` } - if len(s) > 50 || strings.Contains(s, "\n") { + if options.UseFile && (len(s) > 50 || strings.Contains(s, "\n")) { + // Long strings are represented as being loaded from files. v = "@file" } } - return fmt.Sprintf(":%s %v", modifier, v) + return fmt.Sprintf("%s%v", prefix, v) } } diff --git a/shorthand_test.go b/shorthand_test.go index 9712f08..b7b7b22 100644 --- a/shorthand_test.go +++ b/shorthand_test.go @@ -2,245 +2,167 @@ package shorthand import ( "encoding/json" + "strings" "testing" "github.com/stretchr/testify/assert" ) -func parsed(input string, existing ...map[string]interface{}) string { - result, err := Parse(input, ParseOptions{}, existing) - if err != nil { - panic(err) - } - - j, _ := json.Marshal(result) - return string(j) -} - -// func TestGetInput(t *testing.T) { -// file := strings.NewReader(`{ -// "foo": [1], -// "bar": { -// "baz": true -// }, -// "existing": [1, 2, 3] -// }`) - -// result, err := getInput(0, file, []string{"foo[]: 2, bar.another: false, existing: null, existing[]: 1"}, ParseOptions{}) -// assert.NoError(t, err) - -// j, _ := json.Marshal(result) -// assert.JSONEq(t, `{ -// "foo": [1, 2], -// "bar": { -// "another": false, -// "baz": true -// }, -// "existing": [1] -// }`, string(j)) -// } - -// func TestParseCoerce(t *testing.T) { -// result := parsed(`n: null, b: true, i: 1, f: 1.0, s: hello`) -// assert.JSONEq(t, `{"n": null, "b": true, "i": 1, "f": 1.0, "s": "hello"}`, result) -// } - -// func TestParseWhitespace(t *testing.T) { -// assert.JSONEq(t, `{"foo": "hello", "bar": "world"}`, parsed(`foo : hello , bar:world `)) -// } - -// func TestParseIP(t *testing.T) { -// assert.JSONEq(t, `{"foo": "1.2.3.4"}`, parsed(`foo: 1.2.3.4`)) -// } - -// func TestParseTrailingSpace(t *testing.T) { -// assert.JSONEq(t, `{"foo": {"a": 1}}`, parsed(`foo{a: 1 }`)) -// } - -// func TestParseForceString(t *testing.T) { -// assert.JSONEq(t, `{"foo": "1"}`, parsed(`foo:~ 1`)) -// } - -// func TestParseMultipleProperties(t *testing.T) { -// assert.JSONEq(t, `{"foo": {"bar": {"baz": 1}}}`, parsed(`foo.bar.baz: 1`)) -// } - -// func TestParseContext(t *testing.T) { -// result := parsed(`foo.bar: 1, .baz: 2, qux: 3`) -// assert.JSONEq(t, `{"foo": {"bar": 1, "baz": 2}, "qux": 3}`, result) -// } - -// func TestParsePropertyGrouping(t *testing.T) { -// assert.JSONEq(t, `{"foo": {"bar": 1, "baz": 2}}`, parsed(`foo{bar: 1, baz: 2}`)) -// } - -// func TestParserShortList(t *testing.T) { -// assert.JSONEq(t, `{"foo": [1, 2, 3]}`, parsed(`foo: 1, 2, 3`)) -// } - -// func TestParserShortStringList(t *testing.T) { -// assert.JSONEq(t, `{"foo": ["1", "2", "3"]}`, parsed(`foo:~ 1, 2, 3`)) -// } - -// func TestParserListOfList(t *testing.T) { -// assert.JSONEq(t, `{"foo": [[null, [1]]]}`, parsed(`foo[][1][]: 1`)) -// } - -// func TestParserAppendList(t *testing.T) { -// assert.JSONEq(t, `{"foo": [1, 2, 3]}`, parsed(`foo[]: 1, []: 2, []: 3`)) -// } - -// func TestParserAppendNestedList(t *testing.T) { -// assert.JSONEq(t, `{"foo": [[[[2, 3]]]], "bar": [false]}`, parsed(`foo[][]: 1, [0][]: 2, 3, bar[]: true, [0]: false`)) -// } - -// func TestParserListIndex(t *testing.T) { -// assert.JSONEq(t, `{"foo": [true, null, null, "three", null, "five"]}`, -// parsed(`foo[3]: three, foo[5]: five, foo[0]: true`)) -// } - -// func TestParserListIndexNested(t *testing.T) { -// assert.JSONEq(t, `{"foo": [null, null, null, [null, ["three"], true]]}`, -// parsed(`foo[3][1][]: three, foo[3][2]: true`)) -// } - -// func TestParserListIndexObject(t *testing.T) { -// result := parsed(`foo[0].bar: 1, foo[0].baz: 2`) -// assert.JSONEq(t, `{"foo": [{"bar": 1, "baz": 2}]}`, result) -// } - -// func TestParserAppendBackRef(t *testing.T) { -// assert.JSONEq(t, `{"foo": [1, 3], "bar": 2}`, parsed(`foo[]: 1, bar: 2, []: 3`)) -// } - -// func TestParserListObjects(t *testing.T) { -// result := parsed(`foo[].id: 1, .count: 1, [].id: 2, .count: 2`) -// assert.JSONEq(t, `{"foo": [{"id": 1, "count": 1}, {"id": 2, "count": 2}]}`, result) -// } - -// func TestParserListInlineObjects(t *testing.T) { -// result := parsed(`foo[]{id: 1, count: 1}, []{id: 2, count: 2}`) -// assert.JSONEq(t, `{"foo": [{"id": 1, "count": 1}, {"id": 2, "count": 2}]}`, result) -// } - -// func TestParserNonFile(t *testing.T) { -// assert.JSONEq(t, `{"foo": "@user"}`, parsed(`foo:~ @user`)) -// } - -// func TestParserFileStructured(t *testing.T) { -// result := parsed(`foo: @testdata/hello.json`) -// assert.JSONEq(t, `{"foo": {"hello": "world"}}`, result) -// } - -// func TestParserFileForceString(t *testing.T) { -// result := parsed(`foo: @~testdata/hello.json`) -// assert.JSONEq(t, `{"foo": "{\n \"hello\": \"world\"\n}\n"}`, result) -// } - -// func TestExistingInput(t *testing.T) { -// result := parsed(`foo[]: 3, foo[]: 4, bar[0][]: 2, baz.another: test`, map[string]interface{}{ -// "foo": []interface{}{1, 2}, -// "bar": []interface{}{[]interface{}{1}}, -// "baz": true, -// }) -// assert.JSONEq(t, `{ -// "foo": [1, 2, 3, 4], -// "bar": [[1, 2]], -// "baz": { -// "another": "test" -// } -// }`, result) -// } - -func TestGetShorthandSimple(t *testing.T) { - result := Get(map[string]interface{}{ - "foo": "bar", - }) - assert.Equal(t, "foo: bar", result) -} +func TestGetInput(t *testing.T) { + file := strings.NewReader(`{ + "foo": [1], + "bar": { + "baz": true + }, + "existing": [1, 2, 3] + }`) -func TestGetShorthandMultiKey(t *testing.T) { - result := Get(map[string]interface{}{ - "foo": "bar", - "hello": "world", - "num": 1, - "empty": nil, - "bool": false, - }) - assert.Equal(t, "bool: false, empty: null, foo: bar, hello: world, num: 1", result) -} + result, err := getInput(0, file, []string{"foo[]: 2, bar.another: false, existing: null, existing[]: 1"}, ParseOptions{EnableObjectDetection: true}) + assert.NoError(t, err) -func TestGetShorthandNestedSimple(t *testing.T) { - result := Get(map[string]interface{}{ - "foo": map[string]interface{}{ - "bar": 1, + j, _ := json.Marshal(result) + assert.JSONEq(t, `{ + "foo": [1, 2], + "bar": { + "another": false, + "baz": true }, - }) - assert.Equal(t, "foo.bar: 1", result) + "existing": [1] + }`, string(j)) } -func TestGetShorthandNestedMultiKey(t *testing.T) { - result := Get(map[string]interface{}{ - "foo": map[string]interface{}{ - "bar": 1, - "baz": 2, +var marshalExamples = []struct { + Name string + Input any + Output string +}{ + { + Name: "Simple", + Input: true, + Output: "true", + }, + { + Name: "Simple object", + Input: map[string]any{ + "foo": "bar", }, - }) - assert.Equal(t, "foo{bar: 1, baz: 2}", result) -} - -func TestGetShorthandListSimple(t *testing.T) { - result := Get(map[string]interface{}{ - "foo": []interface{}{1, 2, 3}, - }) - assert.Equal(t, "foo: [1, 2, 3]", result) -} - -func TestGetShorthandListOfListScalar(t *testing.T) { - result := Get(map[string]interface{}{ - "foo": []interface{}{ - []interface{}{1, 2, 3}, + Output: "foo: bar", + }, + { + Name: "Multi key", + Input: map[string]any{ + "foo": "bar", + "hello": "world", + "num": 1, + "empty": nil, + "bool": false, }, - }) - assert.Equal(t, "foo[]: [1, 2, 3]", result) -} - -func TestGetShorthandListOfObjects(t *testing.T) { - result := Get(map[string]interface{}{ - "tags": []interface{}{ - map[string]interface{}{ - "id": "tag1", - "count": map[string]interface{}{ - "clicks": 15, - "sales": 3, - }, + Output: "bool: false, empty: null, foo: bar, hello: world, num: 1", + }, + { + Name: "Nested simple", + Input: map[string]any{ + "foo": map[string]any{ + "bar": 1, }, - map[string]interface{}{ - "id": "tag2", - "count": map[string]interface{}{ - "clicks": 7, - "sales": 4, + }, + Output: "foo.bar: 1", + }, + { + Name: "Nested multi key", + Input: map[string]any{ + "foo": map[string]any{ + "bar": 1, + "baz": 2, + }, + }, + Output: "foo{bar: 1, baz: 2}", + }, + { + Name: "List of list of items", + Input: map[string]any{ + "foo": []any{ + []any{1, 2, 3}, + }, + }, + Output: "foo: [[1, 2, 3]]", + }, + { + Name: "List of objects", + Input: map[string]interface{}{ + "tags": []interface{}{ + map[string]interface{}{ + "id": "tag1", + "count": map[string]interface{}{ + "clicks": 15, + "sales": 3, + }, + }, + map[string]interface{}{ + "id": "tag2", + "count": map[string]interface{}{ + "clicks": 7, + "sales": 4, + }, }, }, }, - }) - assert.Equal(t, "tags[]{count{clicks: 15, sales: 3}, id: tag1}, []{count{clicks: 7, sales: 4}, id: tag2}", result) + Output: "tags: [{count{clicks: 15, sales: 3}, id: tag1}, {count{clicks: 7, sales: 4}, id: tag2}]", + }, + { + Name: "Coerced", + Input: map[string]any{ + "null": "null", + "bool": "true", + "num": "1234", + "str": "hello", + }, + Output: `bool: "true", "null": "null", num: "1234", str: hello`, + }, + { + Name: "File", + Input: map[string]any{ + "multi": "I am\na multiline\n value.", + "long": "I am a really long line of text that should probably get loaded from a file", + }, + Output: "long: @file, multi: @file", + }, } -func TestGetShorthandCoerced(t *testing.T) { - result := Get(map[string]interface{}{ - "null": "null", - "bool": "true", - "num": "1234", - "str": "hello", - }) - assert.Equal(t, "bool:~ true, null:~ null, num:~ 1234, str: hello", result) +func TestMarshal(t *testing.T) { + for _, example := range marshalExamples { + t.Run(example.Name, func(t *testing.T) { + t.Logf("Input: %s", example.Input) + out := MarshalCLI(example.Input) + assert.Equal(t, example.Output, out) + }) + } } -func TestGetShorthandFromFile(t *testing.T) { - result := Get(map[string]interface{}{ - "multi": "I am\na multiline\n value.", - "long": "I am a really long line of text that should probably get loaded from a file", +func TestMarshalPretty(t *testing.T) { + result := MarshalPretty(map[string]any{ + "foo": 1, + "bar": []any{ + 2, 3, + }, + "baz": map[string]any{ + "a": map[any]any{ + "b": map[any]any{ + "c": true, + "d": false, + }, + }, + }, }) - assert.Equal(t, "long: @file, multi: @file", result) + assert.Equal(t, `{ + bar: [ + 2 + 3 + ] + baz.a.b{ + c: true + d: false + } + foo: 1 +}`, result) } diff --git a/testdata/fuzz/FuzzGet/2f79470e0d08980f7b9b0f0ff51642701a828ed8c9a233c067e94f248dc1a5de b/testdata/fuzz/FuzzGet/2f79470e0d08980f7b9b0f0ff51642701a828ed8c9a233c067e94f248dc1a5de new file mode 100644 index 0000000..1514da8 --- /dev/null +++ b/testdata/fuzz/FuzzGet/2f79470e0d08980f7b9b0f0ff51642701a828ed8c9a233c067e94f248dc1a5de @@ -0,0 +1,2 @@ +go test fuzz v1 +string("a[-01:00]") diff --git a/testdata/fuzz/FuzzGet/406e598a104f397fe8b6ccfc2ea6b48ffc6130af63659b91cd8457c14857a454 b/testdata/fuzz/FuzzGet/406e598a104f397fe8b6ccfc2ea6b48ffc6130af63659b91cd8457c14857a454 new file mode 100644 index 0000000..af2c1a2 --- /dev/null +++ b/testdata/fuzz/FuzzGet/406e598a104f397fe8b6ccfc2ea6b48ffc6130af63659b91cd8457c14857a454 @@ -0,0 +1,2 @@ +go test fuzz v1 +string("aa[A7][-7]") diff --git a/testdata/fuzz/FuzzGet/40930d75c6901bdbf2e65cf007927bb0bc4f8669bfc58ea14c33f3346f9fdc05 b/testdata/fuzz/FuzzGet/40930d75c6901bdbf2e65cf007927bb0bc4f8669bfc58ea14c33f3346f9fdc05 new file mode 100644 index 0000000..d28cb26 --- /dev/null +++ b/testdata/fuzz/FuzzGet/40930d75c6901bdbf2e65cf007927bb0bc4f8669bfc58ea14c33f3346f9fdc05 @@ -0,0 +1,2 @@ +go test fuzz v1 +string("a[2:0]") diff --git a/testdata/fuzz/FuzzGet/43aa7239034153dc0487b2bbd732b2f241677f215c6ac174dc06e68b420b678b b/testdata/fuzz/FuzzGet/43aa7239034153dc0487b2bbd732b2f241677f215c6ac174dc06e68b420b678b new file mode 100644 index 0000000..0164f0a --- /dev/null +++ b/testdata/fuzz/FuzzGet/43aa7239034153dc0487b2bbd732b2f241677f215c6ac174dc06e68b420b678b @@ -0,0 +1,2 @@ +go test fuzz v1 +string("z,") diff --git a/testdata/fuzz/FuzzGet/4741428453e25583ed8911172dbab5c265208e1fd478da18ae6575fc345e4c1b b/testdata/fuzz/FuzzGet/4741428453e25583ed8911172dbab5c265208e1fd478da18ae6575fc345e4c1b new file mode 100644 index 0000000..1725867 --- /dev/null +++ b/testdata/fuzz/FuzzGet/4741428453e25583ed8911172dbab5c265208e1fd478da18ae6575fc345e4c1b @@ -0,0 +1,2 @@ +go test fuzz v1 +string("a[-4444444430000]\x89Xw") diff --git a/testdata/fuzz/FuzzGet/4946d5954441694fefe136a1aff81415c353f82ef83ff6958348d599633c504c b/testdata/fuzz/FuzzGet/4946d5954441694fefe136a1aff81415c353f82ef83ff6958348d599633c504c new file mode 100644 index 0000000..d6ca7dd --- /dev/null +++ b/testdata/fuzz/FuzzGet/4946d5954441694fefe136a1aff81415c353f82ef83ff6958348d599633c504c @@ -0,0 +1,2 @@ +go test fuzz v1 +string("a[\x15]\f{\"\"\"") diff --git a/testdata/fuzz/FuzzGet/a7443f5e6edd0ba18f227c85ed84a72303c0ab410960f6028d032ecb5028db24 b/testdata/fuzz/FuzzGet/a7443f5e6edd0ba18f227c85ed84a72303c0ab410960f6028d032ecb5028db24 new file mode 100644 index 0000000..32f5333 --- /dev/null +++ b/testdata/fuzz/FuzzGet/a7443f5e6edd0ba18f227c85ed84a72303c0ab410960f6028d032ecb5028db24 @@ -0,0 +1,2 @@ +go test fuzz v1 +string("a[@[2:0]]") diff --git a/testdata/fuzz/FuzzGet/d36cd3eed329c48f98dc153e0515cd74438d1bf1e9c636776ce14c6091a71fe7 b/testdata/fuzz/FuzzGet/d36cd3eed329c48f98dc153e0515cd74438d1bf1e9c636776ce14c6091a71fe7 new file mode 100644 index 0000000..9242612 --- /dev/null +++ b/testdata/fuzz/FuzzGet/d36cd3eed329c48f98dc153e0515cd74438d1bf1e9c636776ce14c6091a71fe7 @@ -0,0 +1,2 @@ +go test fuzz v1 +string("}") diff --git a/testdata/fuzz/FuzzGet/da25bd636594308d5ef6cbb3aa1067211e877fd21f4678868a9aef65b173d392 b/testdata/fuzz/FuzzGet/da25bd636594308d5ef6cbb3aa1067211e877fd21f4678868a9aef65b173d392 new file mode 100644 index 0000000..a69d258 --- /dev/null +++ b/testdata/fuzz/FuzzGet/da25bd636594308d5ef6cbb3aa1067211e877fd21f4678868a9aef65b173d392 @@ -0,0 +1,2 @@ +go test fuzz v1 +string("aa[A[@[0][:0]]]") diff --git a/testdata/fuzz/FuzzGet/ec08094fe77963efb1dc4badab1792fb7b0eaa63ce1a3352d3f4486b763f4747 b/testdata/fuzz/FuzzGet/ec08094fe77963efb1dc4badab1792fb7b0eaa63ce1a3352d3f4486b763f4747 new file mode 100644 index 0000000..acd410c --- /dev/null +++ b/testdata/fuzz/FuzzGet/ec08094fe77963efb1dc4badab1792fb7b0eaa63ce1a3352d3f4486b763f4747 @@ -0,0 +1,2 @@ +go test fuzz v1 +string("aa,0|[A[@[0]]]") From df90035fc25ef79171c3da9bb4be1f3ff2daaacf Mon Sep 17 00:00:00 2001 From: "Daniel G. Taylor" Date: Sat, 29 Oct 2022 13:19:25 -0700 Subject: [PATCH 3/6] feat: better unicode support, comments, wildcard fields, several fixes --- README.md | 147 +++++++++++++++++++++++++++++++++++++++++++++++--- apply.go | 2 +- apply_test.go | 12 +++++ document.go | 7 ++- get.go | 32 ++++++++++- get_test.go | 104 +++++++++++++++++++++++------------ parse.go | 103 +++++++++++++++++++++++++++++------ parse_test.go | 31 ++++++++++- 8 files changed, 377 insertions(+), 61 deletions(-) diff --git a/README.md b/README.md index ddea169..edc583a 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,72 @@ go get -u github.com/danielgtaylor/shorthand/cmd/j Also feel free to use this tool to generate structured data for input to other commands. +Here is a diagram overview of the language syntax, which is similar to [JSON's syntax](https://www.json.org/json-en.html) but adds a few things: + + + +![shorthand-syntax](https://user-images.githubusercontent.com/106826/198850895-a1a8481a-2c63-484c-9bf2-ce472effa8c3.svg) + +Note: + +- `string` can be quoted (with `"`) or unquoted. +- The `query` syntax in the diagram above is described below in the [Querying](#querying) section. + ### Keys & Values At its most basic, a structure is built out of key & value pairs. They are separated by commas: @@ -113,7 +179,7 @@ Shorthand supports the standard JSON types, but adds some of its own as well to | `null` | JSON `null` | | `boolean` | Either `true` or `false` | | `number` | JSON number, e.g. `1`, `2.5`, or `1.4e5` | -| `string` | Quoted or unquoted strings, e.g. `"hello"` | +| `string` | Quoted or unquoted strings, e.g. `hello` or `"hello"` | | `bytes` | `%`-prefixed, unquoted, base64-encoded binary data, e.g. `%wg==` | | `time` | Date/time in ISO8601, e.g. `2022-01-01T12:00:00Z` | | `array` | JSON array, e.g. `[1, 2, 3]` | @@ -318,7 +384,7 @@ $ j + +![shorthand-query-syntax](https://user-images.githubusercontent.com/106826/198693468-fadf8d48-8223-4dd9-a2cb-a1651e342fc5.svg) + +The `filter` syntax is described in the documentation for [mexpr](https://github.com/danielgtaylor/mexpr). + Examples: ```sh @@ -371,7 +489,7 @@ $ j 1].f1.f2`, Go: []any{2.0}, }, + { + Name: "Array filtering nested brackets", + Input: `{"items": [{"id": 1, "tags": ["a", "b"]}]}`, + Query: `items[tags[0] == "abc"[0]].id`, + Go: []any{1.0}, + }, { Name: "Array filtering first match", Input: `{"items": ["a", "b", "c"]}`, @@ -170,51 +204,51 @@ var getBenchInput = map[string]any{ }, } -func BenchmarkGetJMESPathSimple(b *testing.B) { - b.ReportAllocs() +// func BenchmarkGetJMESPathSimple(b *testing.B) { +// b.ReportAllocs() - query := "items[1].name" +// query := "items[1].name" - out, err := jmespath.Search(query, getBenchInput) - require.NoError(b, err) - require.Equal(b, "Item 1", out) +// out, err := jmespath.Search(query, getBenchInput) +// require.NoError(b, err) +// require.Equal(b, "Item 1", out) - for n := 0; n < b.N; n++ { - jmespath.Search(query, getBenchInput) - } -} +// for n := 0; n < b.N; n++ { +// jmespath.Search(query, getBenchInput) +// } +// } -func BenchmarkGetJMESPath(b *testing.B) { - b.ReportAllocs() +// func BenchmarkGetJMESPath(b *testing.B) { +// b.ReportAllocs() - query := "items[-1].{name: name, price: price, f: tags[?starts_with(@, `\"f\"`)]}" +// query := "items[-1].{name: name, price: price, f: tags[?starts_with(@, `\"f\"`)]}" - out, err := jmespath.Search(query, getBenchInput) - require.NoError(b, err) - require.Equal(b, map[string]any{ - "name": "Item 2", - "price": 1.50, - "f": []any{"four", "five"}, - }, out) +// out, err := jmespath.Search(query, getBenchInput) +// require.NoError(b, err) +// require.Equal(b, map[string]any{ +// "name": "Item 2", +// "price": 1.50, +// "f": []any{"four", "five"}, +// }, out) - for n := 0; n < b.N; n++ { - jmespath.Search(query, getBenchInput) - } -} +// for n := 0; n < b.N; n++ { +// jmespath.Search(query, getBenchInput) +// } +// } -func BenchmarkGetJMESPathFlatten(b *testing.B) { - b.ReportAllocs() +// func BenchmarkGetJMESPathFlatten(b *testing.B) { +// b.ReportAllocs() - query := "items[].tags|[]" +// query := "items[].tags|[]" - out, err := jmespath.Search(query, getBenchInput) - require.NoError(b, err) - require.Equal(b, []any{"one", "two", "three", "four", "five", "six"}, out) +// out, err := jmespath.Search(query, getBenchInput) +// require.NoError(b, err) +// require.Equal(b, []any{"one", "two", "three", "four", "five", "six"}, out) - for n := 0; n < b.N; n++ { - GetPath(query, getBenchInput, GetOptions{}) - } -} +// for n := 0; n < b.N; n++ { +// GetPath(query, getBenchInput, GetOptions{}) +// } +// } func BenchmarkGetPathSimple(b *testing.B) { b.ReportAllocs() diff --git a/parse.go b/parse.go index b094435..cee3588 100644 --- a/parse.go +++ b/parse.go @@ -8,6 +8,7 @@ import ( "strings" "time" "unicode" + "unicode/utf16" "unicode/utf8" "github.com/fxamacker/cbor/v2" @@ -41,13 +42,13 @@ func canCoerce(value string) bool { return true } else if len(value) >= 10 && value[0] >= '0' && value[0] <= '9' && value[3] >= '0' && value[3] <= '9' && value[4] == '-' && value[7] == '-' { return true - } else if len(value) > 0 && value[0] >= '0' && value[0] <= '9' { + } else if len(value) > 0 && ((value[0] >= '0' && value[0] <= '9') || value[0] == '-' || value[0] == '+' || value[0] == '.') { return true } return false } -func coerceValue(value string) (any, bool) { +func coerceValue(value string, forceFloat bool) (any, bool) { if value == "null" { return nil, true } else if value == "true" { @@ -59,7 +60,7 @@ func coerceValue(value string) (any, bool) { if t, err := time.Parse(time.RFC3339Nano, value); err == nil { return t, true } - } else if len(value) > 0 && value[0] >= '0' && value[0] <= '9' { + } else if len(value) > 0 && ((value[0] >= '0' && value[0] <= '9') || value[0] == '-' || value[0] == '+' || value[0] == '.') { // This looks like a number. isFloat := false for _, r := range value { @@ -68,7 +69,7 @@ func coerceValue(value string) (any, bool) { break } } - if isFloat { + if isFloat || forceFloat { if f, err := strconv.ParseFloat(value, 64); err == nil { return f, true } @@ -109,8 +110,18 @@ func (d *Document) back() { // peek returns the next rune without moving the position forward. func (d *Document) peek() rune { - r := d.next() - d.back() + if d.pos >= uint(len(d.expression)) { + return -1 + } + + var r rune + if d.expression[d.pos] < utf8.RuneSelf { + // Optimization for a simple ASCII character + r = rune(d.expression[d.pos]) + } else { + r, _ = utf8.DecodeRuneInString(d.expression[d.pos:]) + } + return r } @@ -141,6 +152,44 @@ func (d *Document) skipWhitespace() { } } +func (d *Document) skipComments(r rune) bool { + if r == '/' && d.peek() == '/' { + for { + r = d.next() + if r == -1 || r == '\n' { + break + } + } + d.skipWhitespace() + return true + } + return false +} + +// getu4 decodes \uXXXX from the beginning of s, returning the hex value, +// or it returns -1. +// This is taken from the official Go JSON decoder. +func getu4(s []byte) rune { + if len(s) < 6 || s[0] != '\\' || s[1] != 'u' { + return -1 + } + var r rune + for _, c := range s[2:6] { + switch { + case '0' <= c && c <= '9': + c = c - '0' + case 'a' <= c && c <= 'f': + c = c - 'a' + 10 + case 'A' <= c && c <= 'F': + c = c - 'A' + 10 + default: + return -1 + } + r = r*16 + rune(c) + } + return r +} + func (d *Document) parseEscape(quoted bool, includeEscape bool) bool { peek := d.peek() if !quoted { @@ -168,14 +217,26 @@ func (d *Document) parseEscape(quoted bool, includeEscape bool) bool { d.buf.WriteRune(replace) return true } - if peek == 'u' && len(d.expression) >= int(d.pos)+5 { - if s, err := strconv.Unquote(`"` + d.expression[d.pos-1:d.pos+5] + `"`); err == nil { - d.buf.WriteString(s) - d.next() - d.next() - d.next() - d.next() - d.next() + if (peek == 'u' || peek == 'U') && len(d.expression) >= int(d.pos)+5 { + r := getu4([]byte(d.expression[d.pos-1:])) + if r >= 0 { + b := make([]byte, 4) + d.pos += 5 // We already consumed the '\' + if utf16.IsSurrogate(r) { + // This is a two character UTF16 sequence as two '\uXXXX' pairs. + r2 := getu4([]byte(d.expression[d.pos:])) + if dec := utf16.DecodeRune(r, r2); dec != unicode.ReplacementChar { + d.pos += 6 + w := utf8.EncodeRune(b, dec) + d.buf.Write(b[:w]) + return true + } + // Invalid surrogate! + r = unicode.ReplacementChar + } + // Otherwise: this is a normal single '\uXXXX' encoded character. + w := utf8.EncodeRune(b, r) + d.buf.Write(b[:w]) return true } } @@ -280,6 +341,10 @@ func (d *Document) parseProp(path string, commaStop bool) (string, Error) { } } + if d.skipComments(r) { + continue + } + d.buf.WriteRune(r) } @@ -322,6 +387,7 @@ func (d *Document) parseObject(path string) Error { if r == ',' { d.next() + continue } prop, err := d.parseProp(path, false) @@ -372,6 +438,11 @@ func (d *Document) parseValue(path string, coerce bool, terminateComma bool) Err for { r := d.next() + if d.skipComments(r) { + canSlice = false + continue + } + if r == '\\' { if d.parseEscape(false, false) { canSlice = false @@ -395,6 +466,8 @@ func (d *Document) parseValue(path string, coerce bool, terminateComma bool) Err if !d.expect('}') { return d.error(d.pos-start, "Expected '}' but found %s", runeStr(r)) } + d.skipWhitespace() + d.skipComments(d.peek()) break } else if r == '[' { if d.options.DebugLogger != nil { @@ -553,7 +626,7 @@ func (d *Document) parseValue(path string, coerce bool, terminateComma bool) Err break } - if coerced, ok := coerceValue(value); ok { + if coerced, ok := coerceValue(value, d.options.ForceFloat64Numbers); ok { if d.options.DebugLogger != nil { d.options.DebugLogger("Parse value: %v", coerced) } diff --git a/parse_test.go b/parse_test.go index 2002443..b334d4f 100644 --- a/parse_test.go +++ b/parse_test.go @@ -94,7 +94,8 @@ var parseExamples = []struct { }, { Name: "Multiline optional commas", - Input: `{ + Input: ` + { a: 1 b{ c: 2 @@ -102,6 +103,28 @@ var parseExamples = []struct { }`, JSON: `[["a", 1], ["b.c", 2]]`, }, + { + Name: "Multiline trailing commas", + Input: ` + { + a: 1, + b{ + c: 2, + }, + }`, + JSON: `[["a", 1], ["b.c", 2]]`, + }, + { + Name: "Multiline with comments", + Input: `// Line comment + { + a: 1, // Trailing comment + b{ // Before properties + c: 2 + } // After object + }`, + JSON: `[["a", 1], ["b.c", 2]]`, + }, { Name: "Spacing weirdness", Input: ` { @@ -199,7 +222,11 @@ func TestParser(t *testing.T) { result := d.marshalOps() if example.Error == "" { - require.NoError(t, err) + msg := "" + if err != nil { + msg = err.Pretty() + } + require.NoError(t, err, msg) } else { require.Error(t, err, "result is %v", d.Operations) require.Contains(t, err.Error(), example.Error) From d2702d45043b6949ec3fd6561b6667eb946549f8 Mon Sep 17 00:00:00 2001 From: "Daniel G. Taylor" Date: Sat, 29 Oct 2022 18:05:43 -0700 Subject: [PATCH 4/6] fix: support quoted empty string key --- apply.go | 2 +- apply_test.go | 6 ++++++ parse.go | 2 +- parse_test.go | 5 +++++ 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/apply.go b/apply.go index 679d40a..639f3ad 100644 --- a/apply.go +++ b/apply.go @@ -171,7 +171,7 @@ func (d *Document) applyPathPart(input any, op Operation) (any, Error) { var key any = keystr d.buf.Reset() - if key == "" { + if key == "" && !quoted { // Special case: raw value if r == '[' { // This raw value is an array. diff --git a/apply_test.go b/apply_test.go index 7cd73d4..61c0880 100644 --- a/apply_test.go +++ b/apply_test.go @@ -42,6 +42,12 @@ var applyExamples = []struct { }, JSON: `{"n": null, "b": true, "i": 1, "f": 1.0, "dt": "2020-01-01T12:00:00Z", "s": "hello"}`, }, + { + Name: "Empty property name", + Input: `{"": 0}`, + Go: map[string]any{"": 0}, + JSON: `{"": 0}`, + }, { Name: "Property nested", Input: "{foo.bar.baz: hello}", diff --git a/parse.go b/parse.go index cee3588..cc12a14 100644 --- a/parse.go +++ b/parse.go @@ -323,7 +323,7 @@ func (d *Document) parseProp(path string, commaStop bool) (string, Error) { } prop := d.buf.String() - if canCoerce(prop) { + if canCoerce(prop) || prop == "" { // This could be coerced into another type, so let's keep it wrapped // in quotes to ensure it is treated properly. prop = `"` + prop + `"` diff --git a/parse_test.go b/parse_test.go index b334d4f..38e829b 100644 --- a/parse_test.go +++ b/parse_test.go @@ -52,6 +52,11 @@ var parseExamples = []struct { Input: "{}", JSON: `[["", {}]]`, }, + { + Name: "Empty key", + Input: `{"": 0}`, + JSON: `[["\"\"", 0]]`, + }, { Name: "UTF-8 characters", Input: "รค", From 1d4aa3a4d8575974ff351663ab378e15267db5b5 Mon Sep 17 00:00:00 2001 From: "Daniel G. Taylor" Date: Sat, 29 Oct 2022 18:16:50 -0700 Subject: [PATCH 5/6] feat: update module name to v2 --- README.md | 2 +- cmd/j/main.go | 2 +- go.mod | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index edc583a..3fcc23f 100644 --- a/README.md +++ b/README.md @@ -496,7 +496,7 @@ package main import ( "fmt" - "github.com/danielgtaylor/shorthand" + "github.com/danielgtaylor/shorthand/v2" ) func main() { diff --git a/cmd/j/main.go b/cmd/j/main.go index 160eea3..79b9aba 100644 --- a/cmd/j/main.go +++ b/cmd/j/main.go @@ -7,7 +7,7 @@ import ( "reflect" "strings" - "github.com/danielgtaylor/shorthand" + "github.com/danielgtaylor/shorthand/v2" "github.com/fxamacker/cbor/v2" toml "github.com/pelletier/go-toml" "github.com/spf13/cobra" diff --git a/go.mod b/go.mod index 264c292..4bb80cf 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/danielgtaylor/shorthand +module github.com/danielgtaylor/shorthand/v2 go 1.18 From 4562674750d4eca473050b010feda4fbf570cb24 Mon Sep 17 00:00:00 2001 From: "Daniel G. Taylor" Date: Sat, 29 Oct 2022 19:34:33 -0700 Subject: [PATCH 6/6] feat: GetInput improvements --- README.md | 4 +- cmd/j/main.go | 8 +++- shorthand.go | 50 ++++++++++++++---------- shorthand_test.go | 98 ++++++++++++++++++++++++++++++++++++++--------- 4 files changed, 119 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index 3fcc23f..4a176f6 100644 --- a/README.md +++ b/README.md @@ -318,7 +318,7 @@ $ j 'twitter: "@user"' ### Patch (Partial Update) -Partial updates are supported on existing data, which can be used to implement HTTP `PATCH`, templating, and other similar features. This feature combines the best of both: +Partial updates are supported on existing data, which can be used to implement HTTP `PATCH`, templating, and other similar features. The suggested content type for HTTP `PATCH` is `application/shorthand-patch`. This feature combines the best of both: - [JSON Merge Patch](https://datatracker.ietf.org/doc/html/rfc7386) - [JSON Patch](https://www.rfc-editor.org/rfc/rfc6902) @@ -331,6 +331,8 @@ Partial updates support: - Moving/swapping fields or array items via `^` - The right hand side is a path to the value to swap. See Querying below for the path syntax. +Note: When sending shorthand patches file loading via `@` should be disabled as the files will not exist on the server. + Some examples: ```sh diff --git a/cmd/j/main.go b/cmd/j/main.go index 79b9aba..8d050e2 100644 --- a/cmd/j/main.go +++ b/cmd/j/main.go @@ -24,7 +24,7 @@ func main() { cmd := &cobra.Command{ Use: fmt.Sprintf("%s [flags] key1: value1, key2: value2, ...", os.Args[0]), Short: "Generate shorthand structured data", - Example: fmt.Sprintf("%s foo.bar: 1, .baz: true", os.Args[0]), + Example: fmt.Sprintf("%s foo{bar: 1, baz: true}", os.Args[0]), Run: func(cmd *cobra.Command, args []string) { if len(args) == 0 && *query == "" { fmt.Println("At least one arg or --query need to be passed") @@ -37,7 +37,7 @@ func main() { } fmt.Printf("Input: %s\n", strings.Join(args, " ")) } - result, err := shorthand.GetInputWithOptions(args, shorthand.ParseOptions{ + result, isStructured, err := shorthand.GetInput(args, shorthand.ParseOptions{ EnableFileInput: true, EnableObjectDetection: true, ForceStringKeys: *format == "json", @@ -51,6 +51,10 @@ func main() { panic(err) } } + if !isStructured { + fmt.Println("Input file could not be parsed as structured data") + os.Exit(1) + } if *query != "" { if selected, ok, err := shorthand.GetPath(*query, result, shorthand.GetOptions{DebugLogger: debugLog}); ok { diff --git a/shorthand.go b/shorthand.go index a77739a..e4220cc 100644 --- a/shorthand.go +++ b/shorthand.go @@ -1,16 +1,18 @@ package shorthand import ( + "errors" "fmt" "io" "io/fs" "os" "sort" "strings" - - "gopkg.in/yaml.v3" + "unicode/utf8" ) +var ErrInvalidFile = errors.New("file cannot be parsed as structured data as it contains invalid UTF-8 characters") + func ConvertMapString(value any) any { switch tmp := value.(type) { case map[any]any: @@ -33,41 +35,49 @@ func ConvertMapString(value any) any { } // GetInput loads data from stdin (if present) and from the passed arguments, -// returning the final structure. -func GetInput(args []string) (any, error) { - return GetInputWithOptions(args, ParseOptions{ - EnableFileInput: true, - EnableObjectDetection: true, - }) -} - -func GetInputWithOptions(args []string, options ParseOptions) (any, error) { +// returning the final structure. Returns the result, whether the result is +// structured data (or raw file []byte), and if any errors occurred. +func GetInput(args []string, options ParseOptions) (any, bool, error) { stat, _ := os.Stdin.Stat() return getInput(stat.Mode(), os.Stdin, args, options) } -func getInput(mode fs.FileMode, stdinFile io.Reader, args []string, options ParseOptions) (any, error) { +func getInput(mode fs.FileMode, stdinFile io.Reader, args []string, options ParseOptions) (any, bool, error) { var stdin any if (mode & os.ModeCharDevice) == 0 { d, err := io.ReadAll(stdinFile) if err != nil { - return nil, err + return nil, false, err } - if err := yaml.Unmarshal(d, &stdin); err != nil { - if len(args) > 0 { - return nil, err - } - return nil, err + if len(args) == 0 { + // No modification requested, just pass the raw file through. + return d, false, nil + } + + if !utf8.Valid(d) { + return nil, false, ErrInvalidFile + } + + result, err := Unmarshal(string(d), ParseOptions{ + EnableFileInput: options.EnableFileInput, + ForceStringKeys: options.ForceStringKeys, + ForceFloat64Numbers: options.ForceFloat64Numbers, + DebugLogger: options.DebugLogger, + }, nil) + if err != nil { + return nil, false, err } + stdin = result } if len(args) == 0 { - return stdin, nil + return stdin, true, nil } - return Unmarshal(strings.Join(args, " "), options, stdin) + result, err := Unmarshal(strings.Join(args, " "), options, stdin) + return result, true, err } func Unmarshal(input string, options ParseOptions, existing any) (any, Error) { diff --git a/shorthand_test.go b/shorthand_test.go index b7b7b22..ccd3245 100644 --- a/shorthand_test.go +++ b/shorthand_test.go @@ -2,33 +2,95 @@ package shorthand import ( "encoding/json" + "io" + "io/fs" "strings" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +var getInputExamples = []struct { + Name string + Mode fs.FileMode + File io.Reader + Input string + JSON string + Output []byte +}{ + { + Name: "No file", + Mode: fs.ModeCharDevice, + Input: "foo[]: 2, bar.another: false, existing: null, existing[]: 1", + JSON: `{ + "foo": [2], + "bar": { + "another": false + }, + "existing": [1] + }`, + }, + { + Name: "Raw file", + File: strings.NewReader("a text file"), + Output: []byte("a text file"), + }, + { + Name: "Structured file no args", + File: strings.NewReader(`{"foo":"bar"}`), + Output: []byte(`{"foo":"bar"}`), + }, + { + Name: "JSON edit", + File: strings.NewReader(`{ + "foo": [1], + "bar": { + "baz": true + }, + "existing": [1, 2, 3] + }`), + Input: "foo[]: 2, bar.another: false, existing: null, existing[]: 1", + JSON: `{ + "foo": [1, 2], + "bar": { + "another": false, + "baz": true + }, + "existing": [1] + }`, + }, +} + func TestGetInput(t *testing.T) { - file := strings.NewReader(`{ - "foo": [1], - "bar": { - "baz": true - }, - "existing": [1, 2, 3] - }`) + for _, example := range getInputExamples { + t.Run(example.Name, func(t *testing.T) { + input := []string{} + if example.Input != "" { + input = append(input, example.Input) + } + result, isStruct, err := getInput(example.Mode, example.File, input, ParseOptions{ + EnableObjectDetection: true, + }) + msg := "" + if e, ok := err.(Error); ok { + msg = e.Pretty() + } + require.NoError(t, err, msg) - result, err := getInput(0, file, []string{"foo[]: 2, bar.another: false, existing: null, existing[]: 1"}, ParseOptions{EnableObjectDetection: true}) - assert.NoError(t, err) + if example.JSON != "" { + if !isStruct { + t.Fatal("input not recognized as structured data") + } + j, _ := json.Marshal(result) + assert.JSONEq(t, example.JSON, string(j)) + } - j, _ := json.Marshal(result) - assert.JSONEq(t, `{ - "foo": [1, 2], - "bar": { - "another": false, - "baz": true - }, - "existing": [1] - }`, string(j)) + if example.Output != nil { + assert.Equal(t, example.Output, result) + } + }) + } } var marshalExamples = []struct {