Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Dasel V3 #432

Draft
wants to merge 58 commits into
base: master
Choose a base branch
from
Draft
Changes from 1 commit
Commits
Show all changes
58 commits
Select commit Hold shift + click to select a range
c9f0b5c
Initial start on selector parser rewrite
TomWright Sep 24, 2024
20f73ab
Add spread selectors
TomWright Sep 24, 2024
e09907d
Start getting v3 core functionality working
TomWright Sep 30, 2024
f4427b6
Fix issues
TomWright Sep 30, 2024
daef53f
Merge branch 'master' into v3
TomWright Sep 30, 2024
1657e5d
Get initial binary expressions working
TomWright Oct 1, 2024
5058ae6
Get binary binding powers working
TomWright Oct 1, 2024
b792aaf
Add a null token
TomWright Oct 1, 2024
28cd36a
Implement grouping
TomWright Oct 1, 2024
ff73190
Add string concat test
TomWright Oct 1, 2024
174ec83
Add a test for setting evaluated fields
TomWright Oct 1, 2024
bf9f18f
More progress
TomWright Oct 1, 2024
482c6e0
More setup
TomWright Oct 2, 2024
d92a503
General fixes and improvements
TomWright Oct 3, 2024
1f26dbd
Add if/elseif/else statements
TomWright Oct 5, 2024
dbf367f
Small cleanup
TomWright Oct 5, 2024
2f36b1f
More improvements
TomWright Oct 6, 2024
25b2604
Update container.yaml for new selector format
TomWright Oct 6, 2024
bb3b2b4
Fix build-test.yaml
TomWright Oct 6, 2024
5d3379f
Update tests
TomWright Oct 6, 2024
8f1ea53
Delete the old dasel implementation
TomWright Oct 6, 2024
f8a5e27
Add initial branch support
TomWright Oct 7, 2024
0a7aee3
Add array support
TomWright Oct 7, 2024
1a1e6c9
More progress
TomWright Oct 10, 2024
9010bd5
Add filter support
TomWright Oct 10, 2024
797f001
Simplify map expr
TomWright Oct 10, 2024
605ec48
Add regex support
TomWright Oct 10, 2024
d857cb9
Add missing error handling
TomWright Oct 10, 2024
9999c1b
Improve spreading into object defs and rework functions
TomWright Oct 18, 2024
8bbae1e
Ensure variables are propagated and update root command
TomWright Oct 19, 2024
7a9a924
Fix tests
TomWright Oct 19, 2024
15bffc6
Fix toString and remove test file
TomWright Oct 19, 2024
5d6576e
Add reverse()
TomWright Oct 19, 2024
1b27839
Update tests and fix range
TomWright Oct 20, 2024
cfa37ca
Implement sortBy func
TomWright Oct 20, 2024
bfb0bd8
Rework JSON parser
TomWright Oct 22, 2024
e081e49
Add unary expr support and tests
TomWright Oct 23, 2024
9dbe990
Migrate to kong, make parsers extensible, add tests
TomWright Oct 24, 2024
48ec95d
Add version command
TomWright Oct 24, 2024
662076f
Default to query command
TomWright Oct 24, 2024
9f8b20e
Merge branch 'master' into v3
TomWright Oct 24, 2024
dd5953c
Remove legacy dencoding package
TomWright Oct 24, 2024
28d9744
Fix default command
TomWright Oct 24, 2024
31c8096
Remove unused code
TomWright Oct 24, 2024
99a9e8b
Fix CLI output and improve branching capability
TomWright Oct 24, 2024
0ffe7cc
Fix --root flag
TomWright Oct 24, 2024
ba0aa78
Add mix, max and fix IsNull issue
TomWright Oct 24, 2024
3ac9c8b
Reimplement yaml parser
TomWright Oct 27, 2024
4ef342c
Start implementing CSV parser
TomWright Oct 27, 2024
bbc873f
Add coalesce operator and unstable flag
TomWright Oct 28, 2024
de29bd2
Add interactive mode, fixes and improvements
TomWright Oct 29, 2024
f6f2be3
Pass stdout option
TomWright Oct 29, 2024
d67cb15
Fix bug with parsing unfinished quoted strings
TomWright Oct 29, 2024
11c723e
General fixes and improvements
TomWright Oct 30, 2024
a6074e8
Start implementing HCL parser
TomWright Oct 31, 2024
b5aa01d
HCL improvements
TomWright Oct 31, 2024
38ec57f
Deregister unimplemented XML writer
TomWright Oct 31, 2024
7e48c95
Implement first draft of HCL writer
TomWright Nov 7, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
HCL improvements
TomWright committed Oct 31, 2024
commit b5aa01d76cfc5d12555113d932a7543900dd9524
4 changes: 2 additions & 2 deletions internal/cli/command.go
Original file line number Diff line number Diff line change
@@ -46,8 +46,8 @@ func Run(stdin io.Reader, stdout, stderr io.Writer) (*kong.Context, error) {
"version": internal.Version,
},
kong.Bind(&cli.Globals),
kong.TypeMapper(reflect.TypeFor[*[]variable](), &variableMapper{}),
kong.TypeMapper(reflect.TypeFor[*[]extReadWriteFlag](), &extReadWriteFlagMapper{}),
kong.TypeMapper(reflect.TypeFor[variables](), &variableMapper{}),
kong.TypeMapper(reflect.TypeFor[extReadWriteFlags](), &extReadWriteFlagMapper{}),
kong.OptionFunc(func(k *kong.Kong) error {
k.Stdout = cli.Stdout
k.Stderr = cli.Stderr
10 changes: 5 additions & 5 deletions internal/cli/interactive.go
Original file line number Diff line number Diff line change
@@ -21,11 +21,11 @@ func NewInteractiveCmd(queryCmd *QueryCmd) *InteractiveCmd {
}

type InteractiveCmd struct {
Vars *[]variable `flag:"" name:"var" help:"Variables to pass to the query. E.g. --var foo=\"bar\" --var baz=json:file:./some/file.json"`
ExtReadFlags *[]extReadWriteFlag `flag:"" name:"read-flag" help:"Reader flag to customise parsing. E.g. --read-flag xml-mode=structured"`
ExtWriteFlags *[]extReadWriteFlag `flag:"" name:"write-flag" help:"Writer flag to customise output"`
InFormat string `flag:"" name:"in" short:"i" help:"The format of the input data."`
OutFormat string `flag:"" name:"out" short:"o" help:"The format of the output data."`
Vars variables `flag:"" name:"var" help:"Variables to pass to the query. E.g. --var foo=\"bar\" --var baz=json:file:./some/file.json"`
ExtReadFlags extReadWriteFlags `flag:"" name:"read-flag" help:"Reader flag to customise parsing. E.g. --read-flag xml-mode=structured"`
ExtWriteFlags extReadWriteFlags `flag:"" name:"write-flag" help:"Writer flag to customise output"`
InFormat string `flag:"" name:"in" short:"i" help:"The format of the input data."`
OutFormat string `flag:"" name:"out" short:"o" help:"The format of the output data."`

Query string `arg:"" help:"The query to execute." optional:"" default:""`
}
9 changes: 9 additions & 0 deletions model/value.go
Original file line number Diff line number Diff line change
@@ -272,3 +272,12 @@ func (v *Value) Len() (int, error) {

return l, nil
}

func (v *Value) Copy() (*Value, error) {
switch v.Type() {
case TypeMap:
return v.MapCopy()
default:
return nil, fmt.Errorf("copy not supported for type: %s", v.Type())
}
}
23 changes: 23 additions & 0 deletions model/value_map.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package model

import (
"errors"
"fmt"
"reflect"

@@ -58,6 +59,28 @@ func (v *Value) SetMapKey(key string, value *Value) error {
}
}

func (v *Value) MapCopy() (*Value, error) {
res := NewMapValue()
kvs, err := v.MapKeyValues()
if err != nil {
return nil, fmt.Errorf("error getting map key values: %w", err)
}
for _, kv := range kvs {
if err := res.SetMapKey(kv.Key, kv.Value); err != nil {
return nil, fmt.Errorf("error setting map key: %w", err)
}
}
return res, nil
}

func (v *Value) MapKeyExists(key string) (bool, error) {
_, err := v.GetMapKey(key)
if err != nil && !errors.As(err, &MapKeyNotFound{}) {
return false, err
}
return err == nil, nil
}

// GetMapKey returns the value at the specified key in the map.
func (v *Value) GetMapKey(key string) (*Value, error) {
switch {
6 changes: 3 additions & 3 deletions parsing/hcl/hcl.go
Original file line number Diff line number Diff line change
@@ -10,10 +10,10 @@ const (
)

var _ parsing.Reader = (*hclReader)(nil)

//var _ parsing.Writer = (*hclWriter)(nil)
var _ parsing.Writer = (*hclWriter)(nil)

func init() {
parsing.RegisterReader(HCL, newHCLReader)
parsing.RegisterWriter(HCL, newHCLWriter)
// HCL writer is not implemented yet
//parsing.RegisterWriter(HCL, newHCLWriter)
}
191 changes: 123 additions & 68 deletions parsing/hcl/reader.go
Original file line number Diff line number Diff line change
@@ -2,6 +2,7 @@ package hcl

import (
"fmt"

"github.com/hashicorp/hcl/v2"
"github.com/hashicorp/hcl/v2/gohcl"
"github.com/hashicorp/hcl/v2/hclsyntax"
@@ -11,28 +12,35 @@ import (
)

func newHCLReader(options parsing.ReaderOptions) (parsing.Reader, error) {
return &hclReader{}, nil
return &hclReader{
alwaysReadLabelsToSlice: options.Ext["hcl-block-format"] == "array",
}, nil
}

type hclReader struct{}
type hclReader struct {
alwaysReadLabelsToSlice bool
}

// Read reads a value from a byte slice.
func (j *hclReader) Read(data []byte) (*model.Value, error) {
// Reads the HCL data into a model that follows the HCL JSON spec.
// See https://github.com/hashicorp/hcl/blob/main/json%2Fspec.md
func (r *hclReader) Read(data []byte) (*model.Value, error) {
f, _ := hclsyntax.ParseConfig(data, "input", hcl.InitialPos)

body, ok := f.Body.(*hclsyntax.Body)
if !ok {
return nil, fmt.Errorf("failed to assert file body type")
}

return decodeHCLBody(body)
return r.decodeHCLBody(body)
}

func decodeHCLBody(body *hclsyntax.Body) (*model.Value, error) {
func (r *hclReader) decodeHCLBody(body *hclsyntax.Body) (*model.Value, error) {
res := model.NewMapValue()
var err error

for _, attr := range body.Attributes {
val, err := decodeHCLExpr(attr.Expr)
val, err := r.decodeHCLExpr(attr.Expr)
if err != nil {
return nil, fmt.Errorf("failed to decode attr %q: %w", attr.Name, err)
}
@@ -42,77 +50,116 @@ func decodeHCLBody(body *hclsyntax.Body) (*model.Value, error) {
}
}

blockTypeIndexes := make(map[string]int)
blockValues := make([][]*model.Value, 0)
res, err = r.decodeHCLBodyBlocks(body, res)
if err != nil {
return nil, err
}

return res, nil
}

func (r *hclReader) decodeHCLBodyBlocks(body *hclsyntax.Body, res *model.Value) (*model.Value, error) {
for _, block := range body.Blocks {
if _, ok := blockTypeIndexes[block.Type]; !ok {
blockValues = append(blockValues, make([]*model.Value, 0))
blockTypeIndexes[block.Type] = len(blockValues) - 1
if err := r.decodeHCLBlock(block, res); err != nil {
return nil, err
}
res, err := decodeHCLBlock(block)
}
return res, nil
}

func (r *hclReader) decodeHCLBlock(block *hclsyntax.Block, res *model.Value) error {
key := block.Type
v := res
for _, label := range block.Labels {
exists, err := v.MapKeyExists(key)
if err != nil {
return nil, fmt.Errorf("failed to decode block %q: %w", block.Type, err)
return err
}
blockValues[blockTypeIndexes[block.Type]] = append(blockValues[blockTypeIndexes[block.Type]], res)
}

for t, index := range blockTypeIndexes {
blocks := blockValues[index]
switch len(blocks) {
case 0:
continue
case 1:
if err := res.SetMapKey(t, blocks[0]); err != nil {
return nil, err
if exists {
keyV, err := v.GetMapKey(key)
if err != nil {
return err
}
default:
val := model.NewSliceValue()
for _, b := range blocks {
if err := val.Append(b); err != nil {
return nil, err
}
}
if err := res.SetMapKey(t, val); err != nil {
return nil, err
v = keyV
} else {
keyV := model.NewMapValue()
if err := v.SetMapKey(key, keyV); err != nil {
return err
}
v = keyV
}
}

return res, nil
}

func decodeHCLBlock(block *hclsyntax.Block) (*model.Value, error) {
res, err := decodeHCLBody(block.Body)
if err != nil {
return nil, err
key = label
}

labels := model.NewSliceValue()
for _, l := range block.Labels {
if err := labels.Append(model.NewStringValue(l)); err != nil {
return nil, err
}
body, err := r.decodeHCLBody(block.Body)
if err != nil {
return err
}

if err := res.SetMapKey("labels", labels); err != nil {
return nil, err
exists, err := v.MapKeyExists(key)
if err != nil {
return err
}
if exists {
keyV, err := v.GetMapKey(key)
if err != nil {
return err
}

if err := res.SetMapKey("type", model.NewStringValue(block.Type)); err != nil {
return nil, err
switch keyV.Type() {
case model.TypeSlice:
if err := keyV.Append(body); err != nil {
return err
}
case model.TypeMap:
// Previous value was a map.
// Create a new slice containing the previous map and the new map.
newKeyV := model.NewSliceValue()
previousKeyV, err := keyV.Copy()
if err != nil {
return err
}
if err := newKeyV.Append(previousKeyV); err != nil {
return err
}
if err := newKeyV.Append(body); err != nil {
return err
}
if err := keyV.Set(newKeyV); err != nil {
return err
}
default:
return fmt.Errorf("unexpected type: %s", keyV.Type())
}
} else {
if r.alwaysReadLabelsToSlice {
slice := model.NewSliceValue()
if err := slice.Append(body); err != nil {
return err
}
if err := v.SetMapKey(key, slice); err != nil {
return err
}
} else {
if err := v.SetMapKey(key, body); err != nil {
return err
}
}
}

return res, nil
return nil
}

func decodeHCLExpr(expr hcl.Expression) (*model.Value, error) {
func (r *hclReader) decodeHCLExpr(expr hcl.Expression) (*model.Value, error) {
source := cty.Value{}
_ = gohcl.DecodeExpression(expr, nil, &source)

return decodeCtyValue(source)
return r.decodeCtyValue(source)
}

func decodeCtyValue(source cty.Value) (res *model.Value, err error) {
func (r *hclReader) decodeCtyValue(source cty.Value) (res *model.Value, err error) {
defer func() {
r := recover()
if r != nil {
@@ -126,15 +173,7 @@ func decodeCtyValue(source cty.Value) (res *model.Value, err error) {

sourceT := source.Type()
switch {
case sourceT.IsMapType():
return nil, fmt.Errorf("map type not implemented")
case sourceT.IsListType():
return nil, fmt.Errorf("list type not implemented")
case sourceT.IsCollectionType():
return nil, fmt.Errorf("collection type not implemented")
case sourceT.IsCapsuleType():
return nil, fmt.Errorf("capsule type not implemented")
case sourceT.IsTupleType():
case sourceT.IsListType(), sourceT.IsTupleType():
res = model.NewSliceValue()
it := source.ElementIterator()
for it.Next() {
@@ -143,7 +182,7 @@ func decodeCtyValue(source cty.Value) (res *model.Value, err error) {
// Just validates the key is correct.
_, _ = k.AsBigFloat().Float64()

val, err := decodeCtyValue(v)
val, err := r.decodeCtyValue(v)
if err != nil {
return nil, fmt.Errorf("failed to decode tuple value: %w", err)
}
@@ -153,8 +192,26 @@ func decodeCtyValue(source cty.Value) (res *model.Value, err error) {
}
}
return res, nil
case sourceT.IsObjectType():
return nil, fmt.Errorf("object type not implemented")
case sourceT.IsMapType(), sourceT.IsObjectType(), sourceT.IsSetType():
v := model.NewMapValue()
it := source.ElementIterator()
for it.Next() {
k, el := it.Element()
if k.Type() != cty.String {
return nil, fmt.Errorf("object key must be a string")
}
kStr := k.AsString()

elVal, err := r.decodeCtyValue(el)
if err != nil {
return nil, fmt.Errorf("failed to decode object value: %w", err)
}

if err := v.SetMapKey(kStr, elVal); err != nil {
return nil, err
}
}
return v, nil
case sourceT.IsPrimitiveType():
switch sourceT {
case cty.String:
@@ -173,9 +230,7 @@ func decodeCtyValue(source cty.Value) (res *model.Value, err error) {
default:
return nil, fmt.Errorf("unhandled primitive type %q", source.Type())
}
case sourceT.IsSetType():
return nil, fmt.Errorf("set type not implemented")
default:
return nil, fmt.Errorf("unhandled type: %s", sourceT.FriendlyName())
return nil, fmt.Errorf("unsupported type: %s", sourceT.FriendlyName())
}
}
Loading