From d6caa7199ee642cea3cf20f159a13fc40aa4a942 Mon Sep 17 00:00:00 2001 From: Doug MacEachern Date: Fri, 17 Jan 2025 13:50:55 -0800 Subject: [PATCH 1/3] govc: add disk.metadata ls and update commands - Add disk.ls '-q' option to query disks Signed-off-by: Doug MacEachern --- cli/disk/ls.go | 95 ++++++++++++++++++++++++++++- cli/disk/manager.go | 8 +-- cli/disk/metadata/ls.go | 115 ++++++++++++++++++++++++++++++++++++ cli/disk/metadata/update.go | 80 +++++++++++++++++++++++++ govc/USAGE.md | 62 +++++++++++++++++++ govc/main.go | 1 + 6 files changed, 354 insertions(+), 7 deletions(-) create mode 100644 cli/disk/metadata/ls.go create mode 100644 cli/disk/metadata/update.go diff --git a/cli/disk/ls.go b/cli/disk/ls.go index 67f589ffc..3f1ea3440 100644 --- a/cli/disk/ls.go +++ b/cli/disk/ls.go @@ -5,6 +5,7 @@ package disk import ( + "bytes" "context" "flag" "fmt" @@ -18,6 +19,7 @@ import ( "github.com/vmware/govmomi/fault" "github.com/vmware/govmomi/units" "github.com/vmware/govmomi/vim25/types" + vslm "github.com/vmware/govmomi/vslm/types" ) type ls struct { @@ -29,6 +31,7 @@ type ls struct { category string tag string tags bool + query flags.StringList } func init() { @@ -46,6 +49,7 @@ func (cmd *ls) Register(ctx context.Context, f *flag.FlagSet) { f.StringVar(&cmd.category, "c", "", "Query tag category") f.StringVar(&cmd.tag, "t", "", "Query tag name") f.BoolVar(&cmd.tags, "T", false, "List attached tags") + f.Var(&cmd.query, "q", "Query spec") } func (cmd *ls) Usage() string { @@ -53,13 +57,29 @@ func (cmd *ls) Usage() string { } func (cmd *ls) Description() string { - return `List disk IDs on DS. + var fields vslm.VslmVsoVStorageObjectQuerySpecQueryFieldEnum + return fmt.Sprintf(`List disk IDs on DS. + +The '-q' flag can be used to match disk fields. +Each query must be in the form of: + FIELD.OP=VAL + +Where FIELD can be one of: + %s + +And OP can be one of: +%s Examples: govc disk.ls govc disk.ls -l -T govc disk.ls -l e9b06a8b-d047-4d3c-b15b-43ea9608b1a6 - govc disk.ls -c k8s-region -t us-west-2` + govc disk.ls -c k8s-region -t us-west-2 + govc disk.ls -q capacity.ge=100 # capacity in MB + govc disk.ls -q name.sw=my-disk + govc disk.ls -q metadataKey.eq=cns.k8s.pvc.namespace -q metadataValue.eq=dev`, + strings.Join(fields.Strings(), "\n "), + aliasHelp()) } type VStorageObject struct { @@ -80,6 +100,70 @@ type lsResult struct { Objects []VStorageObject `json:"objects"` } +var alias = []struct { + name string + kind vslm.VslmVsoVStorageObjectQuerySpecQueryOperatorEnum +}{ + {"eq", vslm.VslmVsoVStorageObjectQuerySpecQueryOperatorEnumEquals}, + {"ne", vslm.VslmVsoVStorageObjectQuerySpecQueryOperatorEnumNotEquals}, + {"lt", vslm.VslmVsoVStorageObjectQuerySpecQueryOperatorEnumLessThan}, + {"le", vslm.VslmVsoVStorageObjectQuerySpecQueryOperatorEnumLessThanOrEqual}, + {"gt", vslm.VslmVsoVStorageObjectQuerySpecQueryOperatorEnumGreaterThan}, + {"ge", vslm.VslmVsoVStorageObjectQuerySpecQueryOperatorEnumGreaterThanOrEqual}, + {"ct", vslm.VslmVsoVStorageObjectQuerySpecQueryOperatorEnumContains}, + {"sw", vslm.VslmVsoVStorageObjectQuerySpecQueryOperatorEnumStartsWith}, + {"ew", vslm.VslmVsoVStorageObjectQuerySpecQueryOperatorEnumEndsWith}, +} + +func opAlias(value string) string { + if len(value) != 2 { + return value + } + + for _, a := range alias { + if a.name == value { + return string(a.kind) + } + } + + return value +} + +func aliasHelp() string { + var help bytes.Buffer + + for _, a := range alias { + fmt.Fprintf(&help, " %s %s\n", a.name, a.kind) + } + + return help.String() +} + +func (cmd *ls) querySpec() ([]vslm.VslmVsoVStorageObjectQuerySpec, error) { + q := make([]vslm.VslmVsoVStorageObjectQuerySpec, len(cmd.query)) + + for i, s := range cmd.query { + val := strings.SplitN(s, "=", 2) + if len(val) != 2 { + return nil, fmt.Errorf("invalid query: %s", s) + } + + op := string(vslm.VslmVsoVStorageObjectQuerySpecQueryOperatorEnumEquals) + field := strings.SplitN(val[0], ".", 2) + if len(field) == 2 { + op = field[1] + } + + q[i] = vslm.VslmVsoVStorageObjectQuerySpec{ + QueryField: field[0], + QueryOperator: opAlias(op), + QueryValue: []string{val[1]}, + } + } + + return q, nil +} + func (r *lsResult) Write(w io.Writer) error { tw := tabwriter.NewWriter(r.cmd.Out, 2, 0, 2, ' ', 0) @@ -124,11 +208,16 @@ func (cmd *ls) Run(ctx context.Context, f *flag.FlagSet) error { filterNotFound := false ids := f.Args() + q, err := cmd.querySpec() + if err != nil { + return err + } + if len(ids) == 0 { filterNotFound = true var oids []types.ID if cmd.category == "" { - oids, err = m.List(ctx) + oids, err = m.List(ctx, q...) } else { oids, err = m.ListAttachedObjects(ctx, cmd.category, cmd.tag) } diff --git a/cli/disk/manager.go b/cli/disk/manager.go index b7522b79f..6d15cf986 100644 --- a/cli/disk/manager.go +++ b/cli/disk/manager.go @@ -139,7 +139,7 @@ func (m *Manager) Retrieve(ctx context.Context, id string) (*types.VStorageObjec return m.GlobalObjectManager.Retrieve(ctx, types.ID{Id: id}) } -func (m *Manager) List(ctx context.Context) ([]types.ID, error) { +func (m *Manager) List(ctx context.Context, qs ...vslmtypes.VslmVsoVStorageObjectQuerySpec) ([]types.ID, error) { if m.Datastore != nil { return m.ObjectManager.List(ctx, m.Datastore) } @@ -153,7 +153,7 @@ func (m *Manager) List(ctx context.Context) ([]types.ID, error) { QueryField: string(vslmtypes.VslmVsoVStorageObjectQuerySpecQueryFieldEnumId), QueryOperator: string(vslmtypes.VslmVsoVStorageObjectQuerySpecQueryOperatorEnumGreaterThan), } - var query []vslmtypes.VslmVsoVStorageObjectQuerySpec + query := qs var ids []types.ID for { @@ -164,13 +164,13 @@ func (m *Manager) List(ctx context.Context) ([]types.ID, error) { ids = append(ids, res.Id...) - if res.AllRecordsReturned { + if res.AllRecordsReturned || len(ids) == 0 { break } spec.QueryValue = []string{ids[len(ids)-1].Id} - query = []vslmtypes.VslmVsoVStorageObjectQuerySpec{spec} + query = append(qs, spec) } return ids, nil diff --git a/cli/disk/metadata/ls.go b/cli/disk/metadata/ls.go new file mode 100644 index 000000000..ddfa3ca6a --- /dev/null +++ b/cli/disk/metadata/ls.go @@ -0,0 +1,115 @@ +// © Broadcom. All Rights Reserved. +// The term “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. +// SPDX-License-Identifier: Apache-2.0 + +package metadata + +import ( + "context" + "flag" + "fmt" + "io" + "os" + "text/tabwriter" + + "github.com/vmware/govmomi/cli" + "github.com/vmware/govmomi/cli/flags" + "github.com/vmware/govmomi/vim25/types" + "github.com/vmware/govmomi/vslm" +) + +type ls struct { + *flags.OutputFlag + *flags.ClientFlag + + key string + prefix string + snapshot string +} + +func init() { + cli.Register("disk.metadata.ls", &ls{}) +} + +func (cmd *ls) Register(ctx context.Context, f *flag.FlagSet) { + cmd.OutputFlag, ctx = flags.NewOutputFlag(ctx) + cmd.OutputFlag.Register(ctx, f) + cmd.ClientFlag, ctx = flags.NewClientFlag(ctx) + cmd.ClientFlag.Register(ctx, f) + + f.StringVar(&cmd.key, "K", "", "Get value for key only") + f.StringVar(&cmd.prefix, "p", "", "Key filter prefix") + f.StringVar(&cmd.snapshot, "s", "", "Snapshot ID") +} + +func (cmd *ls) Process(ctx context.Context) error { + if err := cmd.OutputFlag.Process(ctx); err != nil { + return err + } + return cmd.ClientFlag.Process(ctx) +} + +func (cmd *ls) Usage() string { + return "ID" +} + +func (cmd *ls) Description() string { + return `List metadata for disk ID. + +Examples: + govc disk.metadata.ls 9b06a8b-d047-4d3c-b15b-43ea9608b1a6` +} + +type lsResult []types.KeyValue + +func (r lsResult) Write(w io.Writer) error { + tw := tabwriter.NewWriter(os.Stdout, 2, 0, 2, ' ', 0) + for _, data := range r { + fmt.Fprintf(tw, "%s\t%s\n", data.Key, data.Value) + } + return tw.Flush() +} + +func (r lsResult) Dump() interface{} { + return []types.KeyValue(r) +} + +func (cmd *ls) Run(ctx context.Context, f *flag.FlagSet) error { + if f.NArg() != 1 { + return flag.ErrHelp + } + + c, err := cmd.Client() + if err != nil { + return err + } + + vc, err := vslm.NewClient(ctx, c) + if err != nil { + return err + } + + m := vslm.NewGlobalObjectManager(vc) + + id := types.ID{Id: f.Arg(0)} + var data []types.KeyValue + var sid *types.ID + if cmd.snapshot != "" { + sid = &types.ID{Id: cmd.snapshot} + } + + if cmd.key != "" { + val, err := m.RetrieveMetadataValue(ctx, id, sid, cmd.key) + if err != nil { + return err + } + data = []types.KeyValue{{Key: cmd.key, Value: val}} + } else { + data, err = m.RetrieveMetadata(ctx, id, sid, cmd.prefix) + if err != nil { + return err + } + } + + return cmd.WriteResult(lsResult(data)) +} diff --git a/cli/disk/metadata/update.go b/cli/disk/metadata/update.go new file mode 100644 index 000000000..3138a0371 --- /dev/null +++ b/cli/disk/metadata/update.go @@ -0,0 +1,80 @@ +// © Broadcom. All Rights Reserved. +// The term “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. +// SPDX-License-Identifier: Apache-2.0 + +package metadata + +import ( + "context" + "flag" + "strings" + "time" + + "github.com/vmware/govmomi/cli" + "github.com/vmware/govmomi/cli/flags" + "github.com/vmware/govmomi/vim25/types" + "github.com/vmware/govmomi/vslm" +) + +type update struct { + *flags.ClientFlag + + remove flags.StringList +} + +func init() { + cli.Register("disk.metadata.update", &update{}) +} + +func (cmd *update) Register(ctx context.Context, f *flag.FlagSet) { + cmd.ClientFlag, ctx = flags.NewClientFlag(ctx) + cmd.ClientFlag.Register(ctx, f) + + f.Var(&cmd.remove, "d", "Delete keys") +} + +func (cmd *update) Usage() string { + return "ID" +} + +func (cmd *update) Description() string { + return `Update metadata for disk ID. + +Examples: + govc disk.metadata.update $id foo=bar biz=baz + govc disk.metadata.update -d foo -d biz $id` +} + +func (cmd *update) Run(ctx context.Context, f *flag.FlagSet) error { + c, err := cmd.Client() + if err != nil { + return err + } + + vc, err := vslm.NewClient(ctx, c) + if err != nil { + return err + } + + m := vslm.NewGlobalObjectManager(vc) + + id := types.ID{Id: f.Arg(0)} + + var update []types.KeyValue + + for _, arg := range f.Args()[1:] { + kv := strings.SplitN(arg, "=", 2) + if len(kv) == 1 { + kv = append(kv, "") + } + update = append(update, types.KeyValue{Key: kv[0], Value: kv[1]}) + } + + task, err := m.UpdateMetadata(ctx, id, update, cmd.remove) + if err != nil { + return err + } + + _, err = task.Wait(ctx, time.Hour) + return err +} diff --git a/govc/USAGE.md b/govc/USAGE.md index 6fff0723f..9f30096bd 100644 --- a/govc/USAGE.md +++ b/govc/USAGE.md @@ -127,6 +127,8 @@ but appear via `govc $cmd -h`: - [disk.create](#diskcreate) - [disk.detach](#diskdetach) - [disk.ls](#diskls) + - [disk.metadata.ls](#diskmetadatals) + - [disk.metadata.update](#diskmetadataupdate) - [disk.register](#diskregister) - [disk.rm](#diskrm) - [disk.snapshot.create](#disksnapshotcreate) @@ -2149,11 +2151,39 @@ Usage: govc disk.ls [OPTIONS] [ID]... List disk IDs on DS. +The '-q' flag can be used to match disk fields. +Each query must be in the form of: + FIELD.OP=VAL + +Where FIELD can be one of: + id + name + capacity + createTime + backingObjectId + datastoreMoId + metadataKey + metadataValue + +And OP can be one of: + eq equals + ne notEquals + lt lessThan + le lessThanOrEqual + gt greaterThan + ge greaterThanOrEqual + ct contains + sw startsWith + ew endsWith + Examples: govc disk.ls govc disk.ls -l -T govc disk.ls -l e9b06a8b-d047-4d3c-b15b-43ea9608b1a6 govc disk.ls -c k8s-region -t us-west-2 + govc disk.ls -q capacity.ge=100 # capacity in MB + govc disk.ls -q name.sw=my-disk + govc disk.ls -q metadataKey.eq=cns.k8s.pvc.namespace -q metadataValue.eq=dev Options: -L=false Print disk backing path instead of disk name @@ -2163,9 +2193,41 @@ Options: -c= Query tag category -ds= Datastore [GOVC_DATASTORE] -l=false Long listing format + -q=[] Query spec -t= Query tag name ``` +## disk.metadata.ls + +``` +Usage: govc disk.metadata.ls [OPTIONS] ID + +List metadata for disk ID. + +Examples: + govc disk.metadata.ls 9b06a8b-d047-4d3c-b15b-43ea9608b1a6 + +Options: + -K= Get value for key only + -p= Key filter prefix + -s= Snapshot ID +``` + +## disk.metadata.update + +``` +Usage: govc disk.metadata.update [OPTIONS] ID + +Update metadata for disk ID. + +Examples: + govc disk.metadata.update $id foo=bar biz=baz + govc disk.metadata.update -d foo -d biz $id + +Options: + -d=[] Delete keys +``` + ## disk.register ``` diff --git a/govc/main.go b/govc/main.go index 33b334e45..f72f1c97e 100644 --- a/govc/main.go +++ b/govc/main.go @@ -47,6 +47,7 @@ import ( _ "github.com/vmware/govmomi/cli/device/serial" _ "github.com/vmware/govmomi/cli/device/usb" _ "github.com/vmware/govmomi/cli/disk" + _ "github.com/vmware/govmomi/cli/disk/metadata" _ "github.com/vmware/govmomi/cli/disk/snapshot" _ "github.com/vmware/govmomi/cli/dvs" _ "github.com/vmware/govmomi/cli/dvs/portgroup" From c87a209ce3f0a5723c2d732c01814e1d85d60f4a Mon Sep 17 00:00:00 2001 From: Doug MacEachern Date: Fri, 17 Jan 2025 15:01:05 -0800 Subject: [PATCH 2/3] vcsim: add disk query and metadata support Signed-off-by: Doug MacEachern --- govc/test/disk.bats | 89 ++++++++ govc/test/test_helper.bash | 4 + simulator/vstorage_object_manager.go | 53 +++++ vslm/simulator/simulator.go | 302 +++++++++++++++++++++++++-- 4 files changed, 431 insertions(+), 17 deletions(-) diff --git a/govc/test/disk.bats b/govc/test/disk.bats index 79284adc8..c9d6f36b8 100755 --- a/govc/test/disk.bats +++ b/govc/test/disk.bats @@ -193,6 +193,52 @@ load test_helper govc disk.ls -T | grep -v US-WEST } +@test "disk.metadata" { + vcsim_env + + run govc disk.create -size 10M my-disk + assert_success + id="$output" + + run govc disk.metadata.ls "$id" + assert_success "" + + run govc disk.metadata.ls -json "$id" + assert_success null + + run govc disk.metadata.update "$id" foo=bar biz=baz + assert_success + + run govc disk.metadata.ls "$id" + assert_success + assert_output_lines 2 + + run govc disk.metadata.ls -json "$id" + assert_success + + run govc disk.metadata.ls -K foo "$id" + assert_success + assert_output_lines 1 + + run govc disk.metadata.update "$id" foo2=bar2 biz2=baz2 + assert_success + + run govc disk.metadata.ls "$id" + assert_success + assert_output_lines 4 + + run govc disk.metadata.ls -p foo "$id" + assert_success + assert_output_lines 2 + + run govc disk.metadata.update -d foo2 -d biz2 "$id" + assert_success + + run govc disk.metadata.ls "$id" + assert_success + assert_output_lines 2 +} + @test "disk.reconcile" { vcsim_env @@ -291,3 +337,46 @@ load test_helper assert_success [ ${#lines[@]} -eq 0 ] } + +@test "disk query" { + vcsim_start -ds 3 + + name=0 + size=10 + + for ds in $(govc find / -type s) ; do + for prefix in alpha beta ; do + run govc disk.create -ds "$ds" -size ${size}M $prefix-disk-$name + assert_success + id="$output" + + run govc disk.metadata.update "$id" \ + namespace=ns-$prefix \ + name=vol-$prefix-$name + assert_success + done + + name=$((name + 1)) + size=$((size + 10)) + done + + run govc disk.ls + assert_success + assert_output_lines 6 + + run govc disk.ls -q capacity.eq=10 + assert_success + assert_output_lines 2 + + run govc disk.ls -q capacity.gt=10 + assert_success + assert_output_lines 4 + + run govc disk.ls -q capacity.ge=10 -q name.sw=alpha- + assert_success + assert_output_lines 3 + + run govc disk.ls -q metadataKey.eq=namespace -q metadataValue.eq=ns-alpha + assert_success + assert_output_lines 3 +} diff --git a/govc/test/test_helper.bash b/govc/test/test_helper.bash index f4ed76e47..00fadaee0 100644 --- a/govc/test/test_helper.bash +++ b/govc/test/test_helper.bash @@ -241,6 +241,10 @@ assert_output() { assert_equal "$expected" "$output" } +assert_output_lines() { + assert_equal "$1" ${#lines[@]} +} + assert_matches() { local pattern="${1}" local actual="${2}" diff --git a/simulator/vstorage_object_manager.go b/simulator/vstorage_object_manager.go index 963ccb31f..d4ab2bdea 100644 --- a/simulator/vstorage_object_manager.go +++ b/simulator/vstorage_object_manager.go @@ -24,6 +24,7 @@ import ( type VStorageObject struct { types.VStorageObject types.VStorageObjectSnapshotInfo + Metadata []types.KeyValue } type VcenterVStorageObjectManager struct { @@ -463,3 +464,55 @@ func (m *VcenterVStorageObjectManager) ListTagsAttachedToVStorageObject(ctx *Con return body } + +func (m *VcenterVStorageObjectManager) VCenterUpdateVStorageObjectMetadataExTask(ctx *Context, req *types.VCenterUpdateVStorageObjectMetadataEx_Task) soap.HasFault { + task := CreateTask(m, "updateVStorageObjectMetadataEx", func(*Task) (types.AnyType, types.BaseMethodFault) { + obj := m.object(req.Datastore, req.Id) + if obj == nil { + return nil, new(types.InvalidArgument) + } + + var metadata []types.KeyValue + + remove := func(key string) bool { + for _, dk := range req.DeleteKeys { + if key == dk { + return true + } + } + return false + } + + for _, kv := range obj.Metadata { + if !remove(kv.Key) { + metadata = append(metadata, kv) + } + } + + update := func(kv types.KeyValue) bool { + for i := range obj.Metadata { + if obj.Metadata[i].Key == kv.Key { + obj.Metadata[i] = kv + return true + } + } + return false + } + + for _, kv := range req.Metadata { + if !update(kv) { + metadata = append(metadata, kv) + } + } + + obj.Metadata = metadata + + return nil, nil + }) + + return &methods.VCenterUpdateVStorageObjectMetadataEx_TaskBody{ + Res: &types.VCenterUpdateVStorageObjectMetadataEx_TaskResponse{ + Returnval: task.Run(ctx), + }, + } +} diff --git a/vslm/simulator/simulator.go b/vslm/simulator/simulator.go index 2028f5de8..af3f3cc66 100644 --- a/vslm/simulator/simulator.go +++ b/vslm/simulator/simulator.go @@ -5,6 +5,13 @@ package simulator import ( + "cmp" + "fmt" + "slices" + "strconv" + "strings" + "time" + "github.com/vmware/govmomi/simulator" "github.com/vmware/govmomi/vim25" vimx "github.com/vmware/govmomi/vim25/methods" @@ -75,34 +82,206 @@ type VStorageObjectManager struct { vim.ManagedObjectReference } -func (m *VStorageObjectManager) VslmListVStorageObjectForSpec(ctx *simulator.Context, req *types.VslmListVStorageObjectForSpec) soap.HasFault { - res := new(types.VslmVsoVStorageObjectQueryResult) +func matchesTime(q types.VslmVsoVStorageObjectQuerySpec, val time.Time) (bool, error) { + src, err := time.Parse(time.RFC3339Nano, q.QueryValue[0]) + if err != nil { + return false, err + } + + switch types.VslmVsoVStorageObjectQuerySpecQueryOperatorEnum(q.QueryOperator) { + case types.VslmVsoVStorageObjectQuerySpecQueryOperatorEnumEquals: + return src == val, nil + case types.VslmVsoVStorageObjectQuerySpecQueryOperatorEnumLessThan: + return val.Before(src), nil + case types.VslmVsoVStorageObjectQuerySpecQueryOperatorEnumGreaterThan: + return val.After(src), nil + default: + return false, fmt.Errorf("invalid queryOperator %s for time", q.QueryOperator) + } +} + +var longOps = map[types.VslmVsoVStorageObjectQuerySpecQueryOperatorEnum]func(int64, int64) bool{ + types.VslmVsoVStorageObjectQuerySpecQueryOperatorEnumEquals: func(a, b int64) bool { return a == b }, + types.VslmVsoVStorageObjectQuerySpecQueryOperatorEnumNotEquals: func(a, b int64) bool { return a != b }, + types.VslmVsoVStorageObjectQuerySpecQueryOperatorEnumLessThan: func(a, b int64) bool { return a < b }, + types.VslmVsoVStorageObjectQuerySpecQueryOperatorEnumGreaterThan: func(a, b int64) bool { return a > b }, + types.VslmVsoVStorageObjectQuerySpecQueryOperatorEnumLessThanOrEqual: func(a, b int64) bool { return a < b || a == b }, + types.VslmVsoVStorageObjectQuerySpecQueryOperatorEnumGreaterThanOrEqual: func(a, b int64) bool { return a > b || a == b }, +} +func matchesLong(q types.VslmVsoVStorageObjectQuerySpec, field int64) (bool, error) { + num, err := strconv.ParseInt(q.QueryValue[0], 10, 64) + if err != nil { + return false, err + } + + op, ok := longOps[types.VslmVsoVStorageObjectQuerySpecQueryOperatorEnum(q.QueryOperator)] + if !ok { + return false, fmt.Errorf("invalid QueryOperator: %s", q.QueryOperator) + } + + return op(field, int64(num)), nil +} + +var stringOps = map[types.VslmVsoVStorageObjectQuerySpecQueryOperatorEnum]func(string, string) bool{ + types.VslmVsoVStorageObjectQuerySpecQueryOperatorEnumEquals: func(a, b string) bool { return a == b }, + types.VslmVsoVStorageObjectQuerySpecQueryOperatorEnumNotEquals: func(a, b string) bool { return a != b }, + types.VslmVsoVStorageObjectQuerySpecQueryOperatorEnumLessThan: func(a, b string) bool { return a < b }, + types.VslmVsoVStorageObjectQuerySpecQueryOperatorEnumGreaterThan: func(a, b string) bool { return a > b }, + types.VslmVsoVStorageObjectQuerySpecQueryOperatorEnumLessThanOrEqual: func(a, b string) bool { return a < b || a == b }, + types.VslmVsoVStorageObjectQuerySpecQueryOperatorEnumGreaterThanOrEqual: func(a, b string) bool { return a > b || a == b }, + types.VslmVsoVStorageObjectQuerySpecQueryOperatorEnumContains: strings.Contains, + types.VslmVsoVStorageObjectQuerySpecQueryOperatorEnumStartsWith: strings.HasPrefix, + types.VslmVsoVStorageObjectQuerySpecQueryOperatorEnumEndsWith: strings.HasSuffix, +} + +func matches(obj *simulator.VStorageObject, q types.VslmVsoVStorageObjectQuerySpec) (bool, error) { + var field []string + + switch types.VslmVsoVStorageObjectQuerySpecQueryFieldEnum(q.QueryField) { + case types.VslmVsoVStorageObjectQuerySpecQueryFieldEnumCapacity: + return matchesLong(q, obj.Config.CapacityInMB) + case types.VslmVsoVStorageObjectQuerySpecQueryFieldEnumCreateTime: + return matchesTime(q, obj.Config.CreateTime) + case types.VslmVsoVStorageObjectQuerySpecQueryFieldEnumBackingObjectId: + return false, fmt.Errorf("Query field %s is not supported", q.QueryField) // Same as real VC currently + case types.VslmVsoVStorageObjectQuerySpecQueryFieldEnumDatastoreMoId: + return true, nil // Already filtered datastores + case types.VslmVsoVStorageObjectQuerySpecQueryFieldEnumId: + field = append(field, obj.Config.Id.Id) + case types.VslmVsoVStorageObjectQuerySpecQueryFieldEnumName: + field = append(field, obj.Config.Name) + case types.VslmVsoVStorageObjectQuerySpecQueryFieldEnumMetadataKey: + field = make([]string, len(obj.Metadata)) + for i := range obj.Metadata { + field[i] = obj.Metadata[i].Key + } + case types.VslmVsoVStorageObjectQuerySpecQueryFieldEnumMetadataValue: + field = make([]string, len(obj.Metadata)) + for i := range obj.Metadata { + field[i] = obj.Metadata[i].Value + } + default: + return false, fmt.Errorf("invalid QueryField: %s", q.QueryField) + } + + op, ok := stringOps[types.VslmVsoVStorageObjectQuerySpecQueryOperatorEnum(q.QueryOperator)] + if !ok { + return false, fmt.Errorf("invalid QueryOperator: %s", q.QueryOperator) + } + + for _, f := range field { + if op(f, q.QueryValue[0]) { + return true, nil + } + } + + return false, nil +} + +var ( + invalidValues = &vim.InvalidArgument{InvalidProperty: "values"} + + invalidQuery = &vim.SystemError{ + RuntimeFault: vim.RuntimeFault{ + MethodFault: vim.MethodFault{ + FaultCause: &vim.LocalizedMethodFault{ + Fault: &types.VslmFault{ + Msg: "Unexpected exception", + }, + }, + }, + }, + Reason: "Undeclared fault", + } +) + +func matchesSpec(obj *simulator.VStorageObject, query []types.VslmVsoVStorageObjectQuerySpec) (bool, *soap.Fault) { + for _, q := range query { + if len(q.QueryValue) != 1 { // Only 1 value is currently supported by vCenter + return false, simulator.Fault("", invalidValues) + } + match, err := matches(obj, q) + if err != nil { + return false, simulator.Fault(err.Error(), invalidQuery) + } + if !match { + return false, nil + } + } + return true, nil +} + +func (m *VStorageObjectManager) VslmListVStorageObjectForSpec(ctx *simulator.Context, req *types.VslmListVStorageObjectForSpec) soap.HasFault { + body := new(methods.VslmListVStorageObjectForSpecBody) vctx := ctx.For(vim25.Path) vsom := vctx.Map.VStorageObjectManager() - res.AllRecordsReturned = true // TODO: req.MaxResult + datastores := map[vim.ManagedObjectReference]bool{} + for _, q := range req.Query { + if q.QueryField == string(types.VslmVsoVStorageObjectQuerySpecQueryFieldEnumDatastoreMoId) { + for _, id := range q.QueryValue { + datastores[vim.ManagedObjectReference{ + Type: "Datastore", + Value: "datastore-" + id, + }] = true + } + } + } - for _, objs := range vsom.Catalog() { - // TODO: filter req.Query - for id, obj := range objs { - res.Id = append(res.Id, id) + var catalog []vim.VStorageObject + + for ds, objs := range vsom.Catalog() { + if len(datastores) != 0 && !datastores[ds] { + continue + } - vso := types.VslmVsoVStorageObjectResult{ - Id: id, - Name: obj.Config.Name, - CapacityInMB: obj.Config.CapacityInMB, - CreateTime: &obj.Config.CreateTime, + for _, obj := range objs { + matches, err := matchesSpec(obj, req.Query) + if err != nil { + body.Fault_ = err + return body + } + if !matches { + continue } - res.QueryResults = append(res.QueryResults, vso) + + catalog = append(catalog, obj.VStorageObject) } } - return &methods.VslmListVStorageObjectForSpecBody{ - Res: &types.VslmListVStorageObjectForSpecResponse{ - Returnval: res, - }, + // Sort as real VC does, required to support pagination + slices.SortFunc(catalog, func(a, b vim.VStorageObject) int { + return cmp.Compare(a.Config.Id.Id, b.Config.Id.Id) + }) + + res := &types.VslmVsoVStorageObjectQueryResult{ + AllRecordsReturned: true, + } + + for _, obj := range catalog { + res.Id = append(res.Id, obj.Config.Id) + + vso := types.VslmVsoVStorageObjectResult{ + Id: obj.Config.Id, + Name: obj.Config.Name, + CapacityInMB: obj.Config.CapacityInMB, + CreateTime: &obj.Config.CreateTime, + } + + res.QueryResults = append(res.QueryResults, vso) + + if len(res.QueryResults) >= int(req.MaxResult) { + res.AllRecordsReturned = false + break + } } + + body.Res = &types.VslmListVStorageObjectForSpecResponse{ + Returnval: res, + } + + return body } func (m *VStorageObjectManager) VslmRetrieveVStorageObject(ctx *simulator.Context, req *types.VslmRetrieveVStorageObject) soap.HasFault { @@ -429,6 +608,84 @@ func (m *VStorageObjectManager) VslmAttachDiskTask(ctx *simulator.Context, req * return body } +func (m *VStorageObjectManager) VslmUpdateVStorageObjectMetadataTask(ctx *simulator.Context, req *types.VslmUpdateVStorageObjectMetadata_Task) soap.HasFault { + body := new(methods.VslmUpdateVStorageObjectMetadata_TaskBody) + + vctx := ctx.For(vim25.Path) + vsom := vctx.Map.VStorageObjectManager() + + val := vsom.VCenterUpdateVStorageObjectMetadataExTask(vctx, &vim.VCenterUpdateVStorageObjectMetadataEx_Task{ + This: vsom.Self, + Id: req.Id, + Metadata: req.Metadata, + DeleteKeys: req.DeleteKeys, + Datastore: m.ds(vsom, req.Id), + }) + + if val.Fault() != nil { + body.Fault_ = val.Fault() + } else { + ref := val.(*vimx.VCenterUpdateVStorageObjectMetadataEx_TaskBody).Res.Returnval + + body.Res = &types.VslmUpdateVStorageObjectMetadata_TaskResponse{ + Returnval: newVslmTask(ctx, ref), + } + } + + return body +} + +func (m *VStorageObjectManager) VslmRetrieveVStorageObjectMetadata(ctx *simulator.Context, req *types.VslmRetrieveVStorageObjectMetadata) soap.HasFault { + body := new(methods.VslmRetrieveVStorageObjectMetadataBody) + + vctx := ctx.For(vim25.Path) + vsom := vctx.Map.VStorageObjectManager() + + obj := m.object(vsom, req.Id) + if obj == nil { + body.Fault_ = simulator.Fault("", &vim.InvalidArgument{InvalidProperty: "VolumeId"}) + } else { + body.Res = new(types.VslmRetrieveVStorageObjectMetadataResponse) + + for _, kv := range obj.Metadata { + if req.Prefix == "" || strings.HasPrefix(kv.Key, req.Prefix) { + body.Res.Returnval = append(body.Res.Returnval, kv) + } + } + } + + return body +} + +func (m *VStorageObjectManager) VslmRetrieveVStorageObjectMetadataValue(ctx *simulator.Context, req *types.VslmRetrieveVStorageObjectMetadataValue) soap.HasFault { + body := new(methods.VslmRetrieveVStorageObjectMetadataValueBody) + + vctx := ctx.For(vim25.Path) + vsom := vctx.Map.VStorageObjectManager() + + obj := m.object(vsom, req.Id) + if obj == nil { + body.Fault_ = simulator.Fault("", &vim.InvalidArgument{InvalidProperty: "VolumeId"}) + } else { + val, ok := func() (string, bool) { + for _, data := range obj.Metadata { + if data.Key == req.Key { + return data.Value, true + } + } + return "", false + }() + + if ok { + body.Res = &types.VslmRetrieveVStorageObjectMetadataValueResponse{Returnval: val} + } else { + body.Fault_ = simulator.Fault("", &vim.KeyNotFound{Key: req.Key}) + } + } + + return body +} + // VslmTask methods are just a proxy to vim25 Task methods type VslmTask struct { vim.ManagedObjectReference @@ -507,3 +764,14 @@ func (*VStorageObjectManager) ds(vsom *simulator.VcenterVStorageObjectManager, r // vsom calls will fault as they would when ID is NotFound return vim.ManagedObjectReference{Type: "Datastore"} } + +func (*VStorageObjectManager) object(vsom *simulator.VcenterVStorageObjectManager, reqID vim.ID) *simulator.VStorageObject { + for _, objs := range vsom.Catalog() { + for id, obj := range objs { + if id == reqID { + return obj + } + } + } + return nil +} From 116ba7e89fd684ded3e450ea220d4d37b9d7d6a9 Mon Sep 17 00:00:00 2001 From: Doug MacEachern Date: Tue, 21 Jan 2025 20:46:41 -0800 Subject: [PATCH 3/3] api: add vlsm.GlobalObjectManager.List method Signed-off-by: Doug MacEachern --- cli/disk/manager.go | 32 +--- vslm/global_object_manager.go | 35 +++++ vslm/global_object_manager_test.go | 228 +++++++++++++++++++++++++++++ 3 files changed, 267 insertions(+), 28 deletions(-) create mode 100644 vslm/global_object_manager_test.go diff --git a/cli/disk/manager.go b/cli/disk/manager.go index 6d15cf986..939c8bdba 100644 --- a/cli/disk/manager.go +++ b/cli/disk/manager.go @@ -144,36 +144,12 @@ func (m *Manager) List(ctx context.Context, qs ...vslmtypes.VslmVsoVStorageObjec return m.ObjectManager.List(ctx, m.Datastore) } - // TODO: move this logic to vslm.GlobalObjectManager - // Need to better understand the QuerySpec + implement in vcsim. - // For now we just want the complete list of IDs (govc disk.ls) - maxResults := int32(100) - - spec := vslmtypes.VslmVsoVStorageObjectQuerySpec{ - QueryField: string(vslmtypes.VslmVsoVStorageObjectQuerySpecQueryFieldEnumId), - QueryOperator: string(vslmtypes.VslmVsoVStorageObjectQuerySpecQueryOperatorEnumGreaterThan), - } - query := qs - var ids []types.ID - - for { - res, err := m.GlobalObjectManager.ListObjectsForSpec(ctx, query, maxResults) - if err != nil { - return nil, err - } - - ids = append(ids, res.Id...) - - if res.AllRecordsReturned || len(ids) == 0 { - break - } - - spec.QueryValue = []string{ids[len(ids)-1].Id} - - query = append(qs, spec) + res, err := m.GlobalObjectManager.List(ctx, qs...) + if err != nil { + return nil, err } - return ids, nil + return res.Id, nil } func (m *Manager) RegisterDisk(ctx context.Context, path, name string) (*types.VStorageObject, error) { diff --git a/vslm/global_object_manager.go b/vslm/global_object_manager.go index 66b5e700c..7632944bb 100644 --- a/vslm/global_object_manager.go +++ b/vslm/global_object_manager.go @@ -302,6 +302,41 @@ func (this *GlobalObjectManager) ListObjectsForSpec(ctx context.Context, query [ return res.Returnval, nil } +var DefaultMaxResults = 100 + +// List wraps ListObjectsForSpec, using maxResult = DefaultMaxResults +// and looping until AllRecordsReturned == true or error is returned. +func (this *GlobalObjectManager) List(ctx context.Context, qs ...types.VslmVsoVStorageObjectQuerySpec) (*types.VslmVsoVStorageObjectQueryResult, error) { + var res types.VslmVsoVStorageObjectQueryResult + + query := qs + + for { + page, err := this.ListObjectsForSpec(ctx, query, int32(DefaultMaxResults)) + if err != nil { + return nil, err + } + + res.Id = append(res.Id, page.Id...) + res.QueryResults = append(res.QueryResults, page.QueryResults...) + res.AllRecordsReturned = page.AllRecordsReturned + + if page.AllRecordsReturned || len(page.Id) == 0 { + break + } + + spec := types.VslmVsoVStorageObjectQuerySpec{ + QueryField: string(types.VslmVsoVStorageObjectQuerySpecQueryFieldEnumId), + QueryOperator: string(types.VslmVsoVStorageObjectQuerySpecQueryOperatorEnumGreaterThan), + QueryValue: []string{page.Id[len(page.Id)-1].Id}, + } + + query = append(qs, spec) + } + + return &res, nil +} + func (this *GlobalObjectManager) Clone(ctx context.Context, id vim.ID, spec vim.VslmCloneSpec) (*Task, error) { req := types.VslmCloneVStorageObject_Task{ This: this.Reference(), diff --git a/vslm/global_object_manager_test.go b/vslm/global_object_manager_test.go new file mode 100644 index 000000000..232e86792 --- /dev/null +++ b/vslm/global_object_manager_test.go @@ -0,0 +1,228 @@ +// © Broadcom. All Rights Reserved. +// The term “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. +// SPDX-License-Identifier: Apache-2.0 + +package vslm_test + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/vmware/govmomi/find" + "github.com/vmware/govmomi/simulator" + "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/types" + "github.com/vmware/govmomi/vslm" + _ "github.com/vmware/govmomi/vslm/simulator" + vso "github.com/vmware/govmomi/vslm/types" +) + +func TestList(t *testing.T) { + model := simulator.VPX() + model.Datastore = 4 + + now := time.Now().Format(time.RFC3339Nano) + + simulator.Test(func(ctx context.Context, vc *vim25.Client) { + datastores, err := find.NewFinder(vc).DatastoreList(ctx, "*") + if err != nil { + t.Fatal(err) + } + + c, err := vslm.NewClient(ctx, vc) + if err != nil { + t.Fatal(err) + } + + m := vslm.NewGlobalObjectManager(c) + + namespaces := []string{"foo", "bar", "baz"} + + dsIDs := make([]string, len(datastores)) + + for i, ds := range datastores { + dsIDs[i] = strings.SplitN(ds.Reference().Value, "-", 2)[1] + + for j, ns := range namespaces { + spec := types.VslmCreateSpec{ + Name: fmt.Sprintf("%s-disk-%d", ns, i+j), + CapacityInMB: int64(i+j) * 10, + BackingSpec: &types.VslmCreateSpecDiskFileBackingSpec{ + VslmCreateSpecBackingSpec: types.VslmCreateSpecBackingSpec{ + Datastore: ds.Reference(), + }, + }, + } + t.Logf("CreateDisk %s (%dMB) on datastore-%s", spec.Name, spec.CapacityInMB, dsIDs[i]) + task, err := m.CreateDisk(ctx, spec) + if err != nil { + t.Fatal(err) + } + disk, err := task.Wait(ctx, time.Hour) + if err != nil { + t.Fatal(err) + } + + id := disk.(types.VStorageObject).Config.Id + + metadata := []types.KeyValue{ + {Key: "namespace", Value: ns}, + {Key: "name", Value: spec.Name}, + } + + task, err = m.UpdateMetadata(ctx, id, metadata, nil) + if err != nil { + t.Fatal(err) + } + _, err = task.Wait(ctx, time.Hour) + if err != nil { + t.Fatal(err) + } + } + } + + tests := []struct { + expect int + query []vso.VslmVsoVStorageObjectQuerySpec + }{ + {model.Datastore * len(namespaces), nil}, + {-1, []vso.VslmVsoVStorageObjectQuerySpec{{ + QueryField: "invalid", + QueryOperator: string(vso.VslmVsoVStorageObjectQuerySpecQueryOperatorEnumEquals), + QueryValue: []string{"any"}, + }}}, + {-1, []vso.VslmVsoVStorageObjectQuerySpec{{ + QueryField: string(vso.VslmVsoVStorageObjectQuerySpecQueryFieldEnumName), + QueryOperator: "invalid", + QueryValue: []string{"any"}, + }}}, + {-1, []vso.VslmVsoVStorageObjectQuerySpec{{ + QueryField: string(vso.VslmVsoVStorageObjectQuerySpecQueryFieldEnumName), + QueryOperator: string(vso.VslmVsoVStorageObjectQuerySpecQueryOperatorEnumEquals), + QueryValue: nil, + }}}, + {-1, []vso.VslmVsoVStorageObjectQuerySpec{{ + QueryField: string(vso.VslmVsoVStorageObjectQuerySpecQueryFieldEnumName), + QueryOperator: string(vso.VslmVsoVStorageObjectQuerySpecQueryOperatorEnumEquals), + QueryValue: []string{"one", "two"}, + }}}, + {-1, []vso.VslmVsoVStorageObjectQuerySpec{{ + QueryField: string(vso.VslmVsoVStorageObjectQuerySpecQueryFieldEnumCapacity), + QueryOperator: string(vso.VslmVsoVStorageObjectQuerySpecQueryOperatorEnumStartsWith), + QueryValue: []string{"10"}, + }}}, + {-1, []vso.VslmVsoVStorageObjectQuerySpec{{ + QueryField: string(vso.VslmVsoVStorageObjectQuerySpecQueryFieldEnumCapacity), + QueryOperator: string(vso.VslmVsoVStorageObjectQuerySpecQueryOperatorEnumGreaterThan), + QueryValue: []string{"ten"}, + }}}, + {3, []vso.VslmVsoVStorageObjectQuerySpec{{ + QueryField: string(vso.VslmVsoVStorageObjectQuerySpecQueryFieldEnumCapacity), + QueryOperator: string(vso.VslmVsoVStorageObjectQuerySpecQueryOperatorEnumGreaterThan), + QueryValue: []string{"30"}, + }}}, + {0, []vso.VslmVsoVStorageObjectQuerySpec{{ + QueryField: string(vso.VslmVsoVStorageObjectQuerySpecQueryFieldEnumCapacity), + QueryOperator: string(vso.VslmVsoVStorageObjectQuerySpecQueryOperatorEnumGreaterThan), + QueryValue: []string{"5000"}, + }}}, + {len(namespaces), []vso.VslmVsoVStorageObjectQuerySpec{{ + QueryField: string(vso.VslmVsoVStorageObjectQuerySpecQueryFieldEnumDatastoreMoId), + QueryOperator: string(vso.VslmVsoVStorageObjectQuerySpecQueryOperatorEnumEquals), + QueryValue: []string{dsIDs[0]}, + }}}, + {model.Datastore, []vso.VslmVsoVStorageObjectQuerySpec{ + { + QueryField: string(vso.VslmVsoVStorageObjectQuerySpecQueryFieldEnumMetadataKey), + QueryOperator: string(vso.VslmVsoVStorageObjectQuerySpecQueryOperatorEnumEquals), + QueryValue: []string{"namespace"}, + }, + { + QueryField: string(vso.VslmVsoVStorageObjectQuerySpecQueryFieldEnumMetadataValue), + QueryOperator: string(vso.VslmVsoVStorageObjectQuerySpecQueryOperatorEnumEquals), + QueryValue: []string{namespaces[0]}, + }, + }}, + {model.Datastore, []vso.VslmVsoVStorageObjectQuerySpec{{ + QueryField: string(vso.VslmVsoVStorageObjectQuerySpecQueryFieldEnumName), + QueryOperator: string(vso.VslmVsoVStorageObjectQuerySpecQueryOperatorEnumStartsWith), + QueryValue: []string{namespaces[1]}, + }}}, + {model.Datastore, []vso.VslmVsoVStorageObjectQuerySpec{{ + QueryField: string(vso.VslmVsoVStorageObjectQuerySpecQueryFieldEnumName), + QueryOperator: string(vso.VslmVsoVStorageObjectQuerySpecQueryOperatorEnumStartsWith), + QueryValue: []string{namespaces[1]}, + }}}, + {model.Datastore * len(namespaces), []vso.VslmVsoVStorageObjectQuerySpec{{ + QueryField: string(vso.VslmVsoVStorageObjectQuerySpecQueryFieldEnumName), + QueryOperator: string(vso.VslmVsoVStorageObjectQuerySpecQueryOperatorEnumContains), + QueryValue: []string{"disk"}, + }}}, + {0, []vso.VslmVsoVStorageObjectQuerySpec{ + { + QueryField: string(vso.VslmVsoVStorageObjectQuerySpecQueryFieldEnumName), + QueryOperator: string(vso.VslmVsoVStorageObjectQuerySpecQueryOperatorEnumStartsWith), + QueryValue: []string{namespaces[0]}, + }, + { + QueryField: string(vso.VslmVsoVStorageObjectQuerySpecQueryFieldEnumName), + QueryOperator: string(vso.VslmVsoVStorageObjectQuerySpecQueryOperatorEnumStartsWith), + QueryValue: []string{namespaces[1]}, + }, + }}, + {model.Datastore * len(namespaces), []vso.VslmVsoVStorageObjectQuerySpec{{ + QueryField: string(vso.VslmVsoVStorageObjectQuerySpecQueryFieldEnumCreateTime), + QueryOperator: string(vso.VslmVsoVStorageObjectQuerySpecQueryOperatorEnumGreaterThan), + QueryValue: []string{now}, + }}}, + {0, []vso.VslmVsoVStorageObjectQuerySpec{{ + QueryField: string(vso.VslmVsoVStorageObjectQuerySpecQueryFieldEnumCreateTime), + QueryOperator: string(vso.VslmVsoVStorageObjectQuerySpecQueryOperatorEnumLessThan), + QueryValue: []string{now}, + }}}, + {-1, []vso.VslmVsoVStorageObjectQuerySpec{{ + QueryField: string(vso.VslmVsoVStorageObjectQuerySpecQueryFieldEnumCreateTime), + QueryOperator: string(vso.VslmVsoVStorageObjectQuerySpecQueryOperatorEnumContains), + QueryValue: []string{now}, + }}}, + } + + for _, test := range tests { + if test.expect > 2 { + vslm.DefaultMaxResults = test.expect / 2 // test pagination + } + t.Run(queryString(test.query), func(t *testing.T) { + res, err := m.List(ctx, test.query...) + if test.expect == -1 { + if err == nil { + t.Error("expected error") + } + } else { + if err != nil { + t.Fatal(err) + } + + if len(res.Id) != test.expect { + t.Errorf("expected %d, got: %d", test.expect, len(res.Id)) + } + } + }) + } + }, model) +} + +func queryString(query []vso.VslmVsoVStorageObjectQuerySpec) string { + if query == nil { + return "no query" + } + res := make([]string, len(query)) + for i, q := range query { + res[i] = fmt.Sprintf("%s.%s=%s", + q.QueryField, q.QueryOperator, + strings.Join(q.QueryValue, ",")) + } + return strings.Join(res, " and ") +}