From 9050abc0171d404afa6663a3a19a031dca1e16f6 Mon Sep 17 00:00:00 2001 From: Alison Burgess Date: Fri, 29 Mar 2024 15:36:14 -0700 Subject: [PATCH 01/50] feat: adding api to get k8s version in manifest.json --- apiserver/getcachedvhdimages.go | 50 +++++++++++++++++++++++++++++++++ apiserver/routers.go | 6 ++++ pkg/agent/bakerapi.go | 27 ++++++++++++++++++ pkg/agent/bakerapi_test.go | 13 +++++++++ pkg/agent/datamodel/types.go | 4 +++ pkg/agent/utils.go | 11 ++++++++ pkg/agent/utils_test.go | 9 ++++++ 7 files changed, 120 insertions(+) create mode 100644 apiserver/getcachedvhdimages.go diff --git a/apiserver/getcachedvhdimages.go b/apiserver/getcachedvhdimages.go new file mode 100644 index 00000000000..604adee286f --- /dev/null +++ b/apiserver/getcachedvhdimages.go @@ -0,0 +1,50 @@ +package apiserver + +import ( + "encoding/json" + "fmt" + "log" + "net/http" + + agent "github.com/Azure/agentbaker/pkg/agent" + "github.com/Azure/agentbaker/pkg/agent/datamodel" +) + +const ( + // RoutePathGetCachedVHDImages the route path to get cached vhd images. + RoutePathGetCachedVHDImages string = "/getcachedvhdimages" +) + +// GetCachedVHDImages endpoint for getting cached VHD images. +func (api *APIServer) GetCachedVHDImages(w http.ResponseWriter, r *http.Request) { + var config datamodel.GetLatestSigImageConfigRequest + err := json.NewDecoder(r.Body).Decode(&config) + if err != nil { + log.Println(err.Error()) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + agentBaker, err := agent.NewAgentBaker() + if err != nil { + log.Println(err.Error()) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + images, err := agentBaker.GetCachedVHDImages() + if err != nil { + log.Println(err.Error()) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + result, err := json.Marshal(images) + if err != nil { + log.Println(err.Error()) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, string(result)) +} diff --git a/apiserver/routers.go b/apiserver/routers.go index 73afb7e22f3..8566c038ae6 100644 --- a/apiserver/routers.go +++ b/apiserver/routers.go @@ -39,6 +39,12 @@ func (api *APIServer) NewRouter() *mux.Router { Path(RoutePathDistroSIGImageConfig). Name("GetDistroSigImageConfig"). HandlerFunc(api.GetDistroSigImageConfig) + + router. + Methods("POST"). + Path(RoutePathGetCachedVHDImages). + Name("GetCachedVHDImages"). + HandlerFunc(api.GetCachedVHDImages) router.Methods("GET").Path("/healthz").Name("healthz").HandlerFunc(healthz) diff --git a/pkg/agent/bakerapi.go b/pkg/agent/bakerapi.go index 993c6c6090f..d6752c98758 100644 --- a/pkg/agent/bakerapi.go +++ b/pkg/agent/bakerapi.go @@ -5,7 +5,10 @@ package agent import ( "context" + "encoding/json" "fmt" + "os" + "path/filepath" "github.com/Azure/agentbaker/pkg/agent/datamodel" ) @@ -15,12 +18,19 @@ type AgentBaker interface { GetNodeBootstrapping(ctx context.Context, config *datamodel.NodeBootstrappingConfiguration) (*datamodel.NodeBootstrapping, error) GetLatestSigImageConfig(sigConfig datamodel.SIGConfig, region string, distro datamodel.Distro) (*datamodel.SigImageConfig, error) GetDistroSigImageConfig(sigConfig datamodel.SIGConfig, region string) (map[datamodel.Distro]datamodel.SigImageConfig, error) + GetCachedVHDImages() ([]string, error) } func NewAgentBaker() (AgentBaker, error) { return &agentBakerImpl{}, nil } +type Manifest struct { + Kubernetes struct { + Versions []string `json:"versions"` + } `json:"kubernetes"` +} + type agentBakerImpl struct{} //nolint:revive, nolintlint // ctx is not used, but may be in the future @@ -130,3 +140,20 @@ func (agentBaker *agentBakerImpl) GetDistroSigImageConfig( return allDistros, nil } + +func (agentBaker *agentBakerImpl) GetCachedVHDImages() ([]string, error) { + pwd, _ := os.Getwd() + manifestFilePath := filepath.Join(pwd, "../../parts/linux/cloud-init/artifacts/manifest.json") + + data, err := os.ReadFile(manifestFilePath) + if err != nil { + return nil, fmt.Errorf("error reading manifest.json file: %v", err) + } + data = trimEOF(data) + + var manifest Manifest + if err := json.Unmarshal(data, &manifest); err != nil { + return nil, fmt.Errorf("error parsing JSON data: %v", err) + } + return manifest.Kubernetes.Versions, nil +} diff --git a/pkg/agent/bakerapi_test.go b/pkg/agent/bakerapi_test.go index 8b2fd649f42..fadcb836caf 100644 --- a/pkg/agent/bakerapi_test.go +++ b/pkg/agent/bakerapi_test.go @@ -252,4 +252,17 @@ var _ = Describe("AgentBaker API implementation tests", func() { Expect(err).To(HaveOccurred()) }) }) + + Context("GetCachedVHDImages", func() { + It("should return the k8s version in manifest.json", func() { + agentBaker, err := NewAgentBaker() + Expect(err).NotTo(HaveOccurred()) + + images, err := agentBaker.GetCachedVHDImages() + Expect(err).NotTo(HaveOccurred()) + + cachedVersions := []string{"1.26.6", "1.26.10", "1.26.12", "1.27.3", "1.27.7", "1.27.9", "1.28.1", "1.28.3", "1.28.5", "1.29.0", "1.29.2"} + Expect(images).To(Equal(cachedVersions)) + }) + }) }) diff --git a/pkg/agent/datamodel/types.go b/pkg/agent/datamodel/types.go index ce403494514..e05cd325d63 100644 --- a/pkg/agent/datamodel/types.go +++ b/pkg/agent/datamodel/types.go @@ -1628,6 +1628,10 @@ type GetLatestSigImageConfigRequest struct { Distro Distro } +type GetCachedVHDImagesRequest struct { + images []string +} + // NodeBootstrappingConfiguration represents configurations for node bootstrapping. // //nolint:musttag // tags can be added if deemed necessary diff --git a/pkg/agent/utils.go b/pkg/agent/utils.go index af54e3f5353..68cedfcf4b3 100644 --- a/pkg/agent/utils.go +++ b/pkg/agent/utils.go @@ -644,3 +644,14 @@ func addFeatureGateString(featureGates string, key string, value bool) string { } return strings.Join(pairs, ",") } + +func trimEOF(data []byte) []byte { + eofIndex := bytes.LastIndex(data, []byte("#EOF")) + if eofIndex != -1 { // #EOF found + newlineIndex := bytes.LastIndex(data[:eofIndex], []byte("\n")) + if newlineIndex != -1 { + return data[:newlineIndex] + } + } + return data +} diff --git a/pkg/agent/utils_test.go b/pkg/agent/utils_test.go index 0739765450f..ee6267d1216 100644 --- a/pkg/agent/utils_test.go +++ b/pkg/agent/utils_test.go @@ -727,3 +727,12 @@ var _ = Describe("Test removeComments", func() { }) }) + +var _ = Describe("trimEOF should remove the #EOF at the end of a file", func() { + It("Should remove #EOF at the end of a file", func() { + input := []byte(`"versions":["1.26.6","1.26.10","1.26.12","1.27.3","1.27.7","1.27.9","1.28.1","1.28.3","1.28.5","1.29.0","1.29.2"]},"_template":{"fileName":"","downloadLocation":"","downloadURL":"","versions":[]}}` + "\n#EOF") + expected := `"versions":["1.26.6","1.26.10","1.26.12","1.27.3","1.27.7","1.27.9","1.28.1","1.28.3","1.28.5","1.29.0","1.29.2"]},"_template":{"fileName":"","downloadLocation":"","downloadURL":"","versions":[]}}` + result := trimEOF(input) + Expect(string(result)).To(Equal(expected)) + }) +}) From 99e81988b8d8cc8b57305ef1a470a7ac63df4d58 Mon Sep 17 00:00:00 2001 From: Alison Burgess Date: Fri, 29 Mar 2024 15:41:53 -0700 Subject: [PATCH 02/50] don't need any inputs --- apiserver/getcachedvhdimages.go | 9 --------- pkg/agent/datamodel/types.go | 4 ---- 2 files changed, 13 deletions(-) diff --git a/apiserver/getcachedvhdimages.go b/apiserver/getcachedvhdimages.go index 604adee286f..829b6d034bf 100644 --- a/apiserver/getcachedvhdimages.go +++ b/apiserver/getcachedvhdimages.go @@ -7,7 +7,6 @@ import ( "net/http" agent "github.com/Azure/agentbaker/pkg/agent" - "github.com/Azure/agentbaker/pkg/agent/datamodel" ) const ( @@ -17,14 +16,6 @@ const ( // GetCachedVHDImages endpoint for getting cached VHD images. func (api *APIServer) GetCachedVHDImages(w http.ResponseWriter, r *http.Request) { - var config datamodel.GetLatestSigImageConfigRequest - err := json.NewDecoder(r.Body).Decode(&config) - if err != nil { - log.Println(err.Error()) - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - agentBaker, err := agent.NewAgentBaker() if err != nil { log.Println(err.Error()) diff --git a/pkg/agent/datamodel/types.go b/pkg/agent/datamodel/types.go index e05cd325d63..ce403494514 100644 --- a/pkg/agent/datamodel/types.go +++ b/pkg/agent/datamodel/types.go @@ -1628,10 +1628,6 @@ type GetLatestSigImageConfigRequest struct { Distro Distro } -type GetCachedVHDImagesRequest struct { - images []string -} - // NodeBootstrappingConfiguration represents configurations for node bootstrapping. // //nolint:musttag // tags can be added if deemed necessary From d1b0a600ba760a8d91287e0537ffb6765b7c7f93 Mon Sep 17 00:00:00 2001 From: Alison Burgess Date: Fri, 29 Mar 2024 15:44:39 -0700 Subject: [PATCH 03/50] fix linter --- pkg/agent/bakerapi.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pkg/agent/bakerapi.go b/pkg/agent/bakerapi.go index d6752c98758..93dfe625fa9 100644 --- a/pkg/agent/bakerapi.go +++ b/pkg/agent/bakerapi.go @@ -142,18 +142,21 @@ func (agentBaker *agentBakerImpl) GetDistroSigImageConfig( } func (agentBaker *agentBakerImpl) GetCachedVHDImages() ([]string, error) { - pwd, _ := os.Getwd() + pwd, err := os.Getwd() + if err != nil { + return nil, fmt.Errorf("error getting current working directory: %w", err) + } manifestFilePath := filepath.Join(pwd, "../../parts/linux/cloud-init/artifacts/manifest.json") data, err := os.ReadFile(manifestFilePath) if err != nil { - return nil, fmt.Errorf("error reading manifest.json file: %v", err) + return nil, fmt.Errorf("error reading manifest.json file: %w", err) } data = trimEOF(data) var manifest Manifest if err := json.Unmarshal(data, &manifest); err != nil { - return nil, fmt.Errorf("error parsing JSON data: %v", err) + return nil, fmt.Errorf("error parsing JSON data: %w", err) } return manifest.Kubernetes.Versions, nil } From e85e450fe011b3f8fa2a6e17f92b88f0d74b7ff1 Mon Sep 17 00:00:00 2001 From: Alison Burgess Date: Fri, 29 Mar 2024 15:49:11 -0700 Subject: [PATCH 04/50] fix linter --- pkg/agent/bakerapi.go | 2 +- pkg/agent/datamodel/sig_config.go | 2 +- pkg/agent/datamodel/types.go | 6 +++--- pkg/agent/utils_test.go | 1 + 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/pkg/agent/bakerapi.go b/pkg/agent/bakerapi.go index 93dfe625fa9..f438fa6b67a 100644 --- a/pkg/agent/bakerapi.go +++ b/pkg/agent/bakerapi.go @@ -155,7 +155,7 @@ func (agentBaker *agentBakerImpl) GetCachedVHDImages() ([]string, error) { data = trimEOF(data) var manifest Manifest - if err := json.Unmarshal(data, &manifest); err != nil { + if err = json.Unmarshal(data, &manifest); err != nil { return nil, fmt.Errorf("error parsing JSON data: %w", err) } return manifest.Kubernetes.Versions, nil diff --git a/pkg/agent/datamodel/sig_config.go b/pkg/agent/datamodel/sig_config.go index 01e98cb2be6..db2fa35b7dc 100644 --- a/pkg/agent/datamodel/sig_config.go +++ b/pkg/agent/datamodel/sig_config.go @@ -258,7 +258,7 @@ func (d Distro) IsWindowsPIRDistro() bool { // SigImageConfigTemplate represents the SIG image configuration template. // -//nolint:musttag // tags can be added if deemed necessary + type SigImageConfigTemplate struct { ResourceGroup string Gallery string diff --git a/pkg/agent/datamodel/types.go b/pkg/agent/datamodel/types.go index ce403494514..fef58311823 100644 --- a/pkg/agent/datamodel/types.go +++ b/pkg/agent/datamodel/types.go @@ -1621,7 +1621,7 @@ type K8sComponents struct { // GetLatestSigImageConfigRequest describes the input for a GetLatestSigImageConfig HTTP request. // This is mostly a wrapper over existing types so RP doesn't have to manually construct JSON. // -//nolint:musttag // tags can be added if deemed necessary + type GetLatestSigImageConfigRequest struct { SIGConfig SIGConfig Region string @@ -1630,7 +1630,7 @@ type GetLatestSigImageConfigRequest struct { // NodeBootstrappingConfiguration represents configurations for node bootstrapping. // -//nolint:musttag // tags can be added if deemed necessary + type NodeBootstrappingConfiguration struct { ContainerService *ContainerService CloudSpecConfig *AzureEnvironmentSpecConfig @@ -1694,7 +1694,7 @@ const ( // NodeBootstrapping represents the custom data, CSE, and OS image info needed for node bootstrapping. // -//nolint:musttag // tags can be added if deemed necessary + type NodeBootstrapping struct { CustomData string CSE string diff --git a/pkg/agent/utils_test.go b/pkg/agent/utils_test.go index ee6267d1216..137f12241f1 100644 --- a/pkg/agent/utils_test.go +++ b/pkg/agent/utils_test.go @@ -728,6 +728,7 @@ var _ = Describe("Test removeComments", func() { }) +//nolint:lll var _ = Describe("trimEOF should remove the #EOF at the end of a file", func() { It("Should remove #EOF at the end of a file", func() { input := []byte(`"versions":["1.26.6","1.26.10","1.26.12","1.27.3","1.27.7","1.27.9","1.28.1","1.28.3","1.28.5","1.29.0","1.29.2"]},"_template":{"fileName":"","downloadLocation":"","downloadURL":"","versions":[]}}` + "\n#EOF") From 9fb7e245496ec0f80171ace6476fb6bcad65ba77 Mon Sep 17 00:00:00 2001 From: Alison Burgess Date: Fri, 29 Mar 2024 15:54:09 -0700 Subject: [PATCH 05/50] fix formatting --- apiserver/routers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apiserver/routers.go b/apiserver/routers.go index 8566c038ae6..bed60e8b379 100644 --- a/apiserver/routers.go +++ b/apiserver/routers.go @@ -39,7 +39,7 @@ func (api *APIServer) NewRouter() *mux.Router { Path(RoutePathDistroSIGImageConfig). Name("GetDistroSigImageConfig"). HandlerFunc(api.GetDistroSigImageConfig) - + router. Methods("POST"). Path(RoutePathGetCachedVHDImages). From af40354f5757031194a86e75d068bf2b8551c791 Mon Sep 17 00:00:00 2001 From: Alison Burgess Date: Mon, 1 Apr 2024 13:28:28 -0700 Subject: [PATCH 06/50] rename to k8versions --- apiserver/getcachedvhdimages.go | 10 +++++----- apiserver/routers.go | 6 +++--- pkg/agent/bakerapi.go | 4 ++-- pkg/agent/bakerapi_test.go | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/apiserver/getcachedvhdimages.go b/apiserver/getcachedvhdimages.go index 829b6d034bf..b124c211d05 100644 --- a/apiserver/getcachedvhdimages.go +++ b/apiserver/getcachedvhdimages.go @@ -10,12 +10,12 @@ import ( ) const ( - // RoutePathGetCachedVHDImages the route path to get cached vhd images. - RoutePathGetCachedVHDImages string = "/getcachedvhdimages" + // RoutePathGetCachedK8sVersions the route path to get cached vhd images. + RoutePathGetCachedK8sVersions string = "/getcachedk8sversions" ) -// GetCachedVHDImages endpoint for getting cached VHD images. -func (api *APIServer) GetCachedVHDImages(w http.ResponseWriter, r *http.Request) { +// GetCachedK8sVersions endpoint for getting cached VHD images. +func (api *APIServer) GetCachedK8sVersions(w http.ResponseWriter, r *http.Request) { agentBaker, err := agent.NewAgentBaker() if err != nil { log.Println(err.Error()) @@ -23,7 +23,7 @@ func (api *APIServer) GetCachedVHDImages(w http.ResponseWriter, r *http.Request) return } - images, err := agentBaker.GetCachedVHDImages() + images, err := agentBaker.GetCachedK8sVersions() if err != nil { log.Println(err.Error()) http.Error(w, err.Error(), http.StatusBadRequest) diff --git a/apiserver/routers.go b/apiserver/routers.go index bed60e8b379..fef92f5459f 100644 --- a/apiserver/routers.go +++ b/apiserver/routers.go @@ -42,9 +42,9 @@ func (api *APIServer) NewRouter() *mux.Router { router. Methods("POST"). - Path(RoutePathGetCachedVHDImages). - Name("GetCachedVHDImages"). - HandlerFunc(api.GetCachedVHDImages) + Path(RoutePathGetCachedK8sVersions). + Name("GetCachedK8sVersions"). + HandlerFunc(api.GetCachedK8sVersions) router.Methods("GET").Path("/healthz").Name("healthz").HandlerFunc(healthz) diff --git a/pkg/agent/bakerapi.go b/pkg/agent/bakerapi.go index f438fa6b67a..3d949c01847 100644 --- a/pkg/agent/bakerapi.go +++ b/pkg/agent/bakerapi.go @@ -18,7 +18,7 @@ type AgentBaker interface { GetNodeBootstrapping(ctx context.Context, config *datamodel.NodeBootstrappingConfiguration) (*datamodel.NodeBootstrapping, error) GetLatestSigImageConfig(sigConfig datamodel.SIGConfig, region string, distro datamodel.Distro) (*datamodel.SigImageConfig, error) GetDistroSigImageConfig(sigConfig datamodel.SIGConfig, region string) (map[datamodel.Distro]datamodel.SigImageConfig, error) - GetCachedVHDImages() ([]string, error) + GetCachedK8sVersions() ([]string, error) } func NewAgentBaker() (AgentBaker, error) { @@ -141,7 +141,7 @@ func (agentBaker *agentBakerImpl) GetDistroSigImageConfig( return allDistros, nil } -func (agentBaker *agentBakerImpl) GetCachedVHDImages() ([]string, error) { +func (agentBaker *agentBakerImpl) GetCachedK8sVersions() ([]string, error) { pwd, err := os.Getwd() if err != nil { return nil, fmt.Errorf("error getting current working directory: %w", err) diff --git a/pkg/agent/bakerapi_test.go b/pkg/agent/bakerapi_test.go index fadcb836caf..ce1b3f168ee 100644 --- a/pkg/agent/bakerapi_test.go +++ b/pkg/agent/bakerapi_test.go @@ -253,12 +253,12 @@ var _ = Describe("AgentBaker API implementation tests", func() { }) }) - Context("GetCachedVHDImages", func() { + Context("GetCachedK8sVersions", func() { It("should return the k8s version in manifest.json", func() { agentBaker, err := NewAgentBaker() Expect(err).NotTo(HaveOccurred()) - images, err := agentBaker.GetCachedVHDImages() + images, err := agentBaker.GetCachedK8sVersions() Expect(err).NotTo(HaveOccurred()) cachedVersions := []string{"1.26.6", "1.26.10", "1.26.12", "1.27.3", "1.27.7", "1.27.9", "1.28.1", "1.28.3", "1.28.5", "1.29.0", "1.29.2"} From 9a85d8f593c0a79d83b01305a18378f7b7a1ce51 Mon Sep 17 00:00:00 2001 From: Alison Burgess Date: Mon, 1 Apr 2024 13:30:47 -0700 Subject: [PATCH 07/50] use hardcoded path --- pkg/agent/bakerapi.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/pkg/agent/bakerapi.go b/pkg/agent/bakerapi.go index 3d949c01847..9c9f2c298b8 100644 --- a/pkg/agent/bakerapi.go +++ b/pkg/agent/bakerapi.go @@ -8,7 +8,6 @@ import ( "encoding/json" "fmt" "os" - "path/filepath" "github.com/Azure/agentbaker/pkg/agent/datamodel" ) @@ -142,12 +141,7 @@ func (agentBaker *agentBakerImpl) GetDistroSigImageConfig( } func (agentBaker *agentBakerImpl) GetCachedK8sVersions() ([]string, error) { - pwd, err := os.Getwd() - if err != nil { - return nil, fmt.Errorf("error getting current working directory: %w", err) - } - manifestFilePath := filepath.Join(pwd, "../../parts/linux/cloud-init/artifacts/manifest.json") - + manifestFilePath := "../../parts/linux/cloud-init/artifacts/manifest.json" data, err := os.ReadFile(manifestFilePath) if err != nil { return nil, fmt.Errorf("error reading manifest.json file: %w", err) From 91e3bc55984bd1bf61d94d5bac243e978e248a8f Mon Sep 17 00:00:00 2001 From: Alison Burgess Date: Mon, 1 Apr 2024 13:33:02 -0700 Subject: [PATCH 08/50] remove blank lines --- pkg/agent/datamodel/types.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pkg/agent/datamodel/types.go b/pkg/agent/datamodel/types.go index fef58311823..9441108a239 100644 --- a/pkg/agent/datamodel/types.go +++ b/pkg/agent/datamodel/types.go @@ -1620,8 +1620,6 @@ type K8sComponents struct { // GetLatestSigImageConfigRequest describes the input for a GetLatestSigImageConfig HTTP request. // This is mostly a wrapper over existing types so RP doesn't have to manually construct JSON. -// - type GetLatestSigImageConfigRequest struct { SIGConfig SIGConfig Region string @@ -1629,8 +1627,6 @@ type GetLatestSigImageConfigRequest struct { } // NodeBootstrappingConfiguration represents configurations for node bootstrapping. -// - type NodeBootstrappingConfiguration struct { ContainerService *ContainerService CloudSpecConfig *AzureEnvironmentSpecConfig @@ -1693,8 +1689,6 @@ const ( ) // NodeBootstrapping represents the custom data, CSE, and OS image info needed for node bootstrapping. -// - type NodeBootstrapping struct { CustomData string CSE string From 315ee9c3374bb1921cfd44cd5a44014ba92bdf30 Mon Sep 17 00:00:00 2001 From: Alison Burgess Date: Mon, 1 Apr 2024 13:33:34 -0700 Subject: [PATCH 09/50] remove blank lines --- pkg/agent/datamodel/sig_config.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/agent/datamodel/sig_config.go b/pkg/agent/datamodel/sig_config.go index 14c1281b404..e6e55cbab13 100644 --- a/pkg/agent/datamodel/sig_config.go +++ b/pkg/agent/datamodel/sig_config.go @@ -269,8 +269,6 @@ func (d Distro) IsWindowsPIRDistro() bool { } // SigImageConfigTemplate represents the SIG image configuration template. -// - type SigImageConfigTemplate struct { ResourceGroup string Gallery string From f87e2d546fbba7d9128b68caee6d7dbd82fad21d Mon Sep 17 00:00:00 2001 From: Alison Burgess Date: Mon, 1 Apr 2024 16:05:21 -0700 Subject: [PATCH 10/50] change up testing --- apiserver/getcachedvhdimages.go | 9 ++------- pkg/agent/bakerapi.go | 19 +++---------------- pkg/agent/bakerapi_test.go | 2 +- pkg/agent/datamodel/helper.go | 11 +++++++++++ pkg/agent/datamodel/helper_test.go | 27 +++++++++++++++++++++++++++ pkg/agent/datamodel/sig_config.go | 30 ++++++++++++++++++++++++++++++ 6 files changed, 74 insertions(+), 24 deletions(-) diff --git a/apiserver/getcachedvhdimages.go b/apiserver/getcachedvhdimages.go index b124c211d05..a0b642ddddf 100644 --- a/apiserver/getcachedvhdimages.go +++ b/apiserver/getcachedvhdimages.go @@ -23,13 +23,8 @@ func (api *APIServer) GetCachedK8sVersions(w http.ResponseWriter, r *http.Reques return } - images, err := agentBaker.GetCachedK8sVersions() - if err != nil { - log.Println(err.Error()) - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - + images := agentBaker.GetCachedK8sVersions() + result, err := json.Marshal(images) if err != nil { log.Println(err.Error()) diff --git a/pkg/agent/bakerapi.go b/pkg/agent/bakerapi.go index 9c9f2c298b8..629a772370a 100644 --- a/pkg/agent/bakerapi.go +++ b/pkg/agent/bakerapi.go @@ -5,9 +5,7 @@ package agent import ( "context" - "encoding/json" "fmt" - "os" "github.com/Azure/agentbaker/pkg/agent/datamodel" ) @@ -17,7 +15,7 @@ type AgentBaker interface { GetNodeBootstrapping(ctx context.Context, config *datamodel.NodeBootstrappingConfiguration) (*datamodel.NodeBootstrapping, error) GetLatestSigImageConfig(sigConfig datamodel.SIGConfig, region string, distro datamodel.Distro) (*datamodel.SigImageConfig, error) GetDistroSigImageConfig(sigConfig datamodel.SIGConfig, region string) (map[datamodel.Distro]datamodel.SigImageConfig, error) - GetCachedK8sVersions() ([]string, error) + GetCachedK8sVersions() []string } func NewAgentBaker() (AgentBaker, error) { @@ -140,17 +138,6 @@ func (agentBaker *agentBakerImpl) GetDistroSigImageConfig( return allDistros, nil } -func (agentBaker *agentBakerImpl) GetCachedK8sVersions() ([]string, error) { - manifestFilePath := "../../parts/linux/cloud-init/artifacts/manifest.json" - data, err := os.ReadFile(manifestFilePath) - if err != nil { - return nil, fmt.Errorf("error reading manifest.json file: %w", err) - } - data = trimEOF(data) - - var manifest Manifest - if err = json.Unmarshal(data, &manifest); err != nil { - return nil, fmt.Errorf("error parsing JSON data: %w", err) - } - return manifest.Kubernetes.Versions, nil +func (agentBaker *agentBakerImpl) GetCachedK8sVersions() []string { + return datamodel.CachedK8sVersions } diff --git a/pkg/agent/bakerapi_test.go b/pkg/agent/bakerapi_test.go index ce1b3f168ee..fbbecc83a38 100644 --- a/pkg/agent/bakerapi_test.go +++ b/pkg/agent/bakerapi_test.go @@ -258,7 +258,7 @@ var _ = Describe("AgentBaker API implementation tests", func() { agentBaker, err := NewAgentBaker() Expect(err).NotTo(HaveOccurred()) - images, err := agentBaker.GetCachedK8sVersions() + images := agentBaker.GetCachedK8sVersions() Expect(err).NotTo(HaveOccurred()) cachedVersions := []string{"1.26.6", "1.26.10", "1.26.12", "1.27.3", "1.27.7", "1.27.9", "1.28.1", "1.28.3", "1.28.5", "1.29.0", "1.29.2"} diff --git a/pkg/agent/datamodel/helper.go b/pkg/agent/datamodel/helper.go index 80637169a58..19a833d6c17 100644 --- a/pkg/agent/datamodel/helper.go +++ b/pkg/agent/datamodel/helper.go @@ -98,3 +98,14 @@ func IndentString(original string, spaces int) string { } return out.String() } + +func trimEOF(data []byte) []byte { + eofIndex := bytes.LastIndex(data, []byte("#EOF")) + if eofIndex != -1 { // #EOF found + newlineIndex := bytes.LastIndex(data[:eofIndex], []byte("\n")) + if newlineIndex != -1 { + return data[:newlineIndex] + } + } + return data +} diff --git a/pkg/agent/datamodel/helper_test.go b/pkg/agent/datamodel/helper_test.go index c03c4b75287..9b0cdbfe735 100644 --- a/pkg/agent/datamodel/helper_test.go +++ b/pkg/agent/datamodel/helper_test.go @@ -346,3 +346,30 @@ func TestIndentString(t *testing.T) { }) } } + + +func TestTrimEOF(t *testing.T) { + tests := []struct { + name string + input []byte + expected string + }{ + { + name: "Should remove #EOF at the end of a file", + input: []byte(`"versions":["1.26.6","1.26.10","1.26.12","1.27.3","1.27.7","1.27.9","1.28.1","1.28.3","1.28.5","1.29.0","1.29.2"]},"_template":{"fileName":"","downloadLocation":"","downloadURL":"","versions":[]}}` + "\n#EOF"), + expected: `"versions":["1.26.6","1.26.10","1.26.12","1.27.3","1.27.7","1.27.9","1.28.1","1.28.3","1.28.5","1.29.0","1.29.2"]},"_template":{"fileName":"","downloadLocation":"","downloadURL":"","versions":[]}}`, + }, + } + + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + got := trimEOF(test.input) + diff := cmp.Diff(test.expected, string(got)) + if diff != "" { + t.Fatalf(diff) + } + }) + } +} diff --git a/pkg/agent/datamodel/sig_config.go b/pkg/agent/datamodel/sig_config.go index e6e55cbab13..f52d07d2f7a 100644 --- a/pkg/agent/datamodel/sig_config.go +++ b/pkg/agent/datamodel/sig_config.go @@ -4,6 +4,9 @@ import ( _ "embed" "encoding/json" "fmt" + "os" + "path" + "runtime" "strings" ) @@ -12,6 +15,12 @@ const ( AzurePublicCloudSigSubscription string = "109a5e88-712a-48ae-9078-9ca8b3c81345" // AKS VHD ) +func init() { + _, filename, _, _ := runtime.Caller(0) + manifestFilePath := "../../../parts/linux/cloud-init/artifacts/manifest.json" + getCachedK8sVersionFromManifest(path.Join(path.Dir(filename), manifestFilePath)) +} + // SIGAzureEnvironmentSpecConfig is the overall configuration differences in different cloud environments. /* TODO(tonyxu) merge this with AzureEnvironmentSpecConfig from aks-engine(pkg/api/azenvtypes.go) once it's moved into AKS RP. */ @@ -869,3 +878,24 @@ func withSubscription(subscriptionID string) SigImageConfigOpt { c.SubscriptionID = subscriptionID } } + +type Manifest struct { + Kubernetes struct { + Versions []string `json:"versions"` + } `json:"kubernetes"` +} + +var CachedK8sVersions []string + +func getCachedK8sVersionFromManifest(manifestFilePath string) { + data, err := os.ReadFile(manifestFilePath) + if err != nil { + panic(err) + } + data = trimEOF(data) + var manifest Manifest + if err = json.Unmarshal(data, &manifest); err != nil { + panic(err) + } + CachedK8sVersions = manifest.Kubernetes.Versions +} From 233a9378a61719a44e40fdd42826261362b80f9e Mon Sep 17 00:00:00 2001 From: Alison Burgess Date: Mon, 1 Apr 2024 16:09:25 -0700 Subject: [PATCH 11/50] Remove unused util --- pkg/agent/utils.go | 11 ----------- pkg/agent/utils_test.go | 12 +----------- 2 files changed, 1 insertion(+), 22 deletions(-) diff --git a/pkg/agent/utils.go b/pkg/agent/utils.go index 68cedfcf4b3..af54e3f5353 100644 --- a/pkg/agent/utils.go +++ b/pkg/agent/utils.go @@ -644,14 +644,3 @@ func addFeatureGateString(featureGates string, key string, value bool) string { } return strings.Join(pairs, ",") } - -func trimEOF(data []byte) []byte { - eofIndex := bytes.LastIndex(data, []byte("#EOF")) - if eofIndex != -1 { // #EOF found - newlineIndex := bytes.LastIndex(data[:eofIndex], []byte("\n")) - if newlineIndex != -1 { - return data[:newlineIndex] - } - } - return data -} diff --git a/pkg/agent/utils_test.go b/pkg/agent/utils_test.go index 137f12241f1..724e75a813f 100644 --- a/pkg/agent/utils_test.go +++ b/pkg/agent/utils_test.go @@ -726,14 +726,4 @@ var _ = Describe("Test removeComments", func() { Expect(string(result)).To(Equal(expected)) }) -}) - -//nolint:lll -var _ = Describe("trimEOF should remove the #EOF at the end of a file", func() { - It("Should remove #EOF at the end of a file", func() { - input := []byte(`"versions":["1.26.6","1.26.10","1.26.12","1.27.3","1.27.7","1.27.9","1.28.1","1.28.3","1.28.5","1.29.0","1.29.2"]},"_template":{"fileName":"","downloadLocation":"","downloadURL":"","versions":[]}}` + "\n#EOF") - expected := `"versions":["1.26.6","1.26.10","1.26.12","1.27.3","1.27.7","1.27.9","1.28.1","1.28.3","1.28.5","1.29.0","1.29.2"]},"_template":{"fileName":"","downloadLocation":"","downloadURL":"","versions":[]}}` - result := trimEOF(input) - Expect(string(result)).To(Equal(expected)) - }) -}) +}) \ No newline at end of file From 41ff623a92a1f32df908c153bfbed0aeec73d4b0 Mon Sep 17 00:00:00 2001 From: Alison Burgess Date: Mon, 1 Apr 2024 16:10:31 -0700 Subject: [PATCH 12/50] reorder --- pkg/agent/datamodel/sig_config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/agent/datamodel/sig_config.go b/pkg/agent/datamodel/sig_config.go index f52d07d2f7a..e394ed1684f 100644 --- a/pkg/agent/datamodel/sig_config.go +++ b/pkg/agent/datamodel/sig_config.go @@ -879,14 +879,14 @@ func withSubscription(subscriptionID string) SigImageConfigOpt { } } +var CachedK8sVersions []string + type Manifest struct { Kubernetes struct { Versions []string `json:"versions"` } `json:"kubernetes"` } -var CachedK8sVersions []string - func getCachedK8sVersionFromManifest(manifestFilePath string) { data, err := os.ReadFile(manifestFilePath) if err != nil { From dc029866a749b2f9bd422163f713796e32eeae7e Mon Sep 17 00:00:00 2001 From: Alison Burgess Date: Tue, 16 Apr 2024 17:14:43 -0700 Subject: [PATCH 13/50] rename functions --- apiserver/getcachedvhdimages.go | 14 +++++++------- apiserver/routers.go | 8 ++++---- pkg/agent/bakerapi.go | 4 ++-- pkg/agent/bakerapi_test.go | 4 ++-- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/apiserver/getcachedvhdimages.go b/apiserver/getcachedvhdimages.go index a0b642ddddf..d6169b5486d 100644 --- a/apiserver/getcachedvhdimages.go +++ b/apiserver/getcachedvhdimages.go @@ -10,12 +10,12 @@ import ( ) const ( - // RoutePathGetCachedK8sVersions the route path to get cached vhd images. - RoutePathGetCachedK8sVersions string = "/getcachedk8sversions" + // RoutePathGetCachedComponentVersions the route path to get cached vhd images. + RoutePathGetCachedComponentVersions string = "/getcachedcomponentversions" ) -// GetCachedK8sVersions endpoint for getting cached VHD images. -func (api *APIServer) GetCachedK8sVersions(w http.ResponseWriter, r *http.Request) { +// GetCachedComponentVersions endpoint for getting cached VHD images. +func (api *APIServer) GetCachedComponentVersions(w http.ResponseWriter, r *http.Request) { agentBaker, err := agent.NewAgentBaker() if err != nil { log.Println(err.Error()) @@ -23,9 +23,9 @@ func (api *APIServer) GetCachedK8sVersions(w http.ResponseWriter, r *http.Reques return } - images := agentBaker.GetCachedK8sVersions() - - result, err := json.Marshal(images) + cachedVersions := agentBaker.GetCachedComponentVersions() + + result, err := json.Marshal(cachedVersions) if err != nil { log.Println(err.Error()) http.Error(w, err.Error(), http.StatusBadRequest) diff --git a/apiserver/routers.go b/apiserver/routers.go index fef92f5459f..dadfc783705 100644 --- a/apiserver/routers.go +++ b/apiserver/routers.go @@ -41,10 +41,10 @@ func (api *APIServer) NewRouter() *mux.Router { HandlerFunc(api.GetDistroSigImageConfig) router. - Methods("POST"). - Path(RoutePathGetCachedK8sVersions). - Name("GetCachedK8sVersions"). - HandlerFunc(api.GetCachedK8sVersions) + Methods("GET"). + Path(RoutePathGetCachedComponentVersions). + Name("GetCachedComponentVersions"). + HandlerFunc(api.GetCachedComponentVersions) router.Methods("GET").Path("/healthz").Name("healthz").HandlerFunc(healthz) diff --git a/pkg/agent/bakerapi.go b/pkg/agent/bakerapi.go index 629a772370a..9af6819020a 100644 --- a/pkg/agent/bakerapi.go +++ b/pkg/agent/bakerapi.go @@ -15,7 +15,7 @@ type AgentBaker interface { GetNodeBootstrapping(ctx context.Context, config *datamodel.NodeBootstrappingConfiguration) (*datamodel.NodeBootstrapping, error) GetLatestSigImageConfig(sigConfig datamodel.SIGConfig, region string, distro datamodel.Distro) (*datamodel.SigImageConfig, error) GetDistroSigImageConfig(sigConfig datamodel.SIGConfig, region string) (map[datamodel.Distro]datamodel.SigImageConfig, error) - GetCachedK8sVersions() []string + GetCachedComponentVersions() []string } func NewAgentBaker() (AgentBaker, error) { @@ -138,6 +138,6 @@ func (agentBaker *agentBakerImpl) GetDistroSigImageConfig( return allDistros, nil } -func (agentBaker *agentBakerImpl) GetCachedK8sVersions() []string { +func (agentBaker *agentBakerImpl) GetCachedComponentVersions() []string { return datamodel.CachedK8sVersions } diff --git a/pkg/agent/bakerapi_test.go b/pkg/agent/bakerapi_test.go index fbbecc83a38..e09b229f468 100644 --- a/pkg/agent/bakerapi_test.go +++ b/pkg/agent/bakerapi_test.go @@ -253,12 +253,12 @@ var _ = Describe("AgentBaker API implementation tests", func() { }) }) - Context("GetCachedK8sVersions", func() { + Context("GetCachedComponentVersions", func() { It("should return the k8s version in manifest.json", func() { agentBaker, err := NewAgentBaker() Expect(err).NotTo(HaveOccurred()) - images := agentBaker.GetCachedK8sVersions() + images := agentBaker.GetCachedComponentVersions() Expect(err).NotTo(HaveOccurred()) cachedVersions := []string{"1.26.6", "1.26.10", "1.26.12", "1.27.3", "1.27.7", "1.27.9", "1.28.1", "1.28.3", "1.28.5", "1.29.0", "1.29.2"} From 10c7ee4cc808a38e81ef715332d02c86a2d4ecb5 Mon Sep 17 00:00:00 2001 From: Alison Burgess Date: Tue, 16 Apr 2024 17:32:50 -0700 Subject: [PATCH 14/50] tests not fully operational yet --- pkg/agent/bakerapi.go | 6 +- pkg/agent/bakerapi_test.go | 16 +++- pkg/agent/datamodel/helper.go | 10 +++ pkg/agent/datamodel/sig_config.go | 124 +++++++++++++++++++++++++++++- 4 files changed, 145 insertions(+), 11 deletions(-) diff --git a/pkg/agent/bakerapi.go b/pkg/agent/bakerapi.go index 9af6819020a..6541990188f 100644 --- a/pkg/agent/bakerapi.go +++ b/pkg/agent/bakerapi.go @@ -15,7 +15,7 @@ type AgentBaker interface { GetNodeBootstrapping(ctx context.Context, config *datamodel.NodeBootstrappingConfiguration) (*datamodel.NodeBootstrapping, error) GetLatestSigImageConfig(sigConfig datamodel.SIGConfig, region string, distro datamodel.Distro) (*datamodel.SigImageConfig, error) GetDistroSigImageConfig(sigConfig datamodel.SIGConfig, region string) (map[datamodel.Distro]datamodel.SigImageConfig, error) - GetCachedComponentVersions() []string + GetCachedComponentVersions() (map[string]datamodel.ProcessedManifest, map[string]datamodel.ProcessedComponents, error) } func NewAgentBaker() (AgentBaker, error) { @@ -138,6 +138,6 @@ func (agentBaker *agentBakerImpl) GetDistroSigImageConfig( return allDistros, nil } -func (agentBaker *agentBakerImpl) GetCachedComponentVersions() []string { - return datamodel.CachedK8sVersions +func (agentBaker *agentBakerImpl) GetCachedComponentVersions() (map[string]datamodel.ProcessedManifest, map[string]datamodel.ProcessedComponents, error) { + return datamodel.CachedFromManifest, datamodel.CachedFromComponents, nil } diff --git a/pkg/agent/bakerapi_test.go b/pkg/agent/bakerapi_test.go index e09b229f468..803b20991c3 100644 --- a/pkg/agent/bakerapi_test.go +++ b/pkg/agent/bakerapi_test.go @@ -17,6 +17,9 @@ var _ = Describe("AgentBaker API implementation tests", func() { ) BeforeEach(func() { + datamodel.CacheManifest() + datamodel.CacheComponents() + cs = &datamodel.ContainerService{ Location: "southcentralus", Type: "Microsoft.ContainerService/ManagedClusters", @@ -254,15 +257,20 @@ var _ = Describe("AgentBaker API implementation tests", func() { }) Context("GetCachedComponentVersions", func() { - It("should return the k8s version in manifest.json", func() { + It("should return cached VHD data", func() { agentBaker, err := NewAgentBaker() Expect(err).NotTo(HaveOccurred()) - images := agentBaker.GetCachedComponentVersions() + fromManifest, fromComponents, err := agentBaker.GetCachedComponentVersions() Expect(err).NotTo(HaveOccurred()) - cachedVersions := []string{"1.26.6", "1.26.10", "1.26.12", "1.27.3", "1.27.7", "1.27.9", "1.28.1", "1.28.3", "1.28.5", "1.29.0", "1.29.2"} - Expect(images).To(Equal(cachedVersions)) + Expect(fromManifest["runc"].Installed["default"]).To(Equal("1.1.12")) + Expect(fromManifest["containerd"].Pinned["1804"]).To(Equal("1.7.1-1")) + Expect(fromManifest["containerd"].Edge).To(Equal("1.7.14-1")) + Expect(fromManifest["kubernetes"].Versions[0]).To(Equal("1.26.6")) + Expect(fromComponents["pause"].MultiArchVersions[0]).To(Equal("3.6")) + Expect(fromComponents["azure-cns"].PrefetchOptimizations.Version).To(Equal("v1.5.23")) + Expect(fromComponents["azure-cns"].PrefetchOptimizations.Binaries[0]).To(Equal("usr/local/bin/azure-cns")) }) }) }) diff --git a/pkg/agent/datamodel/helper.go b/pkg/agent/datamodel/helper.go index 19a833d6c17..c2b22ab8165 100644 --- a/pkg/agent/datamodel/helper.go +++ b/pkg/agent/datamodel/helper.go @@ -109,3 +109,13 @@ func trimEOF(data []byte) []byte { } return data } + +func processDownloadURL(downloadURL string) string { + // example URL "downloadURL": "mcr.microsoft.com/oss/kubernetes/autoscaler/addon-resizer:*", + // getting the data between the last / and the last : + parts := strings.Split(downloadURL, "/") + lastPart := parts[len(parts)-1] + component := strings.Split(lastPart, ":") + componentName := component[0] + return componentName +} diff --git a/pkg/agent/datamodel/sig_config.go b/pkg/agent/datamodel/sig_config.go index e394ed1684f..9a72d602ca2 100644 --- a/pkg/agent/datamodel/sig_config.go +++ b/pkg/agent/datamodel/sig_config.go @@ -15,12 +15,30 @@ const ( AzurePublicCloudSigSubscription string = "109a5e88-712a-48ae-9078-9ca8b3c81345" // AKS VHD ) +//nolint:gochecknoglobals +var ( + CachedFromComponents = make(map[string]ProcessedComponents) + CachedFromManifest = make(map[string]ProcessedManifest) +) + +//nolint:gochecknoinits func init() { + CacheManifest() + CacheComponents() +} + +func CacheManifest() { _, filename, _, _ := runtime.Caller(0) manifestFilePath := "../../../parts/linux/cloud-init/artifacts/manifest.json" getCachedK8sVersionFromManifest(path.Join(path.Dir(filename), manifestFilePath)) } +func CacheComponents() { + _, filename, _, _ := runtime.Caller(0) + componentsFilePath := "../../../vhdbuilder/packer/components.json" + getCachedComponentsFromComponents(path.Join(path.Dir(filename), componentsFilePath)) +} + // SIGAzureEnvironmentSpecConfig is the overall configuration differences in different cloud environments. /* TODO(tonyxu) merge this with AzureEnvironmentSpecConfig from aks-engine(pkg/api/azenvtypes.go) once it's moved into AKS RP. */ @@ -288,7 +306,9 @@ type SigImageConfigTemplate struct { // SigImageConfig represents the SIG image configuration. type SigImageConfig struct { SigImageConfigTemplate - SubscriptionID string + SubscriptionID string + CachedFromManifest map[string]ProcessedManifest + CachedFromComponents map[string]ProcessedComponents } // WithOptions converts a SigImageConfigTemplate to SigImageConfig instance via function opts. @@ -879,14 +899,36 @@ func withSubscription(subscriptionID string) SigImageConfigOpt { } } -var CachedK8sVersions []string - type Manifest struct { + Containerd struct { + Edge string `json:"edge"` + Versions []string `json:"versions"` + Pinned map[string]string `json:"pinned"` + } `json:"containerd"` + Runc struct { + Versions []string `json:"versions"` + Pinned map[string]string `json:"pinned"` + Installed map[string]string `json:"installed"` + } `json:"runc"` + NvidiaContainerRuntime struct { + Versions []string `json:"versions"` + } `json:"nvidia-container-runtime"` + NvidiaDrivers struct { + Versions []string `json:"versions"` + } `json:"nvidia-drivers"` Kubernetes struct { Versions []string `json:"versions"` } `json:"kubernetes"` } +type ProcessedManifest struct { + Name string + Versions []string + Pinned map[string]string + Edge string + Installed map[string]string +} + func getCachedK8sVersionFromManifest(manifestFilePath string) { data, err := os.ReadFile(manifestFilePath) if err != nil { @@ -897,5 +939,79 @@ func getCachedK8sVersionFromManifest(manifestFilePath string) { if err = json.Unmarshal(data, &manifest); err != nil { panic(err) } - CachedK8sVersions = manifest.Kubernetes.Versions + + CachedFromManifest["kubernetes"] = ProcessedManifest{ + Name: "kubernetes", + Versions: manifest.Kubernetes.Versions, + } + CachedFromManifest["runc"] = ProcessedManifest{ + Name: "runc", + Versions: manifest.Runc.Versions, + Pinned: manifest.Runc.Pinned, + Installed: manifest.Runc.Installed, + } + CachedFromManifest["containerd"] = ProcessedManifest{ + Name: "containerd", + Versions: manifest.Containerd.Versions, + Pinned: manifest.Containerd.Pinned, + Edge: manifest.Containerd.Edge, + } + CachedFromManifest["nvidia-container-runtime"] = ProcessedManifest{ + Name: "nvidia-container-runtime", + Versions: manifest.NvidiaContainerRuntime.Versions, + } + CachedFromManifest["nvidia-drivers"] = ProcessedManifest{ + Name: "nvidia-drivers", + Versions: manifest.NvidiaDrivers.Versions, + } +} + +type Components struct { + ContainerImages []struct { + DownloadURL string `json:"downloadURL"` + Amd64OnlyVersions []string `json:"amd64OnlyVersions"` + MultiArchVersions []string `json:"multiArchVersions"` + PrefetchOptimizations []struct { + Version string `json:"version"` + Binaries []string `json:"binaries"` + } `json:"prefetchOptimizations"` + } `json:"ContainerImages"` +} +type PrefetchOptimizations struct { + Version string + Binaries []string +} + +type ProcessedComponents struct { + Name string + MultiArchVersions []string + Amd64OnlyVersions []string + PrefetchOptimizations PrefetchOptimizations +} + +func getCachedComponentsFromComponents(componentsFilePath string) { + data, err := os.ReadFile(componentsFilePath) + if err != nil { + panic(err) + } + var components Components + if err = json.Unmarshal(data, &components); err != nil { + panic(err) + } + + for _, image := range components.ContainerImages { + componentName := processDownloadURL(image.DownloadURL) + processed := ProcessedComponents{ + Name: componentName, + MultiArchVersions: image.MultiArchVersions, + Amd64OnlyVersions: image.Amd64OnlyVersions, + } + if len(image.PrefetchOptimizations) > 0 { + processed.PrefetchOptimizations = PrefetchOptimizations{ + Version: image.PrefetchOptimizations[0].Version, + Binaries: image.PrefetchOptimizations[0].Binaries, + } + } + CachedFromComponents[componentName] = processed + } } From 8fd9eb55dec0dacf856193a037733bd0dd40e001 Mon Sep 17 00:00:00 2001 From: Alison Burgess Date: Tue, 16 Apr 2024 17:41:36 -0700 Subject: [PATCH 15/50] unit tests --- pkg/agent/bakerapi_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/agent/bakerapi_test.go b/pkg/agent/bakerapi_test.go index 6c7eea7050d..0836c9304d4 100644 --- a/pkg/agent/bakerapi_test.go +++ b/pkg/agent/bakerapi_test.go @@ -537,8 +537,8 @@ var _ = Describe("AgentBaker API implementation tests", func() { Expect(fromManifest["runc"].Installed["default"]).To(Equal("1.1.12")) Expect(fromManifest["containerd"].Pinned["1804"]).To(Equal("1.7.1-1")) - Expect(fromManifest["containerd"].Edge).To(Equal("1.7.14-1")) - Expect(fromManifest["kubernetes"].Versions[0]).To(Equal("1.26.6")) + Expect(fromManifest["containerd"].Edge).To(Equal("1.7.15-1")) + Expect(fromManifest["kubernetes"].Versions[0]).To(Equal("1.27.7")) Expect(fromComponents["pause"].MultiArchVersions[0]).To(Equal("3.6")) Expect(fromComponents["azure-cns"].PrefetchOptimizations.Version).To(Equal("v1.5.23")) Expect(fromComponents["azure-cns"].PrefetchOptimizations.Binaries[0]).To(Equal("usr/local/bin/azure-cns")) From 45817c9eceba7fee73b424a09f7c8caa6057e76c Mon Sep 17 00:00:00 2001 From: Alison Burgess Date: Tue, 16 Apr 2024 17:45:35 -0700 Subject: [PATCH 16/50] fix api server side --- apiserver/getcachedvhdimages.go | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/apiserver/getcachedvhdimages.go b/apiserver/getcachedvhdimages.go index d6169b5486d..d5a91d5bde9 100644 --- a/apiserver/getcachedvhdimages.go +++ b/apiserver/getcachedvhdimages.go @@ -7,6 +7,7 @@ import ( "net/http" agent "github.com/Azure/agentbaker/pkg/agent" + "github.com/Azure/agentbaker/pkg/agent/datamodel" ) const ( @@ -14,6 +15,11 @@ const ( RoutePathGetCachedComponentVersions string = "/getcachedcomponentversions" ) +type CachedOnVHD struct { + CachedFromManifest map[string]datamodel.ProcessedManifest + CachedFromComponents map[string]datamodel.ProcessedComponents +} + // GetCachedComponentVersions endpoint for getting cached VHD images. func (api *APIServer) GetCachedComponentVersions(w http.ResponseWriter, r *http.Request) { agentBaker, err := agent.NewAgentBaker() @@ -23,14 +29,24 @@ func (api *APIServer) GetCachedComponentVersions(w http.ResponseWriter, r *http. return } - cachedVersions := agentBaker.GetCachedComponentVersions() + cachedFromManifest, cachedFromComponents, err := agentBaker.GetCachedComponentVersions() + if err != nil { + log.Println(err.Error()) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + result := CachedOnVHD{ + CachedFromManifest: cachedFromManifest, + CachedFromComponents: cachedFromComponents, + } - result, err := json.Marshal(cachedVersions) + jsonResponse, err := json.Marshal(result) if err != nil { log.Println(err.Error()) http.Error(w, err.Error(), http.StatusBadRequest) return } w.WriteHeader(http.StatusOK) - fmt.Fprint(w, string(result)) + fmt.Fprint(w, string(jsonResponse)) } From ccf9bf4aabc34e3b47ded7e2dbe25328d316da86 Mon Sep 17 00:00:00 2001 From: Alison Burgess Date: Tue, 16 Apr 2024 17:48:50 -0700 Subject: [PATCH 17/50] change name again --- apiserver/getcachedvhdimages.go | 10 +++++----- apiserver/routers.go | 6 +++--- pkg/agent/bakerapi.go | 4 ++-- pkg/agent/bakerapi_test.go | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/apiserver/getcachedvhdimages.go b/apiserver/getcachedvhdimages.go index d5a91d5bde9..5ed42bfee7b 100644 --- a/apiserver/getcachedvhdimages.go +++ b/apiserver/getcachedvhdimages.go @@ -11,8 +11,8 @@ import ( ) const ( - // RoutePathGetCachedComponentVersions the route path to get cached vhd images. - RoutePathGetCachedComponentVersions string = "/getcachedcomponentversions" + // RoutePathGetCachedVersionsOnVHD the route path to get cached vhd images. + RoutePathGetCachedVersionsOnVHD string = "/getcachedversionsonvhd" ) type CachedOnVHD struct { @@ -20,8 +20,8 @@ type CachedOnVHD struct { CachedFromComponents map[string]datamodel.ProcessedComponents } -// GetCachedComponentVersions endpoint for getting cached VHD images. -func (api *APIServer) GetCachedComponentVersions(w http.ResponseWriter, r *http.Request) { +// GetCachedVersionsOnVHD endpoint for getting cached VHD versions. +func (api *APIServer) GetCachedVersionsOnVHD(w http.ResponseWriter, r *http.Request) { agentBaker, err := agent.NewAgentBaker() if err != nil { log.Println(err.Error()) @@ -29,7 +29,7 @@ func (api *APIServer) GetCachedComponentVersions(w http.ResponseWriter, r *http. return } - cachedFromManifest, cachedFromComponents, err := agentBaker.GetCachedComponentVersions() + cachedFromManifest, cachedFromComponents, err := agentBaker.GetCachedVersionsOnVHD() if err != nil { log.Println(err.Error()) http.Error(w, err.Error(), http.StatusBadRequest) diff --git a/apiserver/routers.go b/apiserver/routers.go index dadfc783705..3510ef17315 100644 --- a/apiserver/routers.go +++ b/apiserver/routers.go @@ -42,9 +42,9 @@ func (api *APIServer) NewRouter() *mux.Router { router. Methods("GET"). - Path(RoutePathGetCachedComponentVersions). - Name("GetCachedComponentVersions"). - HandlerFunc(api.GetCachedComponentVersions) + Path(RoutePathGetCachedVersionsOnVHD). + Name("GetCachedVersionsOnVHD"). + HandlerFunc(api.GetCachedVersionsOnVHD) router.Methods("GET").Path("/healthz").Name("healthz").HandlerFunc(healthz) diff --git a/pkg/agent/bakerapi.go b/pkg/agent/bakerapi.go index b84a559e1fa..fc8f63a93e7 100644 --- a/pkg/agent/bakerapi.go +++ b/pkg/agent/bakerapi.go @@ -16,7 +16,7 @@ type AgentBaker interface { GetNodeBootstrapping(ctx context.Context, config *datamodel.NodeBootstrappingConfiguration) (*datamodel.NodeBootstrapping, error) GetLatestSigImageConfig(sigConfig datamodel.SIGConfig, distro datamodel.Distro, envInfo *datamodel.EnvironmentInfo) (*datamodel.SigImageConfig, error) GetDistroSigImageConfig(sigConfig datamodel.SIGConfig, envInfo *datamodel.EnvironmentInfo) (map[datamodel.Distro]datamodel.SigImageConfig, error) - GetCachedComponentVersions() (map[string]datamodel.ProcessedManifest, map[string]datamodel.ProcessedComponents, error) + GetCachedVersionsOnVHD() (map[string]datamodel.ProcessedManifest, map[string]datamodel.ProcessedComponents, error) } type agentBakerImpl struct { @@ -176,6 +176,6 @@ func findSIGImageConfig(sigConfig datamodel.SIGAzureEnvironmentSpecConfig, distr return nil } -func (agentBaker *agentBakerImpl) GetCachedComponentVersions() (map[string]datamodel.ProcessedManifest, map[string]datamodel.ProcessedComponents, error) { +func (agentBaker *agentBakerImpl) GetCachedVersionsOnVHD() (map[string]datamodel.ProcessedManifest, map[string]datamodel.ProcessedComponents, error) { return datamodel.CachedFromManifest, datamodel.CachedFromComponents, nil } diff --git a/pkg/agent/bakerapi_test.go b/pkg/agent/bakerapi_test.go index 0836c9304d4..8dfd1e26957 100644 --- a/pkg/agent/bakerapi_test.go +++ b/pkg/agent/bakerapi_test.go @@ -527,12 +527,12 @@ var _ = Describe("AgentBaker API implementation tests", func() { }) }) - Context("GetCachedComponentVersions", func() { + Context("GetCachedVersionsOnVHD", func() { It("should return cached VHD data", func() { agentBaker, err := NewAgentBaker() Expect(err).NotTo(HaveOccurred()) - fromManifest, fromComponents, err := agentBaker.GetCachedComponentVersions() + fromManifest, fromComponents, err := agentBaker.GetCachedVersionsOnVHD() Expect(err).NotTo(HaveOccurred()) Expect(fromManifest["runc"].Installed["default"]).To(Equal("1.1.12")) From 44ef921c20795d50a5cac8f2c3430bf043223b2f Mon Sep 17 00:00:00 2001 From: Alison Burgess Date: Tue, 16 Apr 2024 17:51:57 -0700 Subject: [PATCH 18/50] fix naming --- apiserver/getcachedvhdimages.go | 2 +- pkg/agent/datamodel/sig_config.go | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/apiserver/getcachedvhdimages.go b/apiserver/getcachedvhdimages.go index 5ed42bfee7b..63a54a5d6f9 100644 --- a/apiserver/getcachedvhdimages.go +++ b/apiserver/getcachedvhdimages.go @@ -20,7 +20,7 @@ type CachedOnVHD struct { CachedFromComponents map[string]datamodel.ProcessedComponents } -// GetCachedVersionsOnVHD endpoint for getting cached VHD versions. +// GetCachedVersionsOnVHD endpoint for getting the current versions of components cached on the vhd. func (api *APIServer) GetCachedVersionsOnVHD(w http.ResponseWriter, r *http.Request) { agentBaker, err := agent.NewAgentBaker() if err != nil { diff --git a/pkg/agent/datamodel/sig_config.go b/pkg/agent/datamodel/sig_config.go index 4c9a52ebb57..734b7fe9dca 100644 --- a/pkg/agent/datamodel/sig_config.go +++ b/pkg/agent/datamodel/sig_config.go @@ -18,7 +18,7 @@ const ( //nolint:gochecknoglobals var ( CachedFromComponents = make(map[string]ProcessedComponents) - CachedFromManifest = make(map[string]ProcessedManifest) + CachedFromManifest = make(map[string]ProcessedManifest) ) //nolint:gochecknoinits @@ -30,13 +30,13 @@ func init() { func CacheManifest() { _, filename, _, _ := runtime.Caller(0) manifestFilePath := "../../../parts/linux/cloud-init/artifacts/manifest.json" - getCachedK8sVersionFromManifest(path.Join(path.Dir(filename), manifestFilePath)) + getCachedVersionsFromManifestJSON(path.Join(path.Dir(filename), manifestFilePath)) } func CacheComponents() { _, filename, _, _ := runtime.Caller(0) componentsFilePath := "../../../vhdbuilder/packer/components.json" - getCachedComponentsFromComponents(path.Join(path.Dir(filename), componentsFilePath)) + getCachedVersionsFromComponentsJSON(path.Join(path.Dir(filename), componentsFilePath)) } // SIGAzureEnvironmentSpecConfig is the overall configuration differences in different cloud environments. @@ -315,8 +315,8 @@ type SigImageConfigTemplate struct { // SigImageConfig represents the SIG image configuration. type SigImageConfig struct { SigImageConfigTemplate - SubscriptionID string - CachedFromManifest map[string]ProcessedManifest + SubscriptionID string + CachedFromManifest map[string]ProcessedManifest CachedFromComponents map[string]ProcessedComponents } @@ -954,7 +954,7 @@ type ProcessedManifest struct { Installed map[string]string } -func getCachedK8sVersionFromManifest(manifestFilePath string) { +func getCachedVersionsFromManifestJSON(manifestFilePath string) { data, err := os.ReadFile(manifestFilePath) if err != nil { panic(err) @@ -1014,7 +1014,7 @@ type ProcessedComponents struct { PrefetchOptimizations PrefetchOptimizations } -func getCachedComponentsFromComponents(componentsFilePath string) { +func getCachedVersionsFromComponentsJSON(componentsFilePath string) { data, err := os.ReadFile(componentsFilePath) if err != nil { panic(err) From 3e0c5e4b13ac00ee3719a946673feb56eb8b36ef Mon Sep 17 00:00:00 2001 From: Alison Burgess Date: Tue, 16 Apr 2024 18:04:37 -0700 Subject: [PATCH 19/50] remove name component for easier readability --- apiserver/getcachedvhdimages.go | 4 ++-- pkg/agent/datamodel/sig_config.go | 8 -------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/apiserver/getcachedvhdimages.go b/apiserver/getcachedvhdimages.go index 63a54a5d6f9..20f2b0c28e1 100644 --- a/apiserver/getcachedvhdimages.go +++ b/apiserver/getcachedvhdimages.go @@ -16,8 +16,8 @@ const ( ) type CachedOnVHD struct { - CachedFromManifest map[string]datamodel.ProcessedManifest - CachedFromComponents map[string]datamodel.ProcessedComponents + CachedFromManifest map[string]datamodel.ProcessedManifest `json:"cached_from_manifest"` + CachedFromComponents map[string]datamodel.ProcessedComponents `json:"cached_from_components"` } // GetCachedVersionsOnVHD endpoint for getting the current versions of components cached on the vhd. diff --git a/pkg/agent/datamodel/sig_config.go b/pkg/agent/datamodel/sig_config.go index 734b7fe9dca..f7b6f48e342 100644 --- a/pkg/agent/datamodel/sig_config.go +++ b/pkg/agent/datamodel/sig_config.go @@ -947,7 +947,6 @@ type Manifest struct { } type ProcessedManifest struct { - Name string Versions []string Pinned map[string]string Edge string @@ -966,27 +965,22 @@ func getCachedVersionsFromManifestJSON(manifestFilePath string) { } CachedFromManifest["kubernetes"] = ProcessedManifest{ - Name: "kubernetes", Versions: manifest.Kubernetes.Versions, } CachedFromManifest["runc"] = ProcessedManifest{ - Name: "runc", Versions: manifest.Runc.Versions, Pinned: manifest.Runc.Pinned, Installed: manifest.Runc.Installed, } CachedFromManifest["containerd"] = ProcessedManifest{ - Name: "containerd", Versions: manifest.Containerd.Versions, Pinned: manifest.Containerd.Pinned, Edge: manifest.Containerd.Edge, } CachedFromManifest["nvidia-container-runtime"] = ProcessedManifest{ - Name: "nvidia-container-runtime", Versions: manifest.NvidiaContainerRuntime.Versions, } CachedFromManifest["nvidia-drivers"] = ProcessedManifest{ - Name: "nvidia-drivers", Versions: manifest.NvidiaDrivers.Versions, } } @@ -1008,7 +1002,6 @@ type PrefetchOptimizations struct { } type ProcessedComponents struct { - Name string MultiArchVersions []string Amd64OnlyVersions []string PrefetchOptimizations PrefetchOptimizations @@ -1027,7 +1020,6 @@ func getCachedVersionsFromComponentsJSON(componentsFilePath string) { for _, image := range components.ContainerImages { componentName := processDownloadURL(image.DownloadURL) processed := ProcessedComponents{ - Name: componentName, MultiArchVersions: image.MultiArchVersions, Amd64OnlyVersions: image.Amd64OnlyVersions, } From bfaf1b3edaa5f82d9e9127d4f3de28c2df2197fd Mon Sep 17 00:00:00 2001 From: Alison Burgess Date: Tue, 16 Apr 2024 18:15:53 -0700 Subject: [PATCH 20/50] fix linter --- pkg/agent/datamodel/helper_test.go | 5 ++--- pkg/agent/utils_test.go | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/pkg/agent/datamodel/helper_test.go b/pkg/agent/datamodel/helper_test.go index 9b0cdbfe735..b441d27fb1c 100644 --- a/pkg/agent/datamodel/helper_test.go +++ b/pkg/agent/datamodel/helper_test.go @@ -347,7 +347,6 @@ func TestIndentString(t *testing.T) { } } - func TestTrimEOF(t *testing.T) { tests := []struct { name string @@ -356,8 +355,8 @@ func TestTrimEOF(t *testing.T) { }{ { name: "Should remove #EOF at the end of a file", - input: []byte(`"versions":["1.26.6","1.26.10","1.26.12","1.27.3","1.27.7","1.27.9","1.28.1","1.28.3","1.28.5","1.29.0","1.29.2"]},"_template":{"fileName":"","downloadLocation":"","downloadURL":"","versions":[]}}` + "\n#EOF"), - expected: `"versions":["1.26.6","1.26.10","1.26.12","1.27.3","1.27.7","1.27.9","1.28.1","1.28.3","1.28.5","1.29.0","1.29.2"]},"_template":{"fileName":"","downloadLocation":"","downloadURL":"","versions":[]}}`, + input: []byte(`"versions":["1.26.6","1.26.10","1.26.12","1.27.3","1.27.7","1.27.9","1.28.1","1.28.3","1.28.5","1.29.0","1.29.2"]},"_template":{"fileName":"","downloadLocation":"","downloadURL":"","versions":[]}}` + "\n#EOF"), //nolint:lll + expected: `"versions":["1.26.6","1.26.10","1.26.12","1.27.3","1.27.7","1.27.9","1.28.1","1.28.3","1.28.5","1.29.0","1.29.2"]},"_template":{"fileName":"","downloadLocation":"","downloadURL":"","versions":[]}}`, //nolint:lll }, } diff --git a/pkg/agent/utils_test.go b/pkg/agent/utils_test.go index 724e75a813f..0739765450f 100644 --- a/pkg/agent/utils_test.go +++ b/pkg/agent/utils_test.go @@ -726,4 +726,4 @@ var _ = Describe("Test removeComments", func() { Expect(string(result)).To(Equal(expected)) }) -}) \ No newline at end of file +}) From 31fb1dfb3f61063b40b8a6e57c06f3d073587224 Mon Sep 17 00:00:00 2001 From: Alison Burgess Date: Wed, 17 Apr 2024 13:51:17 -0700 Subject: [PATCH 21/50] cleanup --- apiserver/getcachedvhdimages.go | 4 ++-- pkg/agent/bakerapi.go | 4 ++-- pkg/agent/datamodel/helper.go | 12 +++++++++--- pkg/agent/datamodel/sig_config.go | 13 ++++++++----- 4 files changed, 21 insertions(+), 12 deletions(-) diff --git a/apiserver/getcachedvhdimages.go b/apiserver/getcachedvhdimages.go index 20f2b0c28e1..d16c6ae6d89 100644 --- a/apiserver/getcachedvhdimages.go +++ b/apiserver/getcachedvhdimages.go @@ -16,8 +16,8 @@ const ( ) type CachedOnVHD struct { - CachedFromManifest map[string]datamodel.ProcessedManifest `json:"cached_from_manifest"` - CachedFromComponents map[string]datamodel.ProcessedComponents `json:"cached_from_components"` + CachedFromManifest map[string]datamodel.ProcessedManifest `json:"cached_from_manifest"` + CachedFromComponents map[string]datamodel.ContainerImage `json:"cached_from_components"` } // GetCachedVersionsOnVHD endpoint for getting the current versions of components cached on the vhd. diff --git a/pkg/agent/bakerapi.go b/pkg/agent/bakerapi.go index fc8f63a93e7..b1f32f14bd5 100644 --- a/pkg/agent/bakerapi.go +++ b/pkg/agent/bakerapi.go @@ -16,7 +16,7 @@ type AgentBaker interface { GetNodeBootstrapping(ctx context.Context, config *datamodel.NodeBootstrappingConfiguration) (*datamodel.NodeBootstrapping, error) GetLatestSigImageConfig(sigConfig datamodel.SIGConfig, distro datamodel.Distro, envInfo *datamodel.EnvironmentInfo) (*datamodel.SigImageConfig, error) GetDistroSigImageConfig(sigConfig datamodel.SIGConfig, envInfo *datamodel.EnvironmentInfo) (map[datamodel.Distro]datamodel.SigImageConfig, error) - GetCachedVersionsOnVHD() (map[string]datamodel.ProcessedManifest, map[string]datamodel.ProcessedComponents, error) + GetCachedVersionsOnVHD() (map[string]datamodel.ProcessedManifest, map[string]datamodel.ContainerImage, error) } type agentBakerImpl struct { @@ -176,6 +176,6 @@ func findSIGImageConfig(sigConfig datamodel.SIGAzureEnvironmentSpecConfig, distr return nil } -func (agentBaker *agentBakerImpl) GetCachedVersionsOnVHD() (map[string]datamodel.ProcessedManifest, map[string]datamodel.ProcessedComponents, error) { +func (agentBaker *agentBakerImpl) GetCachedVersionsOnVHD() (map[string]datamodel.ProcessedManifest, map[string]datamodel.ContainerImage, error) { return datamodel.CachedFromManifest, datamodel.CachedFromComponents, nil } diff --git a/pkg/agent/datamodel/helper.go b/pkg/agent/datamodel/helper.go index 818c73eedfb..d048afa78f9 100644 --- a/pkg/agent/datamodel/helper.go +++ b/pkg/agent/datamodel/helper.go @@ -105,12 +105,18 @@ func trimEOF(data []byte) []byte { return data } -func processDownloadURL(downloadURL string) string { +func processDownloadURL(downloadURL string) (string, error) { // example URL "downloadURL": "mcr.microsoft.com/oss/kubernetes/autoscaler/addon-resizer:*", // getting the data between the last / and the last : parts := strings.Split(downloadURL, "/") + if len(parts) == 0 || len(parts[len(parts)-1]) == 0 { + return "", errors.New("downloadURL is not in the expected format") + } lastPart := parts[len(parts)-1] + component := strings.Split(lastPart, ":") - componentName := component[0] - return componentName + if len(component) == 0 || len(component[0]) == 0 { + return "", errors.New("downloadURL is not in the expected format") + } + return component[0], nil } diff --git a/pkg/agent/datamodel/sig_config.go b/pkg/agent/datamodel/sig_config.go index f7b6f48e342..d85ba0e9e86 100644 --- a/pkg/agent/datamodel/sig_config.go +++ b/pkg/agent/datamodel/sig_config.go @@ -17,7 +17,7 @@ const ( //nolint:gochecknoglobals var ( - CachedFromComponents = make(map[string]ProcessedComponents) + CachedFromComponents = make(map[string]ContainerImage) CachedFromManifest = make(map[string]ProcessedManifest) ) @@ -317,7 +317,7 @@ type SigImageConfig struct { SigImageConfigTemplate SubscriptionID string CachedFromManifest map[string]ProcessedManifest - CachedFromComponents map[string]ProcessedComponents + CachedFromComponents map[string]ContainerImage } // WithOptions converts a SigImageConfigTemplate to SigImageConfig instance via function opts. @@ -1001,7 +1001,7 @@ type PrefetchOptimizations struct { Binaries []string } -type ProcessedComponents struct { +type ContainerImage struct { MultiArchVersions []string Amd64OnlyVersions []string PrefetchOptimizations PrefetchOptimizations @@ -1018,8 +1018,11 @@ func getCachedVersionsFromComponentsJSON(componentsFilePath string) { } for _, image := range components.ContainerImages { - componentName := processDownloadURL(image.DownloadURL) - processed := ProcessedComponents{ + componentName, err := processDownloadURL(image.DownloadURL) + if err != nil { + panic(err) + } + processed := ContainerImage{ MultiArchVersions: image.MultiArchVersions, Amd64OnlyVersions: image.Amd64OnlyVersions, } From aff31901a56a4366d355522a09fedb03db4b17d5 Mon Sep 17 00:00:00 2001 From: Alison Burgess Date: Wed, 17 Apr 2024 13:51:57 -0700 Subject: [PATCH 22/50] rename --- pkg/agent/datamodel/helper.go | 2 +- pkg/agent/datamodel/sig_config.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/agent/datamodel/helper.go b/pkg/agent/datamodel/helper.go index d048afa78f9..d7c8614e81f 100644 --- a/pkg/agent/datamodel/helper.go +++ b/pkg/agent/datamodel/helper.go @@ -105,7 +105,7 @@ func trimEOF(data []byte) []byte { return data } -func processDownloadURL(downloadURL string) (string, error) { +func processContainerImageTag(downloadURL string) (string, error) { // example URL "downloadURL": "mcr.microsoft.com/oss/kubernetes/autoscaler/addon-resizer:*", // getting the data between the last / and the last : parts := strings.Split(downloadURL, "/") diff --git a/pkg/agent/datamodel/sig_config.go b/pkg/agent/datamodel/sig_config.go index d85ba0e9e86..ca375866f79 100644 --- a/pkg/agent/datamodel/sig_config.go +++ b/pkg/agent/datamodel/sig_config.go @@ -1018,7 +1018,7 @@ func getCachedVersionsFromComponentsJSON(componentsFilePath string) { } for _, image := range components.ContainerImages { - componentName, err := processDownloadURL(image.DownloadURL) + componentName, err := processContainerImageTag(image.DownloadURL) if err != nil { panic(err) } From 0bb5cad750afc98bc46fdb9710a809c910ae81b3 Mon Sep 17 00:00:00 2001 From: Alison Burgess Date: Wed, 17 Apr 2024 13:54:53 -0700 Subject: [PATCH 23/50] change how to remove EOF --- pkg/agent/datamodel/helper.go | 11 ----------- pkg/agent/datamodel/helper_test.go | 26 -------------------------- pkg/agent/datamodel/sig_config.go | 3 ++- 3 files changed, 2 insertions(+), 38 deletions(-) diff --git a/pkg/agent/datamodel/helper.go b/pkg/agent/datamodel/helper.go index d7c8614e81f..2714611c801 100644 --- a/pkg/agent/datamodel/helper.go +++ b/pkg/agent/datamodel/helper.go @@ -94,17 +94,6 @@ func IndentString(original string, spaces int) string { return out.String() } -func trimEOF(data []byte) []byte { - eofIndex := bytes.LastIndex(data, []byte("#EOF")) - if eofIndex != -1 { // #EOF found - newlineIndex := bytes.LastIndex(data[:eofIndex], []byte("\n")) - if newlineIndex != -1 { - return data[:newlineIndex] - } - } - return data -} - func processContainerImageTag(downloadURL string) (string, error) { // example URL "downloadURL": "mcr.microsoft.com/oss/kubernetes/autoscaler/addon-resizer:*", // getting the data between the last / and the last : diff --git a/pkg/agent/datamodel/helper_test.go b/pkg/agent/datamodel/helper_test.go index b441d27fb1c..c03c4b75287 100644 --- a/pkg/agent/datamodel/helper_test.go +++ b/pkg/agent/datamodel/helper_test.go @@ -346,29 +346,3 @@ func TestIndentString(t *testing.T) { }) } } - -func TestTrimEOF(t *testing.T) { - tests := []struct { - name string - input []byte - expected string - }{ - { - name: "Should remove #EOF at the end of a file", - input: []byte(`"versions":["1.26.6","1.26.10","1.26.12","1.27.3","1.27.7","1.27.9","1.28.1","1.28.3","1.28.5","1.29.0","1.29.2"]},"_template":{"fileName":"","downloadLocation":"","downloadURL":"","versions":[]}}` + "\n#EOF"), //nolint:lll - expected: `"versions":["1.26.6","1.26.10","1.26.12","1.27.3","1.27.7","1.27.9","1.28.1","1.28.3","1.28.5","1.29.0","1.29.2"]},"_template":{"fileName":"","downloadLocation":"","downloadURL":"","versions":[]}}`, //nolint:lll - }, - } - - for _, test := range tests { - test := test - t.Run(test.name, func(t *testing.T) { - t.Parallel() - got := trimEOF(test.input) - diff := cmp.Diff(test.expected, string(got)) - if diff != "" { - t.Fatalf(diff) - } - }) - } -} diff --git a/pkg/agent/datamodel/sig_config.go b/pkg/agent/datamodel/sig_config.go index ca375866f79..d03c0f5f78f 100644 --- a/pkg/agent/datamodel/sig_config.go +++ b/pkg/agent/datamodel/sig_config.go @@ -1,6 +1,7 @@ package datamodel import ( + "bytes" _ "embed" "encoding/json" "fmt" @@ -958,7 +959,7 @@ func getCachedVersionsFromManifestJSON(manifestFilePath string) { if err != nil { panic(err) } - data = trimEOF(data) + data = bytes.ReplaceAll(data, []byte("#EOF"), []byte("")) var manifest Manifest if err = json.Unmarshal(data, &manifest); err != nil { panic(err) From f43afdefe1d4a22d325ad6a3c7d3763d59001a53 Mon Sep 17 00:00:00 2001 From: Alison Burgess Date: Wed, 17 Apr 2024 13:56:31 -0700 Subject: [PATCH 24/50] error check --- pkg/agent/bakerapi.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/agent/bakerapi.go b/pkg/agent/bakerapi.go index b1f32f14bd5..a42d46e8860 100644 --- a/pkg/agent/bakerapi.go +++ b/pkg/agent/bakerapi.go @@ -177,5 +177,8 @@ func findSIGImageConfig(sigConfig datamodel.SIGAzureEnvironmentSpecConfig, distr } func (agentBaker *agentBakerImpl) GetCachedVersionsOnVHD() (map[string]datamodel.ProcessedManifest, map[string]datamodel.ContainerImage, error) { + if datamodel.CachedFromManifest == nil || datamodel.CachedFromComponents == nil { + return nil, nil, fmt.Errorf("cached versions are not available") + } return datamodel.CachedFromManifest, datamodel.CachedFromComponents, nil } From 15712971936e305837f7da4e012f5a350270a9fc Mon Sep 17 00:00:00 2001 From: Alison Burgess Date: Wed, 17 Apr 2024 14:03:53 -0700 Subject: [PATCH 25/50] cleanup --- pkg/agent/datamodel/sig_config.go | 245 ++++++++++++++++-------------- 1 file changed, 127 insertions(+), 118 deletions(-) diff --git a/pkg/agent/datamodel/sig_config.go b/pkg/agent/datamodel/sig_config.go index d03c0f5f78f..2c1c473d823 100644 --- a/pkg/agent/datamodel/sig_config.go +++ b/pkg/agent/datamodel/sig_config.go @@ -24,20 +24,141 @@ var ( //nolint:gochecknoinits func init() { - CacheManifest() - CacheComponents() + manifest := CacheManifest() + processManifest(manifest) + + components := CacheComponents() + processComponents(components) } -func CacheManifest() { +func CacheManifest() Manifest { _, filename, _, _ := runtime.Caller(0) manifestFilePath := "../../../parts/linux/cloud-init/artifacts/manifest.json" - getCachedVersionsFromManifestJSON(path.Join(path.Dir(filename), manifestFilePath)) + return getCachedVersionsFromManifestJSON(path.Join(path.Dir(filename), manifestFilePath)) } -func CacheComponents() { +func CacheComponents() Components { _, filename, _, _ := runtime.Caller(0) componentsFilePath := "../../../vhdbuilder/packer/components.json" - getCachedVersionsFromComponentsJSON(path.Join(path.Dir(filename), componentsFilePath)) + return getCachedVersionsFromComponentsJSON(path.Join(path.Dir(filename), componentsFilePath)) +} + +func processManifest(manifest Manifest) { + CachedFromManifest["kubernetes"] = ProcessedManifest{ + Versions: manifest.Kubernetes.Versions, + } + CachedFromManifest["runc"] = ProcessedManifest{ + Versions: manifest.Runc.Versions, + Pinned: manifest.Runc.Pinned, + Installed: manifest.Runc.Installed, + } + CachedFromManifest["containerd"] = ProcessedManifest{ + Versions: manifest.Containerd.Versions, + Pinned: manifest.Containerd.Pinned, + Edge: manifest.Containerd.Edge, + } + CachedFromManifest["nvidia-container-runtime"] = ProcessedManifest{ + Versions: manifest.NvidiaContainerRuntime.Versions, + } + CachedFromManifest["nvidia-drivers"] = ProcessedManifest{ + Versions: manifest.NvidiaDrivers.Versions, + } +} + +func processComponents(components Components) { + for _, image := range components.ContainerImages { + componentName, err := processContainerImageTag(image.DownloadURL) + if err != nil { + panic(err) + } + processed := ContainerImage{ + MultiArchVersions: image.MultiArchVersions, + Amd64OnlyVersions: image.Amd64OnlyVersions, + } + if len(image.PrefetchOptimizations) > 0 { + processed.PrefetchOptimizations = PrefetchOptimizations{ + Version: image.PrefetchOptimizations[0].Version, + Binaries: image.PrefetchOptimizations[0].Binaries, + } + } + CachedFromComponents[componentName] = processed + } +} + +type Manifest struct { + Containerd struct { + Edge string `json:"edge"` + Versions []string `json:"versions"` + Pinned map[string]string `json:"pinned"` + } `json:"containerd"` + Runc struct { + Versions []string `json:"versions"` + Pinned map[string]string `json:"pinned"` + Installed map[string]string `json:"installed"` + } `json:"runc"` + NvidiaContainerRuntime struct { + Versions []string `json:"versions"` + } `json:"nvidia-container-runtime"` + NvidiaDrivers struct { + Versions []string `json:"versions"` + } `json:"nvidia-drivers"` + Kubernetes struct { + Versions []string `json:"versions"` + } `json:"kubernetes"` +} + +type ProcessedManifest struct { + Versions []string + Pinned map[string]string + Edge string + Installed map[string]string +} + +func getCachedVersionsFromManifestJSON(manifestFilePath string) Manifest { + data, err := os.ReadFile(manifestFilePath) + if err != nil { + panic(err) + } + data = bytes.ReplaceAll(data, []byte("#EOF"), []byte("")) + var manifest Manifest + if err = json.Unmarshal(data, &manifest); err != nil { + panic(err) + } + return manifest +} + +type Components struct { + ContainerImages []struct { + DownloadURL string `json:"downloadURL"` + Amd64OnlyVersions []string `json:"amd64OnlyVersions"` + MultiArchVersions []string `json:"multiArchVersions"` + PrefetchOptimizations []struct { + Version string `json:"version"` + Binaries []string `json:"binaries"` + } `json:"prefetchOptimizations"` + } `json:"ContainerImages"` +} +type PrefetchOptimizations struct { + Version string + Binaries []string +} + +type ContainerImage struct { + MultiArchVersions []string + Amd64OnlyVersions []string + PrefetchOptimizations PrefetchOptimizations +} + +func getCachedVersionsFromComponentsJSON(componentsFilePath string) Components { + data, err := os.ReadFile(componentsFilePath) + if err != nil { + panic(err) + } + var components Components + if err = json.Unmarshal(data, &components); err != nil { + panic(err) + } + return components } // SIGAzureEnvironmentSpecConfig is the overall configuration differences in different cloud environments. @@ -924,115 +1045,3 @@ func withSubscription(subscriptionID string) SigImageConfigOpt { c.SubscriptionID = subscriptionID } } - -type Manifest struct { - Containerd struct { - Edge string `json:"edge"` - Versions []string `json:"versions"` - Pinned map[string]string `json:"pinned"` - } `json:"containerd"` - Runc struct { - Versions []string `json:"versions"` - Pinned map[string]string `json:"pinned"` - Installed map[string]string `json:"installed"` - } `json:"runc"` - NvidiaContainerRuntime struct { - Versions []string `json:"versions"` - } `json:"nvidia-container-runtime"` - NvidiaDrivers struct { - Versions []string `json:"versions"` - } `json:"nvidia-drivers"` - Kubernetes struct { - Versions []string `json:"versions"` - } `json:"kubernetes"` -} - -type ProcessedManifest struct { - Versions []string - Pinned map[string]string - Edge string - Installed map[string]string -} - -func getCachedVersionsFromManifestJSON(manifestFilePath string) { - data, err := os.ReadFile(manifestFilePath) - if err != nil { - panic(err) - } - data = bytes.ReplaceAll(data, []byte("#EOF"), []byte("")) - var manifest Manifest - if err = json.Unmarshal(data, &manifest); err != nil { - panic(err) - } - - CachedFromManifest["kubernetes"] = ProcessedManifest{ - Versions: manifest.Kubernetes.Versions, - } - CachedFromManifest["runc"] = ProcessedManifest{ - Versions: manifest.Runc.Versions, - Pinned: manifest.Runc.Pinned, - Installed: manifest.Runc.Installed, - } - CachedFromManifest["containerd"] = ProcessedManifest{ - Versions: manifest.Containerd.Versions, - Pinned: manifest.Containerd.Pinned, - Edge: manifest.Containerd.Edge, - } - CachedFromManifest["nvidia-container-runtime"] = ProcessedManifest{ - Versions: manifest.NvidiaContainerRuntime.Versions, - } - CachedFromManifest["nvidia-drivers"] = ProcessedManifest{ - Versions: manifest.NvidiaDrivers.Versions, - } -} - -type Components struct { - ContainerImages []struct { - DownloadURL string `json:"downloadURL"` - Amd64OnlyVersions []string `json:"amd64OnlyVersions"` - MultiArchVersions []string `json:"multiArchVersions"` - PrefetchOptimizations []struct { - Version string `json:"version"` - Binaries []string `json:"binaries"` - } `json:"prefetchOptimizations"` - } `json:"ContainerImages"` -} -type PrefetchOptimizations struct { - Version string - Binaries []string -} - -type ContainerImage struct { - MultiArchVersions []string - Amd64OnlyVersions []string - PrefetchOptimizations PrefetchOptimizations -} - -func getCachedVersionsFromComponentsJSON(componentsFilePath string) { - data, err := os.ReadFile(componentsFilePath) - if err != nil { - panic(err) - } - var components Components - if err = json.Unmarshal(data, &components); err != nil { - panic(err) - } - - for _, image := range components.ContainerImages { - componentName, err := processContainerImageTag(image.DownloadURL) - if err != nil { - panic(err) - } - processed := ContainerImage{ - MultiArchVersions: image.MultiArchVersions, - Amd64OnlyVersions: image.Amd64OnlyVersions, - } - if len(image.PrefetchOptimizations) > 0 { - processed.PrefetchOptimizations = PrefetchOptimizations{ - Version: image.PrefetchOptimizations[0].Version, - Binaries: image.PrefetchOptimizations[0].Binaries, - } - } - CachedFromComponents[componentName] = processed - } -} From 3bd2a8169fb98f1ccef6c4f7059a7df013fdfc4e Mon Sep 17 00:00:00 2001 From: Alison Burgess Date: Wed, 17 Apr 2024 14:17:54 -0700 Subject: [PATCH 26/50] reorganize --- pkg/agent/datamodel/sig_config.go | 50 +++++++++++++++---------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/pkg/agent/datamodel/sig_config.go b/pkg/agent/datamodel/sig_config.go index 2c1c473d823..08f0d412254 100644 --- a/pkg/agent/datamodel/sig_config.go +++ b/pkg/agent/datamodel/sig_config.go @@ -85,6 +85,31 @@ func processComponents(components Components) { } } +func getCachedVersionsFromManifestJSON(manifestFilePath string) Manifest { + data, err := os.ReadFile(manifestFilePath) + if err != nil { + panic(err) + } + data = bytes.ReplaceAll(data, []byte("#EOF"), []byte("")) + var manifest Manifest + if err = json.Unmarshal(data, &manifest); err != nil { + panic(err) + } + return manifest +} + +func getCachedVersionsFromComponentsJSON(componentsFilePath string) Components { + data, err := os.ReadFile(componentsFilePath) + if err != nil { + panic(err) + } + var components Components + if err = json.Unmarshal(data, &components); err != nil { + panic(err) + } + return components +} + type Manifest struct { Containerd struct { Edge string `json:"edge"` @@ -114,19 +139,6 @@ type ProcessedManifest struct { Installed map[string]string } -func getCachedVersionsFromManifestJSON(manifestFilePath string) Manifest { - data, err := os.ReadFile(manifestFilePath) - if err != nil { - panic(err) - } - data = bytes.ReplaceAll(data, []byte("#EOF"), []byte("")) - var manifest Manifest - if err = json.Unmarshal(data, &manifest); err != nil { - panic(err) - } - return manifest -} - type Components struct { ContainerImages []struct { DownloadURL string `json:"downloadURL"` @@ -149,18 +161,6 @@ type ContainerImage struct { PrefetchOptimizations PrefetchOptimizations } -func getCachedVersionsFromComponentsJSON(componentsFilePath string) Components { - data, err := os.ReadFile(componentsFilePath) - if err != nil { - panic(err) - } - var components Components - if err = json.Unmarshal(data, &components); err != nil { - panic(err) - } - return components -} - // SIGAzureEnvironmentSpecConfig is the overall configuration differences in different cloud environments. /* TODO(tonyxu) merge this with AzureEnvironmentSpecConfig from aks-engine(pkg/api/azenvtypes.go) once it's moved into AKS RP. */ From 8610a2a8193440804410cf93ebf5ac6e721024f5 Mon Sep 17 00:00:00 2001 From: Alison Burgess Date: Wed, 17 Apr 2024 14:20:34 -0700 Subject: [PATCH 27/50] remove changes to SigImageConfig --- pkg/agent/datamodel/sig_config.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pkg/agent/datamodel/sig_config.go b/pkg/agent/datamodel/sig_config.go index 08f0d412254..aa5243c8568 100644 --- a/pkg/agent/datamodel/sig_config.go +++ b/pkg/agent/datamodel/sig_config.go @@ -437,9 +437,7 @@ type SigImageConfigTemplate struct { // SigImageConfig represents the SIG image configuration. type SigImageConfig struct { SigImageConfigTemplate - SubscriptionID string - CachedFromManifest map[string]ProcessedManifest - CachedFromComponents map[string]ContainerImage + SubscriptionID string } // WithOptions converts a SigImageConfigTemplate to SigImageConfig instance via function opts. From d9f18f3ce0527fe1bab07b67739cb714350d8e70 Mon Sep 17 00:00:00 2001 From: Alison Burgess Date: Fri, 19 Apr 2024 13:25:32 -0700 Subject: [PATCH 28/50] improve tests --- pkg/agent/bakerapi_test.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/pkg/agent/bakerapi_test.go b/pkg/agent/bakerapi_test.go index 8dfd1e26957..16678377b9a 100644 --- a/pkg/agent/bakerapi_test.go +++ b/pkg/agent/bakerapi_test.go @@ -535,12 +535,18 @@ var _ = Describe("AgentBaker API implementation tests", func() { fromManifest, fromComponents, err := agentBaker.GetCachedVersionsOnVHD() Expect(err).NotTo(HaveOccurred()) - Expect(fromManifest["runc"].Installed["default"]).To(Equal("1.1.12")) - Expect(fromManifest["containerd"].Pinned["1804"]).To(Equal("1.7.1-1")) - Expect(fromManifest["containerd"].Edge).To(Equal("1.7.15-1")) - Expect(fromManifest["kubernetes"].Versions[0]).To(Equal("1.27.7")) - Expect(fromComponents["pause"].MultiArchVersions[0]).To(Equal("3.6")) - Expect(fromComponents["azure-cns"].PrefetchOptimizations.Version).To(Equal("v1.5.23")) + manifest := datamodel.CacheManifest() + component := datamodel.CacheComponents() + + pause_index := 2 + azure_cns_index := 5 + + Expect(fromManifest["runc"].Installed["default"]).To(Equal(manifest.Runc.Installed["default"])) + Expect(fromManifest["containerd"].Pinned["1804"]).To(Equal(manifest.Containerd.Pinned["1804"])) + Expect(fromManifest["containerd"].Edge).To(Equal(manifest.Containerd.Edge)) + Expect(fromManifest["kubernetes"].Versions[0]).To(Equal(manifest.Kubernetes.Versions[0])) + Expect(fromComponents["pause"].MultiArchVersions[0]).To(Equal(component.ContainerImages[pause_index].MultiArchVersions[0])) + Expect(fromComponents["azure-cns"].PrefetchOptimizations.Version).To(Equal(component.ContainerImages[azure_cns_index].PrefetchOptimizations[0].Version)) Expect(fromComponents["azure-cns"].PrefetchOptimizations.Binaries[0]).To(Equal("usr/local/bin/azure-cns")) }) }) From 90c8966c13f4a6412710667e7afe5121e62f82ae Mon Sep 17 00:00:00 2001 From: Alison Burgess Date: Fri, 19 Apr 2024 13:26:58 -0700 Subject: [PATCH 29/50] fix variables for linter --- pkg/agent/bakerapi_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/agent/bakerapi_test.go b/pkg/agent/bakerapi_test.go index 16678377b9a..96ca4750ecd 100644 --- a/pkg/agent/bakerapi_test.go +++ b/pkg/agent/bakerapi_test.go @@ -538,15 +538,15 @@ var _ = Describe("AgentBaker API implementation tests", func() { manifest := datamodel.CacheManifest() component := datamodel.CacheComponents() - pause_index := 2 - azure_cns_index := 5 + pauseIndx := 2 + azureCNSIndx := 5 Expect(fromManifest["runc"].Installed["default"]).To(Equal(manifest.Runc.Installed["default"])) Expect(fromManifest["containerd"].Pinned["1804"]).To(Equal(manifest.Containerd.Pinned["1804"])) Expect(fromManifest["containerd"].Edge).To(Equal(manifest.Containerd.Edge)) Expect(fromManifest["kubernetes"].Versions[0]).To(Equal(manifest.Kubernetes.Versions[0])) - Expect(fromComponents["pause"].MultiArchVersions[0]).To(Equal(component.ContainerImages[pause_index].MultiArchVersions[0])) - Expect(fromComponents["azure-cns"].PrefetchOptimizations.Version).To(Equal(component.ContainerImages[azure_cns_index].PrefetchOptimizations[0].Version)) + Expect(fromComponents["pause"].MultiArchVersions[0]).To(Equal(component.ContainerImages[pauseIndx].MultiArchVersions[0])) + Expect(fromComponents["azure-cns"].PrefetchOptimizations.Version).To(Equal(component.ContainerImages[azureCNSIndx].PrefetchOptimizations[0].Version)) Expect(fromComponents["azure-cns"].PrefetchOptimizations.Binaries[0]).To(Equal("usr/local/bin/azure-cns")) }) }) From b1383dd7dbc4bdda3c015727210f2d32ec79758a Mon Sep 17 00:00:00 2001 From: Alison <32000834+AlisonB319@users.noreply.github.com> Date: Tue, 23 Apr 2024 09:28:59 -0700 Subject: [PATCH 30/50] Update pkg/agent/datamodel/sig_config.go Co-authored-by: Cameron Meissner --- pkg/agent/datamodel/sig_config.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/agent/datamodel/sig_config.go b/pkg/agent/datamodel/sig_config.go index aa5243c8568..5fcd2750ad7 100644 --- a/pkg/agent/datamodel/sig_config.go +++ b/pkg/agent/datamodel/sig_config.go @@ -150,6 +150,7 @@ type Components struct { } `json:"prefetchOptimizations"` } `json:"ContainerImages"` } + type PrefetchOptimizations struct { Version string Binaries []string From 5268f0c8c9f378e9a0fcc35d47dbab4da88e6ab3 Mon Sep 17 00:00:00 2001 From: Alison <32000834+AlisonB319@users.noreply.github.com> Date: Tue, 23 Apr 2024 09:29:27 -0700 Subject: [PATCH 31/50] Update pkg/agent/datamodel/helper.go Co-authored-by: Cameron Meissner --- pkg/agent/datamodel/helper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/agent/datamodel/helper.go b/pkg/agent/datamodel/helper.go index 2714611c801..3f91ba92df6 100644 --- a/pkg/agent/datamodel/helper.go +++ b/pkg/agent/datamodel/helper.go @@ -99,7 +99,7 @@ func processContainerImageTag(downloadURL string) (string, error) { // getting the data between the last / and the last : parts := strings.Split(downloadURL, "/") if len(parts) == 0 || len(parts[len(parts)-1]) == 0 { - return "", errors.New("downloadURL is not in the expected format") + return "", fmt.Errorf("container image component URL is not in the expected format: %s", downloadURL) } lastPart := parts[len(parts)-1] From fdc89e234a49637a52b69fad95a19db22d5e21c1 Mon Sep 17 00:00:00 2001 From: Alison Burgess Date: Tue, 23 Apr 2024 10:09:20 -0700 Subject: [PATCH 32/50] fix imports --- apiserver/getcachedvhdimages.go | 2 +- apiserver/getdistrosigimageconfig.go | 2 +- apiserver/getlatestsigimageconfig.go | 2 +- apiserver/getnodebootstrapdata.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apiserver/getcachedvhdimages.go b/apiserver/getcachedvhdimages.go index d16c6ae6d89..930c5e22127 100644 --- a/apiserver/getcachedvhdimages.go +++ b/apiserver/getcachedvhdimages.go @@ -6,7 +6,7 @@ import ( "log" "net/http" - agent "github.com/Azure/agentbaker/pkg/agent" + "github.com/Azure/agentbaker/pkg/agent" "github.com/Azure/agentbaker/pkg/agent/datamodel" ) diff --git a/apiserver/getdistrosigimageconfig.go b/apiserver/getdistrosigimageconfig.go index 5e9b7dbf567..bbd76ae7685 100644 --- a/apiserver/getdistrosigimageconfig.go +++ b/apiserver/getdistrosigimageconfig.go @@ -6,7 +6,7 @@ import ( "log" "net/http" - agent "github.com/Azure/agentbaker/pkg/agent" + "github.com/Azure/agentbaker/pkg/agent" "github.com/Azure/agentbaker/pkg/agent/datamodel" ) diff --git a/apiserver/getlatestsigimageconfig.go b/apiserver/getlatestsigimageconfig.go index 14c149999fa..eb71d541678 100644 --- a/apiserver/getlatestsigimageconfig.go +++ b/apiserver/getlatestsigimageconfig.go @@ -6,7 +6,7 @@ import ( "log" "net/http" - agent "github.com/Azure/agentbaker/pkg/agent" + "github.com/Azure/agentbaker/pkg/agent" "github.com/Azure/agentbaker/pkg/agent/datamodel" ) diff --git a/apiserver/getnodebootstrapdata.go b/apiserver/getnodebootstrapdata.go index 60b7586dcc4..345e82b7b41 100644 --- a/apiserver/getnodebootstrapdata.go +++ b/apiserver/getnodebootstrapdata.go @@ -8,7 +8,7 @@ import ( "net/http" "time" - agent "github.com/Azure/agentbaker/pkg/agent" + "github.com/Azure/agentbaker/pkg/agent" "github.com/Azure/agentbaker/pkg/agent/datamodel" ) From d6e8f312e5367e38d83db594c0afa1deb17e79ca Mon Sep 17 00:00:00 2001 From: Alison Burgess Date: Tue, 23 Apr 2024 10:09:51 -0700 Subject: [PATCH 33/50] cammel case for json key --- apiserver/getcachedvhdimages.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apiserver/getcachedvhdimages.go b/apiserver/getcachedvhdimages.go index 930c5e22127..ee3574178db 100644 --- a/apiserver/getcachedvhdimages.go +++ b/apiserver/getcachedvhdimages.go @@ -16,8 +16,8 @@ const ( ) type CachedOnVHD struct { - CachedFromManifest map[string]datamodel.ProcessedManifest `json:"cached_from_manifest"` - CachedFromComponents map[string]datamodel.ContainerImage `json:"cached_from_components"` + cachedFromManifest map[string]datamodel.ProcessedManifest `json:"cached_from_manifest"` + cachedFromComponents map[string]datamodel.ContainerImage `json:"cached_from_components"` } // GetCachedVersionsOnVHD endpoint for getting the current versions of components cached on the vhd. @@ -37,8 +37,8 @@ func (api *APIServer) GetCachedVersionsOnVHD(w http.ResponseWriter, r *http.Requ } result := CachedOnVHD{ - CachedFromManifest: cachedFromManifest, - CachedFromComponents: cachedFromComponents, + cachedFromManifest: cachedFromManifest, + cachedFromComponents: cachedFromComponents, } jsonResponse, err := json.Marshal(result) From 662a57cc4bc568e601414f1058b114721f0dc570 Mon Sep 17 00:00:00 2001 From: Alison Burgess Date: Tue, 23 Apr 2024 10:11:38 -0700 Subject: [PATCH 34/50] change function name --- pkg/agent/datamodel/helper.go | 2 +- pkg/agent/datamodel/sig_config.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/agent/datamodel/helper.go b/pkg/agent/datamodel/helper.go index 3f91ba92df6..6cb12d82adb 100644 --- a/pkg/agent/datamodel/helper.go +++ b/pkg/agent/datamodel/helper.go @@ -94,7 +94,7 @@ func IndentString(original string, spaces int) string { return out.String() } -func processContainerImageTag(downloadURL string) (string, error) { +func processContainerImage(downloadURL string) (string, error) { // example URL "downloadURL": "mcr.microsoft.com/oss/kubernetes/autoscaler/addon-resizer:*", // getting the data between the last / and the last : parts := strings.Split(downloadURL, "/") diff --git a/pkg/agent/datamodel/sig_config.go b/pkg/agent/datamodel/sig_config.go index 5fcd2750ad7..680efe00bdf 100644 --- a/pkg/agent/datamodel/sig_config.go +++ b/pkg/agent/datamodel/sig_config.go @@ -67,7 +67,7 @@ func processManifest(manifest Manifest) { func processComponents(components Components) { for _, image := range components.ContainerImages { - componentName, err := processContainerImageTag(image.DownloadURL) + componentName, err := processContainerImage(image.DownloadURL) if err != nil { panic(err) } From 61fd09ccd6fd3273cd450f551cf7753f439af4d3 Mon Sep 17 00:00:00 2001 From: Alison Burgess Date: Tue, 23 Apr 2024 10:13:53 -0700 Subject: [PATCH 35/50] trim suffix --- pkg/agent/datamodel/helper.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pkg/agent/datamodel/helper.go b/pkg/agent/datamodel/helper.go index 6cb12d82adb..f1f64328e18 100644 --- a/pkg/agent/datamodel/helper.go +++ b/pkg/agent/datamodel/helper.go @@ -102,10 +102,6 @@ func processContainerImage(downloadURL string) (string, error) { return "", fmt.Errorf("container image component URL is not in the expected format: %s", downloadURL) } lastPart := parts[len(parts)-1] - - component := strings.Split(lastPart, ":") - if len(component) == 0 || len(component[0]) == 0 { - return "", errors.New("downloadURL is not in the expected format") - } - return component[0], nil + component := strings.TrimSuffix(lastPart, ":*") + return component, nil } From 57b2b6b9cc277787273dfeac8829e4b66ed5898d Mon Sep 17 00:00:00 2001 From: Alison Burgess Date: Tue, 23 Apr 2024 10:16:55 -0700 Subject: [PATCH 36/50] needs to be cap to be exported --- apiserver/getcachedvhdimages.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apiserver/getcachedvhdimages.go b/apiserver/getcachedvhdimages.go index ee3574178db..930c5e22127 100644 --- a/apiserver/getcachedvhdimages.go +++ b/apiserver/getcachedvhdimages.go @@ -16,8 +16,8 @@ const ( ) type CachedOnVHD struct { - cachedFromManifest map[string]datamodel.ProcessedManifest `json:"cached_from_manifest"` - cachedFromComponents map[string]datamodel.ContainerImage `json:"cached_from_components"` + CachedFromManifest map[string]datamodel.ProcessedManifest `json:"cached_from_manifest"` + CachedFromComponents map[string]datamodel.ContainerImage `json:"cached_from_components"` } // GetCachedVersionsOnVHD endpoint for getting the current versions of components cached on the vhd. @@ -37,8 +37,8 @@ func (api *APIServer) GetCachedVersionsOnVHD(w http.ResponseWriter, r *http.Requ } result := CachedOnVHD{ - cachedFromManifest: cachedFromManifest, - cachedFromComponents: cachedFromComponents, + CachedFromManifest: cachedFromManifest, + CachedFromComponents: cachedFromComponents, } jsonResponse, err := json.Marshal(result) From 15801d48309ca049f70ef7647add3461bcbf2795 Mon Sep 17 00:00:00 2001 From: Alison Burgess Date: Tue, 23 Apr 2024 10:56:34 -0700 Subject: [PATCH 37/50] make components a little neater --- pkg/agent/bakerapi_test.go | 4 +- pkg/agent/datamodel/sig_config.go | 61 +++++++++++++++++-------------- 2 files changed, 36 insertions(+), 29 deletions(-) diff --git a/pkg/agent/bakerapi_test.go b/pkg/agent/bakerapi_test.go index 96ca4750ecd..2a79c6206ab 100644 --- a/pkg/agent/bakerapi_test.go +++ b/pkg/agent/bakerapi_test.go @@ -546,8 +546,8 @@ var _ = Describe("AgentBaker API implementation tests", func() { Expect(fromManifest["containerd"].Edge).To(Equal(manifest.Containerd.Edge)) Expect(fromManifest["kubernetes"].Versions[0]).To(Equal(manifest.Kubernetes.Versions[0])) Expect(fromComponents["pause"].MultiArchVersions[0]).To(Equal(component.ContainerImages[pauseIndx].MultiArchVersions[0])) - Expect(fromComponents["azure-cns"].PrefetchOptimizations.Version).To(Equal(component.ContainerImages[azureCNSIndx].PrefetchOptimizations[0].Version)) - Expect(fromComponents["azure-cns"].PrefetchOptimizations.Binaries[0]).To(Equal("usr/local/bin/azure-cns")) + Expect(fromComponents["azure-cns"].PrefetchOptimizations[0].Version).To(Equal(component.ContainerImages[azureCNSIndx].PrefetchOptimizations[0].Version)) + Expect(fromComponents["azure-cns"].PrefetchOptimizations[0].Binaries[0]).To(Equal("usr/local/bin/azure-cns")) }) }) }) diff --git a/pkg/agent/datamodel/sig_config.go b/pkg/agent/datamodel/sig_config.go index 680efe00bdf..c10d0cdf546 100644 --- a/pkg/agent/datamodel/sig_config.go +++ b/pkg/agent/datamodel/sig_config.go @@ -71,17 +71,7 @@ func processComponents(components Components) { if err != nil { panic(err) } - processed := ContainerImage{ - MultiArchVersions: image.MultiArchVersions, - Amd64OnlyVersions: image.Amd64OnlyVersions, - } - if len(image.PrefetchOptimizations) > 0 { - processed.PrefetchOptimizations = PrefetchOptimizations{ - Version: image.PrefetchOptimizations[0].Version, - Binaries: image.PrefetchOptimizations[0].Binaries, - } - } - CachedFromComponents[componentName] = processed + CachedFromComponents[componentName] = image } } @@ -139,27 +129,44 @@ type ProcessedManifest struct { Installed map[string]string } +/* + type Components struct { + ContainerImages []struct { + DownloadURL string `json:"downloadURL"` + Amd64OnlyVersions []string `json:"amd64OnlyVersions"` + MultiArchVersions []string `json:"multiArchVersions"` + PrefetchOptimizations []struct { + Version string `json:"version"` + Binaries []string `json:"binaries"` + } `json:"prefetchOptimizations"` + } `json:"ContainerImages"` + } + + type PrefetchOptimizations struct { + Version string + Binaries []string + } + + type ContainerImage struct { + MultiArchVersions []string + Amd64OnlyVersions []string + PrefetchOptimizations PrefetchOptimizations + } +*/ type Components struct { - ContainerImages []struct { - DownloadURL string `json:"downloadURL"` - Amd64OnlyVersions []string `json:"amd64OnlyVersions"` - MultiArchVersions []string `json:"multiArchVersions"` - PrefetchOptimizations []struct { - Version string `json:"version"` - Binaries []string `json:"binaries"` - } `json:"prefetchOptimizations"` - } `json:"ContainerImages"` + ContainerImages []ContainerImage `json:"containerImages"` } -type PrefetchOptimizations struct { - Version string - Binaries []string +type ContainerImage struct { + DownloadURL string `json:"downloadURL"` + MultiArchVersions []string `json:"multiArchVersions"` + Amd64OnlyVersions []string `json:"amd64OnlyVersions"` + PrefetchOptimizations []PrefetchOptimization `json:"prefetchOptimizations"` } -type ContainerImage struct { - MultiArchVersions []string - Amd64OnlyVersions []string - PrefetchOptimizations PrefetchOptimizations +type PrefetchOptimization struct { + Version string `json:"version"` + Binaries []string `json:"binaries"` } // SIGAzureEnvironmentSpecConfig is the overall configuration differences in different cloud environments. From 17f56cc7b8a0ce27b0626f0e04cf67e856ce982d Mon Sep 17 00:00:00 2001 From: Alison Burgess Date: Tue, 23 Apr 2024 13:03:42 -0700 Subject: [PATCH 38/50] Add downloaded files --- apiserver/getcachedvhdimages.go | 12 ++++---- pkg/agent/bakerapi.go | 11 ++++---- pkg/agent/bakerapi_test.go | 13 ++++++--- pkg/agent/datamodel/helper.go | 9 ++++++ pkg/agent/datamodel/sig_config.go | 46 +++++++++++++------------------ 5 files changed, 50 insertions(+), 41 deletions(-) diff --git a/apiserver/getcachedvhdimages.go b/apiserver/getcachedvhdimages.go index 930c5e22127..b57f7d078e6 100644 --- a/apiserver/getcachedvhdimages.go +++ b/apiserver/getcachedvhdimages.go @@ -16,8 +16,9 @@ const ( ) type CachedOnVHD struct { - CachedFromManifest map[string]datamodel.ProcessedManifest `json:"cached_from_manifest"` - CachedFromComponents map[string]datamodel.ContainerImage `json:"cached_from_components"` + CachedFromManifest map[string]datamodel.ProcessedManifest `json:"cached_from_manifest"` + CachedFromComponentContainerImages map[string]datamodel.ContainerImage `json:"cached_from_component_container_images"` + CachedFromComponentDownloadedFiles map[string]datamodel.DownloadFiles `json:"cached_from_component_downloaded_files"` } // GetCachedVersionsOnVHD endpoint for getting the current versions of components cached on the vhd. @@ -29,7 +30,7 @@ func (api *APIServer) GetCachedVersionsOnVHD(w http.ResponseWriter, r *http.Requ return } - cachedFromManifest, cachedFromComponents, err := agentBaker.GetCachedVersionsOnVHD() + cachedFromManifest, cachedFromComponentContainerImages, cachedFromComponentDownloadFiles, err := agentBaker.GetCachedVersionsOnVHD() if err != nil { log.Println(err.Error()) http.Error(w, err.Error(), http.StatusBadRequest) @@ -37,8 +38,9 @@ func (api *APIServer) GetCachedVersionsOnVHD(w http.ResponseWriter, r *http.Requ } result := CachedOnVHD{ - CachedFromManifest: cachedFromManifest, - CachedFromComponents: cachedFromComponents, + CachedFromManifest: cachedFromManifest, + CachedFromComponentContainerImages: cachedFromComponentContainerImages, + CachedFromComponentDownloadedFiles: cachedFromComponentDownloadFiles, } jsonResponse, err := json.Marshal(result) diff --git a/pkg/agent/bakerapi.go b/pkg/agent/bakerapi.go index a42d46e8860..4b1fad28f5c 100644 --- a/pkg/agent/bakerapi.go +++ b/pkg/agent/bakerapi.go @@ -16,7 +16,7 @@ type AgentBaker interface { GetNodeBootstrapping(ctx context.Context, config *datamodel.NodeBootstrappingConfiguration) (*datamodel.NodeBootstrapping, error) GetLatestSigImageConfig(sigConfig datamodel.SIGConfig, distro datamodel.Distro, envInfo *datamodel.EnvironmentInfo) (*datamodel.SigImageConfig, error) GetDistroSigImageConfig(sigConfig datamodel.SIGConfig, envInfo *datamodel.EnvironmentInfo) (map[datamodel.Distro]datamodel.SigImageConfig, error) - GetCachedVersionsOnVHD() (map[string]datamodel.ProcessedManifest, map[string]datamodel.ContainerImage, error) + GetCachedVersionsOnVHD() (map[string]datamodel.ProcessedManifest, map[string]datamodel.ContainerImage, map[string]datamodel.DownloadFiles, error) } type agentBakerImpl struct { @@ -176,9 +176,10 @@ func findSIGImageConfig(sigConfig datamodel.SIGAzureEnvironmentSpecConfig, distr return nil } -func (agentBaker *agentBakerImpl) GetCachedVersionsOnVHD() (map[string]datamodel.ProcessedManifest, map[string]datamodel.ContainerImage, error) { - if datamodel.CachedFromManifest == nil || datamodel.CachedFromComponents == nil { - return nil, nil, fmt.Errorf("cached versions are not available") +func (agentBaker *agentBakerImpl) GetCachedVersionsOnVHD() (map[string]datamodel.ProcessedManifest, map[string]datamodel.ContainerImage, + map[string]datamodel.DownloadFiles, error) { + if datamodel.CachedFromManifest == nil || datamodel.CachedFromComponentContainerImages == nil || datamodel.CachedFromComponentDownloadedFiles == nil { + return nil, nil, nil, fmt.Errorf("cached versions are not available") } - return datamodel.CachedFromManifest, datamodel.CachedFromComponents, nil + return datamodel.CachedFromManifest, datamodel.CachedFromComponentContainerImages, datamodel.CachedFromComponentDownloadedFiles, nil } diff --git a/pkg/agent/bakerapi_test.go b/pkg/agent/bakerapi_test.go index 2a79c6206ab..444e5a0d0ec 100644 --- a/pkg/agent/bakerapi_test.go +++ b/pkg/agent/bakerapi_test.go @@ -532,22 +532,27 @@ var _ = Describe("AgentBaker API implementation tests", func() { agentBaker, err := NewAgentBaker() Expect(err).NotTo(HaveOccurred()) - fromManifest, fromComponents, err := agentBaker.GetCachedVersionsOnVHD() + fromManifest, fromComponentsContainerImages, fromComponentsDownloadFiles, err := agentBaker.GetCachedVersionsOnVHD() Expect(err).NotTo(HaveOccurred()) manifest := datamodel.CacheManifest() component := datamodel.CacheComponents() + // The indices are hardcoded based on the current components.json. pauseIndx := 2 azureCNSIndx := 5 + cniPluginIndx := 0 + azureCNIIndx := 1 Expect(fromManifest["runc"].Installed["default"]).To(Equal(manifest.Runc.Installed["default"])) Expect(fromManifest["containerd"].Pinned["1804"]).To(Equal(manifest.Containerd.Pinned["1804"])) Expect(fromManifest["containerd"].Edge).To(Equal(manifest.Containerd.Edge)) Expect(fromManifest["kubernetes"].Versions[0]).To(Equal(manifest.Kubernetes.Versions[0])) - Expect(fromComponents["pause"].MultiArchVersions[0]).To(Equal(component.ContainerImages[pauseIndx].MultiArchVersions[0])) - Expect(fromComponents["azure-cns"].PrefetchOptimizations[0].Version).To(Equal(component.ContainerImages[azureCNSIndx].PrefetchOptimizations[0].Version)) - Expect(fromComponents["azure-cns"].PrefetchOptimizations[0].Binaries[0]).To(Equal("usr/local/bin/azure-cns")) + Expect(fromComponentsContainerImages["pause"].MultiArchVersions[0]).To(Equal(component.ContainerImages[pauseIndx].MultiArchVersions[0])) + Expect(fromComponentsContainerImages["azure-cns"].PrefetchOptimizations[0].Version).To(Equal(component.ContainerImages[azureCNSIndx].PrefetchOptimizations[0].Version)) + Expect(fromComponentsContainerImages["azure-cns"].PrefetchOptimizations[0].Binaries[0]).To(Equal("usr/local/bin/azure-cns")) + Expect(fromComponentsDownloadFiles["cni-plugins"].Versions[0]).To(Equal(component.DownloadFiles[cniPluginIndx].Versions[0])) + Expect(fromComponentsDownloadFiles["azure-cni"].Versions[1]).To(Equal(component.DownloadFiles[azureCNIIndx].Versions[1])) }) }) }) diff --git a/pkg/agent/datamodel/helper.go b/pkg/agent/datamodel/helper.go index f1f64328e18..b49899374cf 100644 --- a/pkg/agent/datamodel/helper.go +++ b/pkg/agent/datamodel/helper.go @@ -105,3 +105,12 @@ func processContainerImage(downloadURL string) (string, error) { component := strings.TrimSuffix(lastPart, ":*") return component, nil } + +func processDownloadFile(downloadURL string) (string, error) { + // example URL "downloadURL": "https://acs-mirror.azureedge.net/cni-plugins/v*/binaries", + parts := strings.Split(downloadURL, "/") + if len(parts) == 0 || len(parts) != 6 { + return "", fmt.Errorf("download file image URL is not in the expected format: %s", downloadURL) + } + return parts[3], nil +} diff --git a/pkg/agent/datamodel/sig_config.go b/pkg/agent/datamodel/sig_config.go index c10d0cdf546..798b515b152 100644 --- a/pkg/agent/datamodel/sig_config.go +++ b/pkg/agent/datamodel/sig_config.go @@ -18,8 +18,9 @@ const ( //nolint:gochecknoglobals var ( - CachedFromComponents = make(map[string]ContainerImage) - CachedFromManifest = make(map[string]ProcessedManifest) + CachedFromComponentContainerImages = make(map[string]ContainerImage) + CachedFromComponentDownloadedFiles = make(map[string]DownloadFiles) + CachedFromManifest = make(map[string]ProcessedManifest) ) //nolint:gochecknoinits @@ -71,7 +72,14 @@ func processComponents(components Components) { if err != nil { panic(err) } - CachedFromComponents[componentName] = image + CachedFromComponentContainerImages[componentName] = image + } + for _, file := range components.DownloadFiles { + componetName, err := processDownloadFile(file.DownloadURL) + if err != nil { + panic(err) + } + CachedFromComponentDownloadedFiles[componetName] = file } } @@ -129,32 +137,9 @@ type ProcessedManifest struct { Installed map[string]string } -/* - type Components struct { - ContainerImages []struct { - DownloadURL string `json:"downloadURL"` - Amd64OnlyVersions []string `json:"amd64OnlyVersions"` - MultiArchVersions []string `json:"multiArchVersions"` - PrefetchOptimizations []struct { - Version string `json:"version"` - Binaries []string `json:"binaries"` - } `json:"prefetchOptimizations"` - } `json:"ContainerImages"` - } - - type PrefetchOptimizations struct { - Version string - Binaries []string - } - - type ContainerImage struct { - MultiArchVersions []string - Amd64OnlyVersions []string - PrefetchOptimizations PrefetchOptimizations - } -*/ type Components struct { ContainerImages []ContainerImage `json:"containerImages"` + DownloadFiles []DownloadFiles `json:"downloadFiles"` } type ContainerImage struct { @@ -169,6 +154,13 @@ type PrefetchOptimization struct { Binaries []string `json:"binaries"` } +type DownloadFiles struct { + FileName string `json:"fileName"` + DownloadLocation string `json:"downloadLocation"` + DownloadURL string `json:"downloadURL"` + Versions []string `json:"versions"` +} + // SIGAzureEnvironmentSpecConfig is the overall configuration differences in different cloud environments. /* TODO(tonyxu) merge this with AzureEnvironmentSpecConfig from aks-engine(pkg/api/azenvtypes.go) once it's moved into AKS RP. */ From edde50d09351d8bc79855ab9657badf5a9ba7b2c Mon Sep 17 00:00:00 2001 From: Alison Burgess Date: Tue, 23 Apr 2024 16:27:01 -0700 Subject: [PATCH 39/50] feedback --- apiserver/getcachedvhdimages.go | 17 ++--------------- pkg/agent/bakerapi.go | 17 ++++++++++++----- pkg/agent/bakerapi_test.go | 24 +++++++++++------------- pkg/agent/datamodel/sig_config.go | 6 ++++++ 4 files changed, 31 insertions(+), 33 deletions(-) diff --git a/apiserver/getcachedvhdimages.go b/apiserver/getcachedvhdimages.go index b57f7d078e6..5eae7fcaa88 100644 --- a/apiserver/getcachedvhdimages.go +++ b/apiserver/getcachedvhdimages.go @@ -7,7 +7,6 @@ import ( "net/http" "github.com/Azure/agentbaker/pkg/agent" - "github.com/Azure/agentbaker/pkg/agent/datamodel" ) const ( @@ -15,12 +14,6 @@ const ( RoutePathGetCachedVersionsOnVHD string = "/getcachedversionsonvhd" ) -type CachedOnVHD struct { - CachedFromManifest map[string]datamodel.ProcessedManifest `json:"cached_from_manifest"` - CachedFromComponentContainerImages map[string]datamodel.ContainerImage `json:"cached_from_component_container_images"` - CachedFromComponentDownloadedFiles map[string]datamodel.DownloadFiles `json:"cached_from_component_downloaded_files"` -} - // GetCachedVersionsOnVHD endpoint for getting the current versions of components cached on the vhd. func (api *APIServer) GetCachedVersionsOnVHD(w http.ResponseWriter, r *http.Request) { agentBaker, err := agent.NewAgentBaker() @@ -30,20 +23,14 @@ func (api *APIServer) GetCachedVersionsOnVHD(w http.ResponseWriter, r *http.Requ return } - cachedFromManifest, cachedFromComponentContainerImages, cachedFromComponentDownloadFiles, err := agentBaker.GetCachedVersionsOnVHD() + cachedOnVHD, err := agentBaker.GetCachedVersionsOnVHD() if err != nil { log.Println(err.Error()) http.Error(w, err.Error(), http.StatusBadRequest) return } - result := CachedOnVHD{ - CachedFromManifest: cachedFromManifest, - CachedFromComponentContainerImages: cachedFromComponentContainerImages, - CachedFromComponentDownloadedFiles: cachedFromComponentDownloadFiles, - } - - jsonResponse, err := json.Marshal(result) + jsonResponse, err := json.Marshal(cachedOnVHD) if err != nil { log.Println(err.Error()) http.Error(w, err.Error(), http.StatusBadRequest) diff --git a/pkg/agent/bakerapi.go b/pkg/agent/bakerapi.go index 4b1fad28f5c..f0726407c92 100644 --- a/pkg/agent/bakerapi.go +++ b/pkg/agent/bakerapi.go @@ -16,7 +16,7 @@ type AgentBaker interface { GetNodeBootstrapping(ctx context.Context, config *datamodel.NodeBootstrappingConfiguration) (*datamodel.NodeBootstrapping, error) GetLatestSigImageConfig(sigConfig datamodel.SIGConfig, distro datamodel.Distro, envInfo *datamodel.EnvironmentInfo) (*datamodel.SigImageConfig, error) GetDistroSigImageConfig(sigConfig datamodel.SIGConfig, envInfo *datamodel.EnvironmentInfo) (map[datamodel.Distro]datamodel.SigImageConfig, error) - GetCachedVersionsOnVHD() (map[string]datamodel.ProcessedManifest, map[string]datamodel.ContainerImage, map[string]datamodel.DownloadFiles, error) + GetCachedVersionsOnVHD() (datamodel.CachedOnVHD, error) } type agentBakerImpl struct { @@ -176,10 +176,17 @@ func findSIGImageConfig(sigConfig datamodel.SIGAzureEnvironmentSpecConfig, distr return nil } -func (agentBaker *agentBakerImpl) GetCachedVersionsOnVHD() (map[string]datamodel.ProcessedManifest, map[string]datamodel.ContainerImage, - map[string]datamodel.DownloadFiles, error) { +func (agentBaker *agentBakerImpl) GetCachedVersionsOnVHD() (datamodel.CachedOnVHD, error) { + + cached := datamodel.CachedOnVHD{ + CachedFromManifest: datamodel.CachedFromManifest, + CachedFromComponentContainerImages: datamodel.CachedFromComponentContainerImages, + CachedFromComponentDownloadedFiles: datamodel.CachedFromComponentDownloadedFiles, + } + if datamodel.CachedFromManifest == nil || datamodel.CachedFromComponentContainerImages == nil || datamodel.CachedFromComponentDownloadedFiles == nil { - return nil, nil, nil, fmt.Errorf("cached versions are not available") + return cached, fmt.Errorf("cached versions are not available") } - return datamodel.CachedFromManifest, datamodel.CachedFromComponentContainerImages, datamodel.CachedFromComponentDownloadedFiles, nil + + return cached, nil } diff --git a/pkg/agent/bakerapi_test.go b/pkg/agent/bakerapi_test.go index 444e5a0d0ec..4e64e81b0da 100644 --- a/pkg/agent/bakerapi_test.go +++ b/pkg/agent/bakerapi_test.go @@ -21,9 +21,6 @@ var _ = Describe("AgentBaker API implementation tests", func() { BeforeEach(func() { toggles = agenttoggles.New() - datamodel.CacheManifest() - datamodel.CacheComponents() - cs = &datamodel.ContainerService{ Location: "southcentralus", Type: "Microsoft.ContainerService/ManagedClusters", @@ -532,7 +529,8 @@ var _ = Describe("AgentBaker API implementation tests", func() { agentBaker, err := NewAgentBaker() Expect(err).NotTo(HaveOccurred()) - fromManifest, fromComponentsContainerImages, fromComponentsDownloadFiles, err := agentBaker.GetCachedVersionsOnVHD() + // fromManifest, fromComponentsContainerImages, fromComponentsDownloadFiles, err := agentBaker.GetCachedVersionsOnVHD() + cachedOnVHD, err := agentBaker.GetCachedVersionsOnVHD() Expect(err).NotTo(HaveOccurred()) manifest := datamodel.CacheManifest() @@ -544,15 +542,15 @@ var _ = Describe("AgentBaker API implementation tests", func() { cniPluginIndx := 0 azureCNIIndx := 1 - Expect(fromManifest["runc"].Installed["default"]).To(Equal(manifest.Runc.Installed["default"])) - Expect(fromManifest["containerd"].Pinned["1804"]).To(Equal(manifest.Containerd.Pinned["1804"])) - Expect(fromManifest["containerd"].Edge).To(Equal(manifest.Containerd.Edge)) - Expect(fromManifest["kubernetes"].Versions[0]).To(Equal(manifest.Kubernetes.Versions[0])) - Expect(fromComponentsContainerImages["pause"].MultiArchVersions[0]).To(Equal(component.ContainerImages[pauseIndx].MultiArchVersions[0])) - Expect(fromComponentsContainerImages["azure-cns"].PrefetchOptimizations[0].Version).To(Equal(component.ContainerImages[azureCNSIndx].PrefetchOptimizations[0].Version)) - Expect(fromComponentsContainerImages["azure-cns"].PrefetchOptimizations[0].Binaries[0]).To(Equal("usr/local/bin/azure-cns")) - Expect(fromComponentsDownloadFiles["cni-plugins"].Versions[0]).To(Equal(component.DownloadFiles[cniPluginIndx].Versions[0])) - Expect(fromComponentsDownloadFiles["azure-cni"].Versions[1]).To(Equal(component.DownloadFiles[azureCNIIndx].Versions[1])) + Expect(cachedOnVHD.CachedFromManifest["runc"].Installed["default"]).To(Equal(manifest.Runc.Installed["default"])) + Expect(cachedOnVHD.CachedFromManifest["containerd"].Pinned["1804"]).To(Equal(manifest.Containerd.Pinned["1804"])) + Expect(cachedOnVHD.CachedFromManifest["containerd"].Edge).To(Equal(manifest.Containerd.Edge)) + Expect(cachedOnVHD.CachedFromManifest["kubernetes"].Versions[0]).To(Equal(manifest.Kubernetes.Versions[0])) + Expect(cachedOnVHD.CachedFromComponentContainerImages["pause"].MultiArchVersions[0]).To(Equal(component.ContainerImages[pauseIndx].MultiArchVersions[0])) + Expect(cachedOnVHD.CachedFromComponentContainerImages["azure-cns"].PrefetchOptimizations[0].Version).To(Equal(component.ContainerImages[azureCNSIndx].PrefetchOptimizations[0].Version)) + Expect(cachedOnVHD.CachedFromComponentContainerImages["azure-cns"].PrefetchOptimizations[0].Binaries[0]).To(Equal("usr/local/bin/azure-cns")) + Expect(cachedOnVHD.CachedFromComponentDownloadedFiles["cni-plugins"].Versions[0]).To(Equal(component.DownloadFiles[cniPluginIndx].Versions[0])) + Expect(cachedOnVHD.CachedFromComponentDownloadedFiles["azure-cni"].Versions[1]).To(Equal(component.DownloadFiles[azureCNIIndx].Versions[1])) }) }) }) diff --git a/pkg/agent/datamodel/sig_config.go b/pkg/agent/datamodel/sig_config.go index 798b515b152..26963de6a01 100644 --- a/pkg/agent/datamodel/sig_config.go +++ b/pkg/agent/datamodel/sig_config.go @@ -108,6 +108,12 @@ func getCachedVersionsFromComponentsJSON(componentsFilePath string) Components { return components } +type CachedOnVHD struct { + CachedFromManifest map[string]ProcessedManifest `json:"cached_from_manifest"` + CachedFromComponentContainerImages map[string]ContainerImage `json:"cached_from_component_container_images"` + CachedFromComponentDownloadedFiles map[string]DownloadFiles `json:"cached_from_component_downloaded_files"` +} + type Manifest struct { Containerd struct { Edge string `json:"edge"` From b0d0867d41095f16ca69003ca73335f2af47d15c Mon Sep 17 00:00:00 2001 From: Alison Burgess Date: Tue, 23 Apr 2024 16:29:26 -0700 Subject: [PATCH 40/50] linter --- pkg/agent/bakerapi.go | 1 - pkg/agent/bakerapi_test.go | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/agent/bakerapi.go b/pkg/agent/bakerapi.go index f0726407c92..4a7699e719d 100644 --- a/pkg/agent/bakerapi.go +++ b/pkg/agent/bakerapi.go @@ -177,7 +177,6 @@ func findSIGImageConfig(sigConfig datamodel.SIGAzureEnvironmentSpecConfig, distr } func (agentBaker *agentBakerImpl) GetCachedVersionsOnVHD() (datamodel.CachedOnVHD, error) { - cached := datamodel.CachedOnVHD{ CachedFromManifest: datamodel.CachedFromManifest, CachedFromComponentContainerImages: datamodel.CachedFromComponentContainerImages, diff --git a/pkg/agent/bakerapi_test.go b/pkg/agent/bakerapi_test.go index 4e64e81b0da..225994d64d1 100644 --- a/pkg/agent/bakerapi_test.go +++ b/pkg/agent/bakerapi_test.go @@ -547,7 +547,8 @@ var _ = Describe("AgentBaker API implementation tests", func() { Expect(cachedOnVHD.CachedFromManifest["containerd"].Edge).To(Equal(manifest.Containerd.Edge)) Expect(cachedOnVHD.CachedFromManifest["kubernetes"].Versions[0]).To(Equal(manifest.Kubernetes.Versions[0])) Expect(cachedOnVHD.CachedFromComponentContainerImages["pause"].MultiArchVersions[0]).To(Equal(component.ContainerImages[pauseIndx].MultiArchVersions[0])) - Expect(cachedOnVHD.CachedFromComponentContainerImages["azure-cns"].PrefetchOptimizations[0].Version).To(Equal(component.ContainerImages[azureCNSIndx].PrefetchOptimizations[0].Version)) + Expect(cachedOnVHD.CachedFromComponentContainerImages["azure-cns"].PrefetchOptimizations[0].Version).To( + Equal(component.ContainerImages[azureCNSIndx].PrefetchOptimizations[0].Version)) Expect(cachedOnVHD.CachedFromComponentContainerImages["azure-cns"].PrefetchOptimizations[0].Binaries[0]).To(Equal("usr/local/bin/azure-cns")) Expect(cachedOnVHD.CachedFromComponentDownloadedFiles["cni-plugins"].Versions[0]).To(Equal(component.DownloadFiles[cniPluginIndx].Versions[0])) Expect(cachedOnVHD.CachedFromComponentDownloadedFiles["azure-cni"].Versions[1]).To(Equal(component.DownloadFiles[azureCNIIndx].Versions[1])) From 95166acf46edf6e4f4093723b070ba06f51d41c1 Mon Sep 17 00:00:00 2001 From: Alison Burgess Date: Tue, 23 Apr 2024 17:09:49 -0700 Subject: [PATCH 41/50] Re-work manifest --- pkg/agent/bakerapi.go | 3 +- pkg/agent/bakerapi_test.go | 14 +++--- pkg/agent/datamodel/sig_config.go | 73 +++++++++++-------------------- 3 files changed, 35 insertions(+), 55 deletions(-) diff --git a/pkg/agent/bakerapi.go b/pkg/agent/bakerapi.go index 4a7699e719d..2c50a21cb7f 100644 --- a/pkg/agent/bakerapi.go +++ b/pkg/agent/bakerapi.go @@ -177,13 +177,14 @@ func findSIGImageConfig(sigConfig datamodel.SIGAzureEnvironmentSpecConfig, distr } func (agentBaker *agentBakerImpl) GetCachedVersionsOnVHD() (datamodel.CachedOnVHD, error) { + cached := datamodel.CachedOnVHD{ CachedFromManifest: datamodel.CachedFromManifest, CachedFromComponentContainerImages: datamodel.CachedFromComponentContainerImages, CachedFromComponentDownloadedFiles: datamodel.CachedFromComponentDownloadedFiles, } - if datamodel.CachedFromManifest == nil || datamodel.CachedFromComponentContainerImages == nil || datamodel.CachedFromComponentDownloadedFiles == nil { + if datamodel.CachedFromManifest == (&datamodel.Manifest{}) || datamodel.CachedFromComponentContainerImages == nil || datamodel.CachedFromComponentDownloadedFiles == nil { return cached, fmt.Errorf("cached versions are not available") } diff --git a/pkg/agent/bakerapi_test.go b/pkg/agent/bakerapi_test.go index 225994d64d1..6509eb55c6a 100644 --- a/pkg/agent/bakerapi_test.go +++ b/pkg/agent/bakerapi_test.go @@ -529,7 +529,6 @@ var _ = Describe("AgentBaker API implementation tests", func() { agentBaker, err := NewAgentBaker() Expect(err).NotTo(HaveOccurred()) - // fromManifest, fromComponentsContainerImages, fromComponentsDownloadFiles, err := agentBaker.GetCachedVersionsOnVHD() cachedOnVHD, err := agentBaker.GetCachedVersionsOnVHD() Expect(err).NotTo(HaveOccurred()) @@ -542,13 +541,14 @@ var _ = Describe("AgentBaker API implementation tests", func() { cniPluginIndx := 0 azureCNIIndx := 1 - Expect(cachedOnVHD.CachedFromManifest["runc"].Installed["default"]).To(Equal(manifest.Runc.Installed["default"])) - Expect(cachedOnVHD.CachedFromManifest["containerd"].Pinned["1804"]).To(Equal(manifest.Containerd.Pinned["1804"])) - Expect(cachedOnVHD.CachedFromManifest["containerd"].Edge).To(Equal(manifest.Containerd.Edge)) - Expect(cachedOnVHD.CachedFromManifest["kubernetes"].Versions[0]).To(Equal(manifest.Kubernetes.Versions[0])) + Expect(cachedOnVHD.CachedFromManifest.Runc.Installed["default"]).To(Equal(manifest.Runc.Installed["default"])) + Expect(cachedOnVHD.CachedFromManifest.Runc.Pinned["1804"]).To(Equal(manifest.Runc.Pinned["1804"])) + Expect(cachedOnVHD.CachedFromManifest.Runc.FileName).To(Equal(manifest.Runc.FileName)) + Expect(cachedOnVHD.CachedFromManifest.Containerd.Pinned["1804"]).To(Equal(manifest.Containerd.Pinned["1804"])) + Expect(cachedOnVHD.CachedFromManifest.Containerd.Edge).To(Equal(manifest.Containerd.Edge)) + Expect(cachedOnVHD.CachedFromManifest.Kubernetes.Versions[0]).To(Equal(manifest.Kubernetes.Versions[0])) Expect(cachedOnVHD.CachedFromComponentContainerImages["pause"].MultiArchVersions[0]).To(Equal(component.ContainerImages[pauseIndx].MultiArchVersions[0])) - Expect(cachedOnVHD.CachedFromComponentContainerImages["azure-cns"].PrefetchOptimizations[0].Version).To( - Equal(component.ContainerImages[azureCNSIndx].PrefetchOptimizations[0].Version)) + Expect(cachedOnVHD.CachedFromComponentContainerImages["azure-cns"].PrefetchOptimizations[0].Version).To(Equal(component.ContainerImages[azureCNSIndx].PrefetchOptimizations[0].Version)) Expect(cachedOnVHD.CachedFromComponentContainerImages["azure-cns"].PrefetchOptimizations[0].Binaries[0]).To(Equal("usr/local/bin/azure-cns")) Expect(cachedOnVHD.CachedFromComponentDownloadedFiles["cni-plugins"].Versions[0]).To(Equal(component.DownloadFiles[cniPluginIndx].Versions[0])) Expect(cachedOnVHD.CachedFromComponentDownloadedFiles["azure-cni"].Versions[1]).To(Equal(component.DownloadFiles[azureCNIIndx].Versions[1])) diff --git a/pkg/agent/datamodel/sig_config.go b/pkg/agent/datamodel/sig_config.go index 26963de6a01..6870665f4fc 100644 --- a/pkg/agent/datamodel/sig_config.go +++ b/pkg/agent/datamodel/sig_config.go @@ -20,7 +20,7 @@ const ( var ( CachedFromComponentContainerImages = make(map[string]ContainerImage) CachedFromComponentDownloadedFiles = make(map[string]DownloadFiles) - CachedFromManifest = make(map[string]ProcessedManifest) + CachedFromManifest = &Manifest{} ) //nolint:gochecknoinits @@ -45,25 +45,11 @@ func CacheComponents() Components { } func processManifest(manifest Manifest) { - CachedFromManifest["kubernetes"] = ProcessedManifest{ - Versions: manifest.Kubernetes.Versions, - } - CachedFromManifest["runc"] = ProcessedManifest{ - Versions: manifest.Runc.Versions, - Pinned: manifest.Runc.Pinned, - Installed: manifest.Runc.Installed, - } - CachedFromManifest["containerd"] = ProcessedManifest{ - Versions: manifest.Containerd.Versions, - Pinned: manifest.Containerd.Pinned, - Edge: manifest.Containerd.Edge, - } - CachedFromManifest["nvidia-container-runtime"] = ProcessedManifest{ - Versions: manifest.NvidiaContainerRuntime.Versions, - } - CachedFromManifest["nvidia-drivers"] = ProcessedManifest{ - Versions: manifest.NvidiaDrivers.Versions, - } + CachedFromManifest.Kubernetes = manifest.Kubernetes + CachedFromManifest.Runc = manifest.Runc + CachedFromManifest.Containerd = manifest.Containerd + CachedFromManifest.NvidiaContainerRuntime = manifest.NvidiaContainerRuntime + CachedFromManifest.NvidiaDrivers = manifest.NvidiaDrivers } func processComponents(components Components) { @@ -109,38 +95,31 @@ func getCachedVersionsFromComponentsJSON(componentsFilePath string) Components { } type CachedOnVHD struct { - CachedFromManifest map[string]ProcessedManifest `json:"cached_from_manifest"` - CachedFromComponentContainerImages map[string]ContainerImage `json:"cached_from_component_container_images"` - CachedFromComponentDownloadedFiles map[string]DownloadFiles `json:"cached_from_component_downloaded_files"` + CachedFromManifest *Manifest `json:"cached_from_manifest"` + CachedFromComponentContainerImages map[string]ContainerImage `json:"cached_from_component_container_images"` + CachedFromComponentDownloadedFiles map[string]DownloadFiles `json:"cached_from_component_downloaded_files"` +} + +type Dependency struct { + FileName string `json:"fileName"` + DownloadLocation string `json:"downloadLocation"` + DownloadURL string `json:"downloadURL"` + Versions []string `json:"versions"` + Installed map[string]string `json:"installed"` + Pinned map[string]string `json:"pinned"` + Edge string `json:"edge"` } type Manifest struct { - Containerd struct { - Edge string `json:"edge"` - Versions []string `json:"versions"` - Pinned map[string]string `json:"pinned"` - } `json:"containerd"` - Runc struct { - Versions []string `json:"versions"` - Pinned map[string]string `json:"pinned"` - Installed map[string]string `json:"installed"` - } `json:"runc"` - NvidiaContainerRuntime struct { - Versions []string `json:"versions"` - } `json:"nvidia-container-runtime"` - NvidiaDrivers struct { - Versions []string `json:"versions"` - } `json:"nvidia-drivers"` - Kubernetes struct { - Versions []string `json:"versions"` - } `json:"kubernetes"` + Containerd Dependency `json:"containerd"` + Runc Dependency `json:"runc"` + NvidiaContainerRuntime Dependency `json:"nvidia-container-runtime"` + NvidiaDrivers Dependency `json:"nvidia-drivers"` + Kubernetes Dependency `json:"kubernetes"` } -type ProcessedManifest struct { - Versions []string - Pinned map[string]string - Edge string - Installed map[string]string +type Versions struct { + Versions []string `json:"versions"` } type Components struct { From 7809e726235c22a0e52a21731879da5bfa5378b3 Mon Sep 17 00:00:00 2001 From: Alison Burgess Date: Tue, 23 Apr 2024 17:10:30 -0700 Subject: [PATCH 42/50] fix linter --- pkg/agent/bakerapi.go | 1 - pkg/agent/bakerapi_test.go | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/agent/bakerapi.go b/pkg/agent/bakerapi.go index 2c50a21cb7f..057002593e5 100644 --- a/pkg/agent/bakerapi.go +++ b/pkg/agent/bakerapi.go @@ -177,7 +177,6 @@ func findSIGImageConfig(sigConfig datamodel.SIGAzureEnvironmentSpecConfig, distr } func (agentBaker *agentBakerImpl) GetCachedVersionsOnVHD() (datamodel.CachedOnVHD, error) { - cached := datamodel.CachedOnVHD{ CachedFromManifest: datamodel.CachedFromManifest, CachedFromComponentContainerImages: datamodel.CachedFromComponentContainerImages, diff --git a/pkg/agent/bakerapi_test.go b/pkg/agent/bakerapi_test.go index 6509eb55c6a..6029267ef09 100644 --- a/pkg/agent/bakerapi_test.go +++ b/pkg/agent/bakerapi_test.go @@ -548,7 +548,8 @@ var _ = Describe("AgentBaker API implementation tests", func() { Expect(cachedOnVHD.CachedFromManifest.Containerd.Edge).To(Equal(manifest.Containerd.Edge)) Expect(cachedOnVHD.CachedFromManifest.Kubernetes.Versions[0]).To(Equal(manifest.Kubernetes.Versions[0])) Expect(cachedOnVHD.CachedFromComponentContainerImages["pause"].MultiArchVersions[0]).To(Equal(component.ContainerImages[pauseIndx].MultiArchVersions[0])) - Expect(cachedOnVHD.CachedFromComponentContainerImages["azure-cns"].PrefetchOptimizations[0].Version).To(Equal(component.ContainerImages[azureCNSIndx].PrefetchOptimizations[0].Version)) + Expect(cachedOnVHD.CachedFromComponentContainerImages["azure-cns"].PrefetchOptimizations[0].Version).To( + Equal(component.ContainerImages[azureCNSIndx].PrefetchOptimizations[0].Version)) Expect(cachedOnVHD.CachedFromComponentContainerImages["azure-cns"].PrefetchOptimizations[0].Binaries[0]).To(Equal("usr/local/bin/azure-cns")) Expect(cachedOnVHD.CachedFromComponentDownloadedFiles["cni-plugins"].Versions[0]).To(Equal(component.DownloadFiles[cniPluginIndx].Versions[0])) Expect(cachedOnVHD.CachedFromComponentDownloadedFiles["azure-cni"].Versions[1]).To(Equal(component.DownloadFiles[azureCNIIndx].Versions[1])) From 11304f16f3983cd9c6ddf2da00fa80a00ac80a80 Mon Sep 17 00:00:00 2001 From: Alison Burgess Date: Tue, 23 Apr 2024 17:14:16 -0700 Subject: [PATCH 43/50] clarification comment --- pkg/agent/datamodel/helper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/agent/datamodel/helper.go b/pkg/agent/datamodel/helper.go index b49899374cf..0efacd14f29 100644 --- a/pkg/agent/datamodel/helper.go +++ b/pkg/agent/datamodel/helper.go @@ -108,7 +108,7 @@ func processContainerImage(downloadURL string) (string, error) { func processDownloadFile(downloadURL string) (string, error) { // example URL "downloadURL": "https://acs-mirror.azureedge.net/cni-plugins/v*/binaries", - parts := strings.Split(downloadURL, "/") + parts := strings.Split(downloadURL, "/") // [https:, "", acs-mirror.azureedge.net, cni-plugins, v*, binaries] if len(parts) == 0 || len(parts) != 6 { return "", fmt.Errorf("download file image URL is not in the expected format: %s", downloadURL) } From d622b24126b556373bcec5ca7f89bc924d2a1430 Mon Sep 17 00:00:00 2001 From: Alison Burgess Date: Thu, 25 Apr 2024 11:16:21 -0700 Subject: [PATCH 44/50] continue slight reworks --- pkg/agent/bakerapi_test.go | 1 + pkg/agent/datamodel/helper.go | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/pkg/agent/bakerapi_test.go b/pkg/agent/bakerapi_test.go index 6029267ef09..d42046c9b87 100644 --- a/pkg/agent/bakerapi_test.go +++ b/pkg/agent/bakerapi_test.go @@ -536,6 +536,7 @@ var _ = Describe("AgentBaker API implementation tests", func() { component := datamodel.CacheComponents() // The indices are hardcoded based on the current components.json. + // Add new components to the bottom of components.json, or update the indices. pauseIndx := 2 azureCNSIndx := 5 cniPluginIndx := 0 diff --git a/pkg/agent/datamodel/helper.go b/pkg/agent/datamodel/helper.go index 0efacd14f29..2c3f4816bbb 100644 --- a/pkg/agent/datamodel/helper.go +++ b/pkg/agent/datamodel/helper.go @@ -7,6 +7,7 @@ import ( "bufio" "bytes" "fmt" + "net/url" "regexp" "sort" "strings" @@ -108,9 +109,10 @@ func processContainerImage(downloadURL string) (string, error) { func processDownloadFile(downloadURL string) (string, error) { // example URL "downloadURL": "https://acs-mirror.azureedge.net/cni-plugins/v*/binaries", - parts := strings.Split(downloadURL, "/") // [https:, "", acs-mirror.azureedge.net, cni-plugins, v*, binaries] - if len(parts) == 0 || len(parts) != 6 { + url, err := url.Parse(downloadURL) // /cni-plugins/v*/binaries + if err != nil { return "", fmt.Errorf("download file image URL is not in the expected format: %s", downloadURL) } - return parts[3], nil + urlSplit := strings.Split(url.Path, "/") // ["", cni-plugins, v*, binaries] + return urlSplit[1], nil } From 6e50921d8ddb3b140c9e63ddbfdab693ab01dfb8 Mon Sep 17 00:00:00 2001 From: Alison Burgess Date: Thu, 25 Apr 2024 12:05:52 -0700 Subject: [PATCH 45/50] pr comments --- pkg/agent/bakerapi_test.go | 5 +- pkg/agent/datamodel/helper.go | 13 +++-- pkg/agent/datamodel/sig_config.go | 79 +++++++------------------------ pkg/agent/datamodel/types.go | 57 ++++++++++++++++++++++ 4 files changed, 85 insertions(+), 69 deletions(-) diff --git a/pkg/agent/bakerapi_test.go b/pkg/agent/bakerapi_test.go index d42046c9b87..cf7b512dd63 100644 --- a/pkg/agent/bakerapi_test.go +++ b/pkg/agent/bakerapi_test.go @@ -532,8 +532,8 @@ var _ = Describe("AgentBaker API implementation tests", func() { cachedOnVHD, err := agentBaker.GetCachedVersionsOnVHD() Expect(err).NotTo(HaveOccurred()) - manifest := datamodel.CacheManifest() - component := datamodel.CacheComponents() + manifest := datamodel.GetManifest() + component := datamodel.GetComponents() // The indices are hardcoded based on the current components.json. // Add new components to the bottom of components.json, or update the indices. @@ -544,7 +544,6 @@ var _ = Describe("AgentBaker API implementation tests", func() { Expect(cachedOnVHD.CachedFromManifest.Runc.Installed["default"]).To(Equal(manifest.Runc.Installed["default"])) Expect(cachedOnVHD.CachedFromManifest.Runc.Pinned["1804"]).To(Equal(manifest.Runc.Pinned["1804"])) - Expect(cachedOnVHD.CachedFromManifest.Runc.FileName).To(Equal(manifest.Runc.FileName)) Expect(cachedOnVHD.CachedFromManifest.Containerd.Pinned["1804"]).To(Equal(manifest.Containerd.Pinned["1804"])) Expect(cachedOnVHD.CachedFromManifest.Containerd.Edge).To(Equal(manifest.Containerd.Edge)) Expect(cachedOnVHD.CachedFromManifest.Kubernetes.Versions[0]).To(Equal(manifest.Kubernetes.Versions[0])) diff --git a/pkg/agent/datamodel/helper.go b/pkg/agent/datamodel/helper.go index 2c3f4816bbb..b6277b5bf6e 100644 --- a/pkg/agent/datamodel/helper.go +++ b/pkg/agent/datamodel/helper.go @@ -95,7 +95,7 @@ func IndentString(original string, spaces int) string { return out.String() } -func processContainerImage(downloadURL string) (string, error) { +func getContainerImageNameFromURL(downloadURL string) (string, error) { // example URL "downloadURL": "mcr.microsoft.com/oss/kubernetes/autoscaler/addon-resizer:*", // getting the data between the last / and the last : parts := strings.Split(downloadURL, "/") @@ -107,12 +107,19 @@ func processContainerImage(downloadURL string) (string, error) { return component, nil } -func processDownloadFile(downloadURL string) (string, error) { +func getComponentNameFromURL(downloadURL string) (string, error) { // example URL "downloadURL": "https://acs-mirror.azureedge.net/cni-plugins/v*/binaries", url, err := url.Parse(downloadURL) // /cni-plugins/v*/binaries if err != nil { return "", fmt.Errorf("download file image URL is not in the expected format: %s", downloadURL) } urlSplit := strings.Split(url.Path, "/") // ["", cni-plugins, v*, binaries] - return urlSplit[1], nil + if len(urlSplit) < 2 { + return "", fmt.Errorf("download file image URL is not in the expected format: %s", downloadURL) + } + componentName := urlSplit[1] + if componentName == "" { + return "", fmt.Errorf("component name is empty in the URL: %s", downloadURL) + } + return componentName, nil } diff --git a/pkg/agent/datamodel/sig_config.go b/pkg/agent/datamodel/sig_config.go index 6870665f4fc..e868748aa94 100644 --- a/pkg/agent/datamodel/sig_config.go +++ b/pkg/agent/datamodel/sig_config.go @@ -19,26 +19,30 @@ const ( //nolint:gochecknoglobals var ( CachedFromComponentContainerImages = make(map[string]ContainerImage) - CachedFromComponentDownloadedFiles = make(map[string]DownloadFiles) + CachedFromComponentDownloadedFiles = make(map[string]DownloadFile) CachedFromManifest = &Manifest{} ) //nolint:gochecknoinits func init() { - manifest := CacheManifest() + manifest := GetManifest() processManifest(manifest) - components := CacheComponents() - processComponents(components) + components := GetComponents() + err := processComponents(components) + if err != nil { + panic(err) + } + } -func CacheManifest() Manifest { +func GetManifest() Manifest { _, filename, _, _ := runtime.Caller(0) manifestFilePath := "../../../parts/linux/cloud-init/artifacts/manifest.json" return getCachedVersionsFromManifestJSON(path.Join(path.Dir(filename), manifestFilePath)) } -func CacheComponents() Components { +func GetComponents() Components { _, filename, _, _ := runtime.Caller(0) componentsFilePath := "../../../vhdbuilder/packer/components.json" return getCachedVersionsFromComponentsJSON(path.Join(path.Dir(filename), componentsFilePath)) @@ -52,21 +56,22 @@ func processManifest(manifest Manifest) { CachedFromManifest.NvidiaDrivers = manifest.NvidiaDrivers } -func processComponents(components Components) { +func processComponents(components Components) error { for _, image := range components.ContainerImages { - componentName, err := processContainerImage(image.DownloadURL) + componentName, err := getContainerImageNameFromURL(image.DownloadURL) if err != nil { - panic(err) + return err } CachedFromComponentContainerImages[componentName] = image } for _, file := range components.DownloadFiles { - componetName, err := processDownloadFile(file.DownloadURL) + componetName, err := getComponentNameFromURL(file.DownloadURL) if err != nil { - panic(err) + return err } CachedFromComponentDownloadedFiles[componetName] = file } + return nil } func getCachedVersionsFromManifestJSON(manifestFilePath string) Manifest { @@ -94,58 +99,6 @@ func getCachedVersionsFromComponentsJSON(componentsFilePath string) Components { return components } -type CachedOnVHD struct { - CachedFromManifest *Manifest `json:"cached_from_manifest"` - CachedFromComponentContainerImages map[string]ContainerImage `json:"cached_from_component_container_images"` - CachedFromComponentDownloadedFiles map[string]DownloadFiles `json:"cached_from_component_downloaded_files"` -} - -type Dependency struct { - FileName string `json:"fileName"` - DownloadLocation string `json:"downloadLocation"` - DownloadURL string `json:"downloadURL"` - Versions []string `json:"versions"` - Installed map[string]string `json:"installed"` - Pinned map[string]string `json:"pinned"` - Edge string `json:"edge"` -} - -type Manifest struct { - Containerd Dependency `json:"containerd"` - Runc Dependency `json:"runc"` - NvidiaContainerRuntime Dependency `json:"nvidia-container-runtime"` - NvidiaDrivers Dependency `json:"nvidia-drivers"` - Kubernetes Dependency `json:"kubernetes"` -} - -type Versions struct { - Versions []string `json:"versions"` -} - -type Components struct { - ContainerImages []ContainerImage `json:"containerImages"` - DownloadFiles []DownloadFiles `json:"downloadFiles"` -} - -type ContainerImage struct { - DownloadURL string `json:"downloadURL"` - MultiArchVersions []string `json:"multiArchVersions"` - Amd64OnlyVersions []string `json:"amd64OnlyVersions"` - PrefetchOptimizations []PrefetchOptimization `json:"prefetchOptimizations"` -} - -type PrefetchOptimization struct { - Version string `json:"version"` - Binaries []string `json:"binaries"` -} - -type DownloadFiles struct { - FileName string `json:"fileName"` - DownloadLocation string `json:"downloadLocation"` - DownloadURL string `json:"downloadURL"` - Versions []string `json:"versions"` -} - // SIGAzureEnvironmentSpecConfig is the overall configuration differences in different cloud environments. /* TODO(tonyxu) merge this with AzureEnvironmentSpecConfig from aks-engine(pkg/api/azenvtypes.go) once it's moved into AKS RP. */ diff --git a/pkg/agent/datamodel/types.go b/pkg/agent/datamodel/types.go index 93db026b73a..77a851988b8 100644 --- a/pkg/agent/datamodel/types.go +++ b/pkg/agent/datamodel/types.go @@ -818,6 +818,63 @@ type ContainerService struct { Properties *Properties `json:"properties,omitempty"` } +// CachedOnVHD represents the cached components on VHD. +type CachedOnVHD struct { + CachedFromManifest *Manifest `json:"cachedFromManifest"` + CachedFromComponentContainerImages map[string]ContainerImage `json:"cachedFromComponentContainerImages"` + CachedFromComponentDownloadedFiles map[string]DownloadFile `json:"cachedFromComponentDownloadedFiles"` +} + +// Dependency represents fields that occur on manifest.json +type Dependency struct { + Versions []string `json:"versions"` + Installed map[string]string `json:"installed"` + Pinned map[string]string `json:"pinned"` + Edge string `json:"edge"` +} + +// Manifest represents the manifest.json file. +type Manifest struct { + Containerd Dependency `json:"containerd"` + Runc Dependency `json:"runc"` + NvidiaContainerRuntime Dependency `json:"nvidia-container-runtime"` + NvidiaDrivers Dependency `json:"nvidia-drivers"` + Kubernetes Dependency `json:"kubernetes"` +} + +// Versions of components on manifest.json +type Versions struct { + Versions []string `json:"versions"` +} + +// Components represents the components.json file. +type Components struct { + ContainerImages []ContainerImage `json:"containerImages"` + DownloadFiles []DownloadFile `json:"downloadFiles"` +} + +// ContainerImage represents fields that occur on components.json +type ContainerImage struct { + DownloadURL string `json:"downloadURL"` + MultiArchVersions []string `json:"multiArchVersions"` + Amd64OnlyVersions []string `json:"amd64OnlyVersions"` + PrefetchOptimizations []PrefetchOptimization `json:"prefetchOptimizations"` +} + +// PrefetchOptimization represents fields that occur on components.json +type PrefetchOptimization struct { + Version string `json:"version"` + Binaries []string `json:"binaries"` +} + +// DownloadFile represents DownloadFile fields that occur on components.json +type DownloadFile struct { + FileName string `json:"fileName"` + DownloadLocation string `json:"downloadLocation"` + DownloadURL string `json:"downloadURL"` + Versions []string `json:"versions"` +} + // IsAKSCustomCloud checks if it's in AKS custom cloud. func (cs *ContainerService) IsAKSCustomCloud() bool { return cs.Properties.CustomCloudEnv != nil && From c1be03d43ef7f3f1721903a3dd18118586befbf1 Mon Sep 17 00:00:00 2001 From: Alison Burgess Date: Thu, 25 Apr 2024 12:18:53 -0700 Subject: [PATCH 46/50] fix linter --- pkg/agent/bakerapi_test.go | 6 ++-- pkg/agent/datamodel/helper.go | 5 ++-- pkg/agent/datamodel/sig_config.go | 49 +++++++++++++++++++------------ pkg/agent/datamodel/types.go | 10 +++---- 4 files changed, 43 insertions(+), 27 deletions(-) diff --git a/pkg/agent/bakerapi_test.go b/pkg/agent/bakerapi_test.go index cf7b512dd63..c504dc6a866 100644 --- a/pkg/agent/bakerapi_test.go +++ b/pkg/agent/bakerapi_test.go @@ -532,8 +532,10 @@ var _ = Describe("AgentBaker API implementation tests", func() { cachedOnVHD, err := agentBaker.GetCachedVersionsOnVHD() Expect(err).NotTo(HaveOccurred()) - manifest := datamodel.GetManifest() - component := datamodel.GetComponents() + manifest, err := datamodel.GetManifest() + Expect(err).NotTo(HaveOccurred()) + component, err := datamodel.GetComponents() + Expect(err).NotTo(HaveOccurred()) // The indices are hardcoded based on the current components.json. // Add new components to the bottom of components.json, or update the indices. diff --git a/pkg/agent/datamodel/helper.go b/pkg/agent/datamodel/helper.go index b6277b5bf6e..1715e8cca28 100644 --- a/pkg/agent/datamodel/helper.go +++ b/pkg/agent/datamodel/helper.go @@ -114,10 +114,11 @@ func getComponentNameFromURL(downloadURL string) (string, error) { return "", fmt.Errorf("download file image URL is not in the expected format: %s", downloadURL) } urlSplit := strings.Split(url.Path, "/") // ["", cni-plugins, v*, binaries] - if len(urlSplit) < 2 { + componentIndx, minURLSplit := 1, 2 + if len(urlSplit) < minURLSplit { return "", fmt.Errorf("download file image URL is not in the expected format: %s", downloadURL) } - componentName := urlSplit[1] + componentName := urlSplit[componentIndx] if componentName == "" { return "", fmt.Errorf("component name is empty in the URL: %s", downloadURL) } diff --git a/pkg/agent/datamodel/sig_config.go b/pkg/agent/datamodel/sig_config.go index e868748aa94..e3fec92e7b4 100644 --- a/pkg/agent/datamodel/sig_config.go +++ b/pkg/agent/datamodel/sig_config.go @@ -25,27 +25,40 @@ var ( //nolint:gochecknoinits func init() { - manifest := GetManifest() + manifest, err := GetManifest() + if err != nil { + panic(err) + } processManifest(manifest) - components := GetComponents() - err := processComponents(components) + components, err := GetComponents() + if err != nil { + panic(err) + } + err = processComponents(components) if err != nil { panic(err) } - } -func GetManifest() Manifest { +func GetManifest() (Manifest, error) { _, filename, _, _ := runtime.Caller(0) manifestFilePath := "../../../parts/linux/cloud-init/artifacts/manifest.json" - return getCachedVersionsFromManifestJSON(path.Join(path.Dir(filename), manifestFilePath)) + manifest, err := getCachedVersionsFromManifestJSON(path.Join(path.Dir(filename), manifestFilePath)) + if err != nil { + return manifest, err + } + return manifest, nil } -func GetComponents() Components { +func GetComponents() (Components, error) { _, filename, _, _ := runtime.Caller(0) componentsFilePath := "../../../vhdbuilder/packer/components.json" - return getCachedVersionsFromComponentsJSON(path.Join(path.Dir(filename), componentsFilePath)) + components, err := getCachedVersionsFromComponentsJSON(path.Join(path.Dir(filename), componentsFilePath)) + if err != nil { + return components, err + } + return components, nil } func processManifest(manifest Manifest) { @@ -60,43 +73,43 @@ func processComponents(components Components) error { for _, image := range components.ContainerImages { componentName, err := getContainerImageNameFromURL(image.DownloadURL) if err != nil { - return err + return fmt.Errorf("error getting component name from URL: %s", err.Error()) } CachedFromComponentContainerImages[componentName] = image } for _, file := range components.DownloadFiles { componetName, err := getComponentNameFromURL(file.DownloadURL) if err != nil { - return err + return fmt.Errorf("error getting component name from URL: %s", err.Error()) } CachedFromComponentDownloadedFiles[componetName] = file } return nil } -func getCachedVersionsFromManifestJSON(manifestFilePath string) Manifest { +func getCachedVersionsFromManifestJSON(manifestFilePath string) (Manifest, error) { data, err := os.ReadFile(manifestFilePath) if err != nil { - panic(err) + return Manifest{}, fmt.Errorf("error reading manifest file: %s", err.Error()) } data = bytes.ReplaceAll(data, []byte("#EOF"), []byte("")) var manifest Manifest if err = json.Unmarshal(data, &manifest); err != nil { - panic(err) + return Manifest{}, fmt.Errorf("error unmarshalling manifest file: %s", err.Error()) } - return manifest + return manifest, nil } -func getCachedVersionsFromComponentsJSON(componentsFilePath string) Components { +func getCachedVersionsFromComponentsJSON(componentsFilePath string) (Components, error) { data, err := os.ReadFile(componentsFilePath) if err != nil { - panic(err) + return Components{}, fmt.Errorf("error reading components file: %s", err.Error()) } var components Components if err = json.Unmarshal(data, &components); err != nil { - panic(err) + return Components{}, fmt.Errorf("error unmarshalling components file: %s", err.Error()) } - return components + return components, nil } // SIGAzureEnvironmentSpecConfig is the overall configuration differences in different cloud environments. diff --git a/pkg/agent/datamodel/types.go b/pkg/agent/datamodel/types.go index 77a851988b8..577da2acf8c 100644 --- a/pkg/agent/datamodel/types.go +++ b/pkg/agent/datamodel/types.go @@ -825,7 +825,7 @@ type CachedOnVHD struct { CachedFromComponentDownloadedFiles map[string]DownloadFile `json:"cachedFromComponentDownloadedFiles"` } -// Dependency represents fields that occur on manifest.json +// Dependency represents fields that occur on manifest.json. type Dependency struct { Versions []string `json:"versions"` Installed map[string]string `json:"installed"` @@ -842,7 +842,7 @@ type Manifest struct { Kubernetes Dependency `json:"kubernetes"` } -// Versions of components on manifest.json +// Versions of components on manifest.json. type Versions struct { Versions []string `json:"versions"` } @@ -853,7 +853,7 @@ type Components struct { DownloadFiles []DownloadFile `json:"downloadFiles"` } -// ContainerImage represents fields that occur on components.json +// ContainerImage represents fields that occur on components.json. type ContainerImage struct { DownloadURL string `json:"downloadURL"` MultiArchVersions []string `json:"multiArchVersions"` @@ -861,13 +861,13 @@ type ContainerImage struct { PrefetchOptimizations []PrefetchOptimization `json:"prefetchOptimizations"` } -// PrefetchOptimization represents fields that occur on components.json +// PrefetchOptimization represents fields that occur on components.json. type PrefetchOptimization struct { Version string `json:"version"` Binaries []string `json:"binaries"` } -// DownloadFile represents DownloadFile fields that occur on components.json +// DownloadFile represents DownloadFile fields that occur on components.json. type DownloadFile struct { FileName string `json:"fileName"` DownloadLocation string `json:"downloadLocation"` From 1df434aaf4b75fd3d4ffe558cd554d588fc7ebc1 Mon Sep 17 00:00:00 2001 From: Alison Burgess Date: Thu, 25 Apr 2024 12:27:14 -0700 Subject: [PATCH 47/50] small refactor --- pkg/agent/bakerapi.go | 2 +- pkg/agent/datamodel/sig_config.go | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/pkg/agent/bakerapi.go b/pkg/agent/bakerapi.go index 057002593e5..4a7699e719d 100644 --- a/pkg/agent/bakerapi.go +++ b/pkg/agent/bakerapi.go @@ -183,7 +183,7 @@ func (agentBaker *agentBakerImpl) GetCachedVersionsOnVHD() (datamodel.CachedOnVH CachedFromComponentDownloadedFiles: datamodel.CachedFromComponentDownloadedFiles, } - if datamodel.CachedFromManifest == (&datamodel.Manifest{}) || datamodel.CachedFromComponentContainerImages == nil || datamodel.CachedFromComponentDownloadedFiles == nil { + if datamodel.CachedFromManifest == nil || datamodel.CachedFromComponentContainerImages == nil || datamodel.CachedFromComponentDownloadedFiles == nil { return cached, fmt.Errorf("cached versions are not available") } diff --git a/pkg/agent/datamodel/sig_config.go b/pkg/agent/datamodel/sig_config.go index e3fec92e7b4..dc25a7e4170 100644 --- a/pkg/agent/datamodel/sig_config.go +++ b/pkg/agent/datamodel/sig_config.go @@ -18,9 +18,9 @@ const ( //nolint:gochecknoglobals var ( - CachedFromComponentContainerImages = make(map[string]ContainerImage) - CachedFromComponentDownloadedFiles = make(map[string]DownloadFile) - CachedFromManifest = &Manifest{} + CachedFromComponentContainerImages map[string]ContainerImage + CachedFromComponentDownloadedFiles map[string]DownloadFile + CachedFromManifest *Manifest ) //nolint:gochecknoinits @@ -62,6 +62,8 @@ func GetComponents() (Components, error) { } func processManifest(manifest Manifest) { + CachedFromManifest = &Manifest{} + CachedFromManifest.Kubernetes = manifest.Kubernetes CachedFromManifest.Runc = manifest.Runc CachedFromManifest.Containerd = manifest.Containerd @@ -70,6 +72,9 @@ func processManifest(manifest Manifest) { } func processComponents(components Components) error { + CachedFromComponentContainerImages = make(map[string]ContainerImage) + CachedFromComponentDownloadedFiles = make(map[string]DownloadFile) + for _, image := range components.ContainerImages { componentName, err := getContainerImageNameFromURL(image.DownloadURL) if err != nil { From ca9e0f421c5610fe75b2a9db6a8f0ad945b12c50 Mon Sep 17 00:00:00 2001 From: Alison Burgess Date: Thu, 25 Apr 2024 12:29:11 -0700 Subject: [PATCH 48/50] help linter out --- pkg/agent/datamodel/sig_config.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/agent/datamodel/sig_config.go b/pkg/agent/datamodel/sig_config.go index dc25a7e4170..c9e88c44ecd 100644 --- a/pkg/agent/datamodel/sig_config.go +++ b/pkg/agent/datamodel/sig_config.go @@ -78,14 +78,14 @@ func processComponents(components Components) error { for _, image := range components.ContainerImages { componentName, err := getContainerImageNameFromURL(image.DownloadURL) if err != nil { - return fmt.Errorf("error getting component name from URL: %s", err.Error()) + return fmt.Errorf("error getting component name from URL: %w", err) } CachedFromComponentContainerImages[componentName] = image } for _, file := range components.DownloadFiles { componetName, err := getComponentNameFromURL(file.DownloadURL) if err != nil { - return fmt.Errorf("error getting component name from URL: %s", err.Error()) + return fmt.Errorf("error getting component name from URL: %w", err) } CachedFromComponentDownloadedFiles[componetName] = file } @@ -95,12 +95,12 @@ func processComponents(components Components) error { func getCachedVersionsFromManifestJSON(manifestFilePath string) (Manifest, error) { data, err := os.ReadFile(manifestFilePath) if err != nil { - return Manifest{}, fmt.Errorf("error reading manifest file: %s", err.Error()) + return Manifest{}, fmt.Errorf("error reading manifest file: %w", err) } data = bytes.ReplaceAll(data, []byte("#EOF"), []byte("")) var manifest Manifest if err = json.Unmarshal(data, &manifest); err != nil { - return Manifest{}, fmt.Errorf("error unmarshalling manifest file: %s", err.Error()) + return Manifest{}, fmt.Errorf("error unmarshalling manifest file: %w", err) } return manifest, nil } @@ -108,11 +108,11 @@ func getCachedVersionsFromManifestJSON(manifestFilePath string) (Manifest, error func getCachedVersionsFromComponentsJSON(componentsFilePath string) (Components, error) { data, err := os.ReadFile(componentsFilePath) if err != nil { - return Components{}, fmt.Errorf("error reading components file: %s", err.Error()) + return Components{}, fmt.Errorf("error reading components file: %w", err) } var components Components if err = json.Unmarshal(data, &components); err != nil { - return Components{}, fmt.Errorf("error unmarshalling components file: %s", err.Error()) + return Components{}, fmt.Errorf("error unmarshalling components file: %w", err) } return components, nil } From bc5d973bec348643347add8f81c37c374cf52e3e Mon Sep 17 00:00:00 2001 From: Alison Burgess Date: Thu, 25 Apr 2024 12:42:59 -0700 Subject: [PATCH 49/50] add unit tests for helper functions --- pkg/agent/datamodel/helper_test.go | 78 ++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/pkg/agent/datamodel/helper_test.go b/pkg/agent/datamodel/helper_test.go index c03c4b75287..92289395402 100644 --- a/pkg/agent/datamodel/helper_test.go +++ b/pkg/agent/datamodel/helper_test.go @@ -346,3 +346,81 @@ func TestIndentString(t *testing.T) { }) } } + +func TestGetContainerImageNameFromURL(t *testing.T) { + tests := []struct { + name string + downloadURL string + expected string + expectedErr string + }{ + { + name: "empty URL", + downloadURL: "", + expected: "", + expectedErr: "container image component URL is not in the expected format: ", + }, + { + name: "valid URL", + downloadURL: "mcr.microsoft.com/oss/kubernetes/autoscaler/addon-resizer:*", + expected: "addon-resizer", + expectedErr: "", + }, + } + + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + got, err := getContainerImageNameFromURL(test.downloadURL) + if err != nil { + if test.expectedErr != err.Error() { + t.Fatalf("expected error %s, instead got %s", test.expectedErr, err.Error()) + } + } else { + if test.expected != got { + t.Fatalf("expected %s, instead got %s", test.expected, got) + } + } + }) + } +} + +func TestGetComponentNameFromURL(t *testing.T) { + tests := []struct { + name string + downloadURL string + expected string + expectedErr string + }{ + { + name: "empty URL", + downloadURL: "", + expected: "", + expectedErr: "download file image URL is not in the expected format: ", + }, + { + name: "valid URL", + downloadURL: "https://acs-mirror.azureedge.net/cni-plugins/v*/binaries", + expected: "cni-plugins", + expectedErr: "", + }, + } + + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + got, err := getComponentNameFromURL(test.downloadURL) + if err != nil { + if test.expectedErr != err.Error() { + t.Fatalf("expected error %s, instead got %s", test.expectedErr, err.Error()) + } + } else { + if test.expected != got { + t.Fatalf("expected %s, instead got %s", test.expected, got) + } + } + }) + } +} From 2e13fb51c4ee7f91e9d2b4a01cd71ac9b181d369 Mon Sep 17 00:00:00 2001 From: Alison Burgess Date: Thu, 25 Apr 2024 12:47:21 -0700 Subject: [PATCH 50/50] have cached be returned as a pointer --- pkg/agent/bakerapi.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/agent/bakerapi.go b/pkg/agent/bakerapi.go index 4a7699e719d..e724406df14 100644 --- a/pkg/agent/bakerapi.go +++ b/pkg/agent/bakerapi.go @@ -16,7 +16,7 @@ type AgentBaker interface { GetNodeBootstrapping(ctx context.Context, config *datamodel.NodeBootstrappingConfiguration) (*datamodel.NodeBootstrapping, error) GetLatestSigImageConfig(sigConfig datamodel.SIGConfig, distro datamodel.Distro, envInfo *datamodel.EnvironmentInfo) (*datamodel.SigImageConfig, error) GetDistroSigImageConfig(sigConfig datamodel.SIGConfig, envInfo *datamodel.EnvironmentInfo) (map[datamodel.Distro]datamodel.SigImageConfig, error) - GetCachedVersionsOnVHD() (datamodel.CachedOnVHD, error) + GetCachedVersionsOnVHD() (*datamodel.CachedOnVHD, error) } type agentBakerImpl struct { @@ -176,7 +176,7 @@ func findSIGImageConfig(sigConfig datamodel.SIGAzureEnvironmentSpecConfig, distr return nil } -func (agentBaker *agentBakerImpl) GetCachedVersionsOnVHD() (datamodel.CachedOnVHD, error) { +func (agentBaker *agentBakerImpl) GetCachedVersionsOnVHD() (*datamodel.CachedOnVHD, error) { cached := datamodel.CachedOnVHD{ CachedFromManifest: datamodel.CachedFromManifest, CachedFromComponentContainerImages: datamodel.CachedFromComponentContainerImages, @@ -184,8 +184,8 @@ func (agentBaker *agentBakerImpl) GetCachedVersionsOnVHD() (datamodel.CachedOnVH } if datamodel.CachedFromManifest == nil || datamodel.CachedFromComponentContainerImages == nil || datamodel.CachedFromComponentDownloadedFiles == nil { - return cached, fmt.Errorf("cached versions are not available") + return &cached, fmt.Errorf("cached versions are not available") } - return cached, nil + return &cached, nil }