configurable cron schedule, local registry auth and fixing memory leaks

container-registry · Nov 30, 2024 · 20578ff · 20578ff
1 parent a6e0e10
commit 20578ff
Show file tree

Hide file tree

Showing 25 changed files with 552 additions and 1,021 deletions.
diff --git a/cmd/container_runtime/containerd.go b/cmd/container_runtime/containerd.go
@@ -1,7 +1,6 @@
 package runtime
 
 import (
-	"context"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -24,11 +23,6 @@ const (
 	DefaultConfigVersion        = 2
 )
 
-type ContainerdController interface {
-	Load(ctx context.Context, log *zerolog.Logger) (*registry.DefaultZotConfig, error)
-	Generate(ctx context.Context, configPath string, log *zerolog.Logger) error
-}
-
 var DefaultContainerDGenPath string
 
 func init() {
@@ -49,7 +43,7 @@ func init() {
 
 func NewContainerdCommand() *cobra.Command {
 	var generateConfig bool
-	var defaultZotConfig *registry.DefaultZotConfig
+	var defaultZotConfig registry.DefaultZotConfig
 	var containerdConfigPath string
 	var containerDCertPath string
 
@@ -61,8 +55,8 @@ func NewContainerdCommand() *cobra.Command {
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			log := logger.FromContext(cmd.Context())
-			sourceRegistry := config.GetRemoteRegistryURL()
-			satelliteHostConfig := NewSatelliteHostConfig(defaultZotConfig.GetLocalRegistryURL(), sourceRegistry)
+			sourceRegistry := config.GetSourceRegistryURL()
+			satelliteHostConfig := NewSatelliteHostConfig(defaultZotConfig.RemoteURL, sourceRegistry)
 			if generateConfig {
 				log.Info().Msg("Generating containerd config file for containerd ...")
 				log.Info().Msgf("Fetching containerd config from path: %s", containerdConfigPath)
@@ -71,7 +65,7 @@ func NewContainerdCommand() *cobra.Command {
 					log.Err(err).Msg("Error generating containerd config")
 					return fmt.Errorf("could not generate containerd config: %w", err)
 				}
-				return GenerateContainerdConfig(defaultZotConfig, log, containerdConfigPath, containerDCertPath)
+				return GenerateContainerdConfig(log, containerdConfigPath, containerDCertPath)
 			}
 			return nil
 		},
@@ -87,12 +81,17 @@ func NewContainerdCommand() *cobra.Command {
 // GenerateContainerdConfig generates the containerd config file for the containerd runtime
 // It takes the zot config a logger and the containerd config path
 // It reads the containerd config file and adds the local registry to the config file
-func GenerateContainerdConfig(defaultZotConfig *registry.DefaultZotConfig, log *zerolog.Logger, containerdConfigPath, containerdCertPath string) error {
+func GenerateContainerdConfig(log *zerolog.Logger, containerdConfigPath, containerdCertPath string) error {
 	// First Read the present config file at the configPath
 	data, err := utils.ReadFile(containerdConfigPath, false)
 	if err != nil {
-		log.Err(err).Msg("Error reading config file")
-		return fmt.Errorf("could not read config file: %w", err)
+		if os.IsNotExist(err) {
+			log.Warn().Msg("Config file does not exist, proceeding with default values")
+			data = []byte{}
+		} else {
+			log.Err(err).Msg("Error reading config file")
+			return fmt.Errorf("could not read config file: %w", err)
+		}
 	}
 	// Now we marshal the data into the containerd config
 	containerdConfig := &ContainerdConfigToml{}

diff --git a/cmd/container_runtime/crio.go b/cmd/container_runtime/crio.go
@@ -2,6 +2,7 @@ package runtime
 
 import (
 	"fmt"
+	"net/url"
 	"os"
 	"path/filepath"
 
@@ -45,7 +46,7 @@ func NewCrioCommand() *cobra.Command {
 		Use:   "crio",
 		Short: "Creates the config file for the crio runtime to fetch the images from the local repository",
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			return SetupContainerRuntimeCommand(cmd, &defaultZotConfig, DefaultCrioGenPath)
+			return SetupContainerRuntimeCommand(cmd, defaultZotConfig, DefaultCrioGenPath)
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			log := logger.FromContext(cmd.Context())
@@ -71,7 +72,13 @@ func GenerateCrioRegistryConfig(defaultZotConfig *registry.DefaultZotConfig, cri
 	// Read the current crio registry config file
 	data, err := utils.ReadFile(crioConfigPath, false)
 	if err != nil {
-		return fmt.Errorf("could not read crio registry config file: %w", err)
+		if os.IsNotExist(err) {
+			log.Warn().Msg("Config file does not exist, proceeding with default values")
+			data = []byte{}
+		} else {
+			log.Err(err).Msg("Error reading config file")
+			return fmt.Errorf("could not read config file: %w", err)
+		}
 	}
 	var crioRegistryConfig CriORegistryConfig
 	err = toml.Unmarshal(data, &crioRegistryConfig)
@@ -82,7 +89,7 @@ func GenerateCrioRegistryConfig(defaultZotConfig *registry.DefaultZotConfig, cri
 	// Update the crio registry config file
 	// - Add the local registry to the unqualified search registries if not already present
 	var found bool = false
-	var localRegistry string = utils.FormatRegistryURL(defaultZotConfig.GetLocalRegistryURL())
+	var localRegistry string = utils.FormatRegistryURL(defaultZotConfig.RemoteURL)
 	for _, registry := range crioRegistryConfig.UnqualifiedSearchRegistries {
 		if registry == localRegistry {
 			found = true
@@ -141,42 +148,27 @@ func GenerateCrioRegistryConfig(defaultZotConfig *registry.DefaultZotConfig, cri
 	return nil
 }
 
-func SetupContainerRuntimeCommand(cmd *cobra.Command, defaultZotConfig **registry.DefaultZotConfig, defaultGenPath string) error {
+func SetupContainerRuntimeCommand(cmd *cobra.Command, defaultZotConfig *registry.DefaultZotConfig, defaultGenPath string) error {
+	utils.CommandRunSetup(cmd)
 	var err error
-	checks, warnings := config.InitConfig(config.DefaultConfigPath)
-	if len(checks) > 0 || len(warnings) > 0 {
-		log := logger.FromContext(cmd.Context())
-		for _, warn := range warnings {
-			log.Warn().Msg(warn)
-		}
-		for _, err := range checks {
-			log.Error().Err(err).Msg("Error initializing config")
-		}
-		return fmt.Errorf("error initializing config")
-	}
-	if err != nil {
-		return fmt.Errorf("could not initialize config: %w", err)
-	}
-	utils.SetupContextForCommand(cmd)
 	log := logger.FromContext(cmd.Context())
 
 	if config.GetOwnRegistry() {
 		log.Info().Msg("Using own registry for config generation")
-		address, err := utils.ValidateRegistryAddress(config.GetOwnRegistryAdr(), config.GetOwnRegistryPort())
+		log.Info().Msgf("Remote registry URL: %s", config.GetRemoteRegistryURL())
+		_, err := url.Parse(config.GetRemoteRegistryURL())
 		if err != nil {
-			log.Err(err).Msg("Error validating registry address")
-			return err
+			return fmt.Errorf("could not parse remote registry URL: %w", err)
 		}
-		log.Info().Msgf("Registry address validated: %s", address)
-		(*defaultZotConfig).HTTP.Address = config.GetOwnRegistryAdr()
-		(*defaultZotConfig).HTTP.Port = config.GetOwnRegistryPort()
+		defaultZotConfig.RemoteURL = config.GetRemoteRegistryURL()
 	} else {
 		log.Info().Msg("Using default registry for config generation")
-		*defaultZotConfig, err = registry.ReadConfig(config.GetZotConfigPath())
-		if err != nil || *defaultZotConfig == nil {
+		defaultZotConfig, err = registry.ReadConfig(config.GetZotConfigPath())
+		if err != nil || defaultZotConfig == nil {
 			return fmt.Errorf("could not read config: %w", err)
 		}
-		log.Info().Msgf("Default config read successfully: %v", (*defaultZotConfig).HTTP.Address+":"+(*defaultZotConfig).HTTP.Port)
+		defaultZotConfig.RemoteURL = defaultZotConfig.GetLocalRegistryURL()
+		log.Info().Msgf("Default config read successfully: %v", defaultZotConfig.HTTP.Address+":"+defaultZotConfig.HTTP.Port)
 	}
 	return utils.CreateRuntimeDirectory(defaultGenPath)
 }
diff --git a/cmd/container_runtime/read_config.go b/cmd/container_runtime/read_config.go
@@ -2,9 +2,7 @@ package runtime
 
 import (
 	"fmt"
-	"os"
 
-	"container-registry.com/harbor-satellite/internal/config"
 	"container-registry.com/harbor-satellite/internal/utils"
 	"container-registry.com/harbor-satellite/logger"
 	"github.com/spf13/cobra"
@@ -14,23 +12,8 @@ func NewReadConfigCommand(runtime string) *cobra.Command {
 	readContainerdConfig := &cobra.Command{
 		Use:   "read",
 		Short: fmt.Sprintf("Reads the config file for the %s runtime", runtime),
-		PersistentPreRun: func(cmd *cobra.Command, args []string) {
-			checks, warnings := config.InitConfig(config.DefaultConfigPath)
-			if len(checks) > 0 || len(warnings) > 0 {
-				ctx := cmd.Context()
-				ctx, cancel := utils.SetupContext(ctx)
-				ctx = logger.AddLoggerToContext(ctx, "info")
-				log := logger.FromContext(ctx)
-				for _, warn := range warnings {
-					log.Warn().Msg(warn)
-				}
-				for _, err := range checks {
-					log.Error().Err(err).Msg("Error initializing config")
-				}
-				cancel()
-				os.Exit(1)
-			}
-			utils.SetupContextForCommand(cmd)
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			return utils.CommandRunSetup(cmd)
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			//Parse the flags

diff --git a/cmd/root.go b/cmd/root.go
@@ -2,7 +2,6 @@ package cmd
 
 import (
 	"context"
-	"os"
 
 	runtime "container-registry.com/harbor-satellite/cmd/container_runtime"
 	"container-registry.com/harbor-satellite/internal/config"
@@ -20,29 +19,12 @@ func NewRootCommand() *cobra.Command {
 	rootCmd := &cobra.Command{
 		Use:   "harbor-satellite",
 		Short: "Harbor Satellite is a tool to replicate images from source registry to Harbor registry",
-		PersistentPreRun: func(cmd *cobra.Command, args []string) {
-			errors, warnings := config.InitConfig(config.DefaultConfigPath)
-			if len(errors) > 0 || len(warnings) > 0 {
-				ctx := cmd.Context()
-				ctx, cancel := utils.SetupContext(ctx)
-				ctx = logger.AddLoggerToContext(ctx, "info")
-				log := logger.FromContext(ctx)
-				for _, warn := range warnings {
-					log.Warn().Msg(warn)
-				}
-				for _, err := range errors {
-					log.Error().Err(err).Msg("Error initializing config")
-				}
-				if len(errors) > 0 {
-					cancel()
-					os.Exit(1)
-				}
-			}
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			return utils.CommandRunSetup(cmd)
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			ctx := cmd.Context()
 			ctx, cancel := utils.SetupContext(ctx)
-			ctx = logger.AddLoggerToContext(ctx, config.GetLogLevel())
 			return run(ctx, cancel)
 		},
 	}
@@ -75,15 +57,20 @@ func run(ctx context.Context, cancel context.CancelFunc) error {
 		log.Error().Err(err).Msg("Error starting scheduler")
 		return err
 	}
-
-	satelliteService := satellite.NewSatellite(ctx, scheduler.GetSchedulerKey())
+	localRegistryConfig := satellite.NewRegistryConfig(config.GetRemoteRegistryURL(), config.GetRemoteRegistryUsername(), config.GetRemoteRegistryPassword())
+	sourceRegistryConfig := satellite.NewRegistryConfig(config.GetSourceRegistryURL(), config.GetSourceRegistryUsername(), config.GetSourceRegistryPassword())
+	states := config.GetStates()
+	useUnsecure := config.UseUnsecure()
+	satelliteService := satellite.NewSatellite(ctx, scheduler.GetSchedulerKey(), localRegistryConfig, sourceRegistryConfig, useUnsecure, states)
 
 	g.Go(func() error {
 		return satelliteService.Run(ctx)
 	})
 
 	log.Info().Msg("Startup complete 🚀")
-	return g.Wait()
+	g.Wait()
+	scheduler.Stop()
+	return nil
 }
 
 func setupServerApp(ctx context.Context, log *zerolog.Logger) *server.App {

diff --git a/config.json b/config.json
@@ -1,34 +1,39 @@
 {
+  "state_config": {
+    "auth": {
+      "name": "admin",
+      "registry": "https://registry.bupd.xyz",
+      "secret": "Harbor12345"
+    },
+    "states": [
+      "https://registry.bupd.xyz/satellite/mehulsat/state"
+    ]
+  },
   "environment_variables": {
-    "bring_own_registry": false,
     "ground_control_url": "https://gc.bupd.xyz",
     "log_level": "info",
-    "own_registry_adr": "127.0.0.1",
-    "own_registry_port": "8585",
-    "zot_config_path": "./registry/config.json",
     "use_unsecure": true,
-    "token": "187916a603c135d681ef0553752994f3",
-    "state_fetch_period": "",
-    "config_fetch_period": "",
+    "zot_config_path": "./registry/config.json",
+    "token": "",
     "jobs": [
       {
         "name": "replicate_state",
-        "seconds": "10",
-        "minutes": "*",
-        "hours": "*",
-        "day_of_month": "*",
-        "month": "*",
-        "day_of_week": "*"
+        "schedule": "@every 00h00m10s"
       },
       {
         "name": "update_config",
-        "seconds": "5",
-        "minutes": "asdfasdfsaf",
-        "hours": "asdfas",
-        "day_of_month": "*",
-        "month": "*",
-        "day_of_week": "dsffa"
+        "schedule": "@every 00h00m30s"
+      },
+      {
+        "name": "register_satellite",
+        "schedule": "@every 00h00m05s"
       }
-    ]
+    ],
+    "local_registry": {
+      "url": "http://127.0.0.1:8585",
+      "username": "methylisocyanide",
+      "password": "Harbor12345",
+      "bring_own_registry": false
+    }
   }
-}
+}