From b0304ff9b5da6dbfba50be99e7c9e0c5092a0aa2 Mon Sep 17 00:00:00 2001 From: Mehul-Kumar-27 Date: Tue, 24 Sep 2024 03:13:29 +0530 Subject: [PATCH 01/36] adding state artifact fetcher to fetch the state from harbor --- .env | 2 + ci/utils.go | 4 +- config.toml | 3 +- internal/config/artifact.go | 34 ++++++ internal/config/config.go | 77 ++++++++----- internal/config/state_aritfact.go | 180 ++++++++++++++++++++++++++++++ main.go | 8 +- 7 files changed, 276 insertions(+), 32 deletions(-) create mode 100644 internal/config/artifact.go create mode 100644 internal/config/state_aritfact.go diff --git a/.env b/.env index 6a08828..73ed245 100644 --- a/.env +++ b/.env @@ -4,3 +4,5 @@ ZOT_URL="127.0.0.1:8585" TOKEN="" ENV=dev USE_UNSECURE=true +GROUP_NAME=test-satellite-group +STATE_ARTIFACT_NAME=state-artifact diff --git a/ci/utils.go b/ci/utils.go index b0367d4..d641451 100644 --- a/ci/utils.go +++ b/ci/utils.go @@ -59,8 +59,8 @@ func (m *HarborSatellite) Service( } // Would build the project with the source provided. The name should be the name of the project. -func (m *HarborSatellite) build(source *dagger.Directory, name string) *dagger.Directory { - fmt.Printf("Building %s\n", name) +func (m *HarborSatellite) build(source *dagger.Directory, component string) *dagger.Directory { + fmt.Printf("Building %s\n", component) gooses := []string{"linux", "darwin"} goarches := []string{"amd64", "arm64"} diff --git a/config.toml b/config.toml index 8e02820..1d0161d 100644 --- a/config.toml +++ b/config.toml @@ -6,7 +6,8 @@ own_registry_adr = "127.0.0.1" own_registry_port = "8585" # URL of remote registry OR local file path -url_or_file = "https://demo.goharbor.io/v2/myproject/album-server" +# url_or_file = "https://demo.goharbor.io/v2/myproject/album-server" +url_or_file = "https://demo.goharbor.io" # Default path for Zot registry config.json zotConfigPath = "./registry/config.json" diff --git a/internal/config/artifact.go b/internal/config/artifact.go new file mode 100644 index 0000000..287c5dc --- /dev/null +++ b/internal/config/artifact.go @@ -0,0 +1,34 @@ +package config + +// ArtifactReader defines an interface for reading artifact data +type ArtifactReader interface { + // GetRepository returns the repository of the artifact + GetRepository() string + // GetTag returns the tag of the artifact + GetTag() string + // GetHash returns the hash of the artifact + GetHash() string +} + +// Artifact represents an artifact object in the registry +type Artifact struct { + Repository string `json:"repository"` + Tag string `json:"tag"` + Hash string `json:"hash"` +} + +func NewArtifact() ArtifactReader { + return &Artifact{} +} + +func (a *Artifact) GetRepository() string { + return a.Repository +} + +func (a *Artifact) GetTag() string { + return a.Tag +} + +func (a *Artifact) GetHash() string { + return a.Hash +} diff --git a/internal/config/config.go b/internal/config/config.go index 93ae830..3915322 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -11,23 +11,26 @@ import ( var AppConfig *Config type Config struct { - log_level string - own_registry bool - own_registry_adr string - own_registry_port string - zot_config_path string - input string - zot_url string - registry string - repository string - user_input string - scheme string - api_version string - image string - harbor_password string - harbor_username string - env string - use_unsecure bool + log_level string + own_registry bool + own_registry_adr string + own_registry_port string + zot_config_path string + input string + zot_url string + registry string + repository string + user_input string + scheme string + api_version string + image string + harbor_password string + harbor_username string + env string + use_unsecure bool + remote_registry_url string + group_name string + state_artifact_name string } func GetLogLevel() string { @@ -122,6 +125,22 @@ func GetHarborUsername() string { return AppConfig.harbor_username } +func SetRemoteRegistryURL(url string) { + AppConfig.remote_registry_url = url +} + +func GetRemoteRegistryURL() string { + return AppConfig.remote_registry_url +} + +func GetGroupName() string { + return AppConfig.group_name +} + +func GetStateArtifactName() string { + return AppConfig.state_artifact_name +} + func LoadConfig() (*Config, error) { viper.SetConfigName("config") viper.SetConfigType("toml") @@ -142,17 +161,19 @@ func LoadConfig() (*Config, error) { } return &Config{ - log_level: viper.GetString("log_level"), - own_registry: viper.GetBool("bring_own_registry"), - own_registry_adr: viper.GetString("own_registry_adr"), - own_registry_port: viper.GetString("own_registry_port"), - zot_config_path: viper.GetString("zotConfigPath"), - input: viper.GetString("url_or_file"), - harbor_password: os.Getenv("HARBOR_PASSWORD"), - harbor_username: os.Getenv("HARBOR_USERNAME"), - env: os.Getenv("ENV"), - zot_url: os.Getenv("ZOT_URL"), - use_unsecure: use_unsecure, + log_level: viper.GetString("log_level"), + own_registry: viper.GetBool("bring_own_registry"), + own_registry_adr: viper.GetString("own_registry_adr"), + own_registry_port: viper.GetString("own_registry_port"), + zot_config_path: viper.GetString("zotConfigPath"), + input: viper.GetString("url_or_file"), + harbor_password: os.Getenv("HARBOR_PASSWORD"), + harbor_username: os.Getenv("HARBOR_USERNAME"), + env: os.Getenv("ENV"), + zot_url: os.Getenv("ZOT_URL"), + use_unsecure: use_unsecure, + group_name: os.Getenv("GROUP_NAME"), + state_artifact_name: os.Getenv("STATE_ARTIFACT_NAME"), }, nil } diff --git a/internal/config/state_aritfact.go b/internal/config/state_aritfact.go new file mode 100644 index 0000000..ed43bc9 --- /dev/null +++ b/internal/config/state_aritfact.go @@ -0,0 +1,180 @@ +package config + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + + "oras.land/oras-go/v2" + "oras.land/oras-go/v2/content/file" + "oras.land/oras-go/v2/registry/remote" + "oras.land/oras-go/v2/registry/remote/auth" + "oras.land/oras-go/v2/registry/remote/retry" +) + +// Registry defines an interface for registry operations +type StateReader interface { + // GetRegistryURL returns the URL of the registry + GetRegistryURL() string + // GetRegistryType returns the list of artifacts that needs to be pulled + GetArtifacts() []ArtifactReader + // GetArtifactByRepository takes in the repository name and returns the artifact associated with it + GetArtifactByRepository(repo string) (ArtifactReader, error) +} + +type State struct { + Registry string `json:"registry"` + Artifacts []Artifact `json:"artifacts"` +} + +func NewState(artifact *Artifact) StateReader { + state := &State{ + Registry: "", + Artifacts: []Artifact{*artifact}, + } + return state +} + +func (a *State) GetRegistryURL() string { + return a.Registry +} + +func (a *State) GetArtifacts() []ArtifactReader { + var artifact_readers []ArtifactReader + for _, artifact := range a.Artifacts { + artifact_readers = append(artifact_readers, &artifact) + } + return artifact_readers +} + +func (a *State) GetArtifactByRepository(repo string) (ArtifactReader, error) { + for _, artifact := range a.Artifacts { + if artifact.GetRepository() == repo { + return &artifact, nil + } + } + return &Artifact{}, fmt.Errorf("artifact not found in the list") +} + +type StateArtifactFetcher interface { + // Fetches the state artifact from the registry + FetchStateArtifact() error +} + +type URLStateArtifactFetcher struct { + url string + group_name string + state_artifact_name string + state_artifact_reader StateReader +} + +func NewURLStateArtifactFetcher() StateArtifactFetcher { + url := GetRemoteRegistryURL() + // Trim the "https://" or "http://" prefix if present + if len(url) >= 8 && url[:8] == "https://" { + url = url[8:] + } else if len(url) >= 7 && url[:7] == "http://" { + url = url[7:] + } + artifact := NewArtifact() + state_artifact_reader := NewState(artifact.(*Artifact)) + return &URLStateArtifactFetcher{ + url: url, + group_name: GetGroupName(), + state_artifact_name: GetStateArtifactName(), + state_artifact_reader: state_artifact_reader, + } +} + +type FileStateArtifactFetcher struct { + filePath string +} + +func (f *URLStateArtifactFetcher) FetchStateArtifact() error { + cwd, err := os.Getwd() + if err != nil { + return fmt.Errorf("failed to get current working directory: %v", err) + } + // Creating a file store in the current working directory + fs, err := file.New(fmt.Sprintf("%s/state-artifact", cwd)) + if err != nil { + return fmt.Errorf("failed to create file store: %v", err) + } + defer fs.Close() + + ctx := context.Background() + + repo, err := remote.NewRepository(fmt.Sprintf("%s/%s/%s", f.url, f.group_name, f.state_artifact_name)) + if err != nil { + return fmt.Errorf("failed to create remote repository: %v", err) + } + + // Setting up the authentication for the remote registry + repo.Client = &auth.Client{ + Client: retry.DefaultClient, + Cache: auth.NewCache(), + Credential: auth.StaticCredential( + f.url, + auth.Credential{ + Username: GetHarborUsername(), + Password: GetHarborPassword(), + }, + ), + } + // Copy from the remote repository to the file store + tag := "latest" + _, err = oras.Copy(ctx, repo, tag, fs, tag, oras.DefaultCopyOptions) + if err != nil { + return fmt.Errorf("failed to copy from remote repository to file store: %v", err) + } + stateArtifactDir := filepath.Join(cwd, "state-artifact") + // Find the state artifact file in the state-artifact directory that is created temporarily + err = filepath.Walk(stateArtifactDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if filepath.Ext(info.Name()) == ".json" { + content, err := os.ReadFile(path) + if err != nil { + return err + } + fmt.Printf("Contents of %s:\n", info.Name()) + fmt.Println(string(content)) + + state_artifact_reader, err := FromJSON(content, f.state_artifact_reader.(*State)) + if err != nil { + return fmt.Errorf("failed to parse the state artifact file: %v", err) + } + fmt.Println(state_artifact_reader) + + } + return nil + }) + + if err != nil { + return fmt.Errorf("failed to read the state artifact file: %v", err) + } + // Clean up everything inside the state-artifact folder + err = os.RemoveAll(stateArtifactDir) + if err != nil { + return fmt.Errorf("failed to remove state-artifact directory: %v", err) + } + return nil +} + +// FromJSON parses the input JSON data into a StateArtifactReader +func FromJSON(data []byte, reg *State) (StateReader, error) { + + if err := json.Unmarshal(data, ®); err != nil { + fmt.Print("Error in unmarshalling") + return nil, err + } + fmt.Print(reg) + // Validation + if reg.Registry == "" { + return nil, fmt.Errorf("registry URL is required") + } + return reg, nil +} diff --git a/main.go b/main.go index 68d8d1f..12df1db 100644 --- a/main.go +++ b/main.go @@ -128,8 +128,14 @@ func processInput(ctx context.Context, log *zerolog.Logger) (store.ImageFetcher, } log.Info().Msg("Input is a valid URL") + config.SetRemoteRegistryURL(input) + state_arifact_fetcher := config.NewURLStateArtifactFetcher() + if err := state_arifact_fetcher.FetchStateArtifact(); err != nil { + log.Error().Err(err).Msg("Error fetching state artifact") + return nil, err + } fetcher := store.RemoteImageListFetcher(ctx, input) - utils.SetUrlConfig(input) + // utils.SetUrlConfig(input) return fetcher, nil } From 8a5ba111123408ef6f0e893fb41a589a0b8ddf8f Mon Sep 17 00:00:00 2001 From: Mehul-Kumar-27 Date: Wed, 25 Sep 2024 19:34:37 +0530 Subject: [PATCH 02/36] completing the url fetcher --- internal/replicate/replicate.go | 55 ++++++++++++- internal/satellite/satellite.go | 2 +- internal/{config => state}/artifact.go | 2 +- .../state_aritfact.go => state/state.go} | 77 ++++++++++--------- internal/utils/utils.go | 10 +++ main.go | 9 ++- 6 files changed, 112 insertions(+), 43 deletions(-) rename internal/{config => state}/artifact.go (97%) rename internal/{config/state_aritfact.go => state/state.go} (64%) diff --git a/internal/replicate/replicate.go b/internal/replicate/replicate.go index 59bb805..bc323e6 100644 --- a/internal/replicate/replicate.go +++ b/internal/replicate/replicate.go @@ -9,7 +9,9 @@ import ( "strings" "container-registry.com/harbor-satellite/internal/config" + "container-registry.com/harbor-satellite/internal/state" "container-registry.com/harbor-satellite/internal/store" + "container-registry.com/harbor-satellite/internal/utils" "container-registry.com/harbor-satellite/logger" "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/crane" @@ -17,7 +19,7 @@ import ( type Replicator interface { // Replicate copies images from the source registry to the local registry. - Replicate(ctx context.Context, image string) error + Replicate(ctx context.Context) error DeleteExtraImages(ctx context.Context, imgs []store.Image) error } @@ -26,6 +28,7 @@ type BasicReplicator struct { password string use_unsecure bool zot_url string + state_reader state.StateReader } type ImageInfo struct { @@ -42,16 +45,62 @@ type RegistryInfo struct { Repositories []Repository `json:"repositories"` } -func NewReplicator(context context.Context) Replicator { +func NewReplicator(state_reader state.StateReader) Replicator { return &BasicReplicator{ username: config.GetHarborUsername(), password: config.GetHarborPassword(), use_unsecure: config.UseUnsecure(), zot_url: config.GetZotURL(), + state_reader: state_reader, } } -func (r *BasicReplicator) Replicate(ctx context.Context, image string) error { +func (r *BasicReplicator) Replicate(ctx context.Context) error { + log := logger.FromContext(ctx) + auth := authn.FromConfig(authn.AuthConfig{ + Username: r.username, + Password: r.password, + }) + + options := []crane.Option{crane.WithAuth(auth)} + if r.use_unsecure { + options = append(options, crane.Insecure) + } + source_registry := r.state_reader.GetRegistryURL() + for _, artifact := range r.state_reader.GetArtifacts() { + // Extract the image name from the repository of the artifact + repo, image, err := utils.GetRepositoryAndImageNameFromArtifact(artifact.GetRepository()) + if err != nil { + log.Error().Msgf("Error getting repository and image name: %v", err) + return err + } + log.Info().Msgf("Pulling image %s from repository %s at registry %s", image, repo, source_registry) + // Pull the image at the given repository at the source registry + srcImage, err := crane.Pull(fmt.Sprintf("%s/%s/%s", source_registry, repo, image), options...) + if err != nil { + logger.FromContext(ctx).Error().Msgf("Failed to pull image: %v", err) + return err + } + + // Push the image to the local registry + err = crane.Push(srcImage, fmt.Sprintf("%s/%s", r.zot_url, image), options...) + if err != nil { + logger.FromContext(ctx).Error().Msgf("Failed to push image: %v", err) + return err + } + log.Info().Msgf("Image %s pushed successfully", image) + } + // Delete ./local-oci-layout directory + // This is required because it is a temporary directory used by crane to pull and push images to and from + // And crane does not automatically clean it + if err := os.RemoveAll("./local-oci-layout"); err != nil { + log.Error().Msgf("Failed to remove directory: %v", err) + return fmt.Errorf("failed to remove directory: %w", err) + } + return nil +} + +func (r *BasicReplicator) Rep(ctx context.Context, image string) error { source := getPullSource(ctx, image) diff --git a/internal/satellite/satellite.go b/internal/satellite/satellite.go index b2f6b0a..71c2a16 100644 --- a/internal/satellite/satellite.go +++ b/internal/satellite/satellite.go @@ -34,7 +34,7 @@ func (s *Satellite) Run(ctx context.Context) error { log.Info().Msg("No images to replicate") } else { for _, img := range imgs { - err = s.replicator.Replicate(ctx, img.Name) + err = s.replicator.Replicate(ctx) if err != nil { log.Error().Err(err).Msg("Error replicating image") return err diff --git a/internal/config/artifact.go b/internal/state/artifact.go similarity index 97% rename from internal/config/artifact.go rename to internal/state/artifact.go index 287c5dc..8989307 100644 --- a/internal/config/artifact.go +++ b/internal/state/artifact.go @@ -1,4 +1,4 @@ -package config +package state // ArtifactReader defines an interface for reading artifact data type ArtifactReader interface { diff --git a/internal/config/state_aritfact.go b/internal/state/state.go similarity index 64% rename from internal/config/state_aritfact.go rename to internal/state/state.go index ed43bc9..c97a598 100644 --- a/internal/config/state_aritfact.go +++ b/internal/state/state.go @@ -1,4 +1,4 @@ -package config +package state import ( "context" @@ -7,6 +7,7 @@ import ( "os" "path/filepath" + "container-registry.com/harbor-satellite/internal/config" "oras.land/oras-go/v2" "oras.land/oras-go/v2/content/file" "oras.land/oras-go/v2/registry/remote" @@ -16,7 +17,7 @@ import ( // Registry defines an interface for registry operations type StateReader interface { - // GetRegistryURL returns the URL of the registry + // GetRegistryURL returns the URL of the registry after removing the "https://" or "http://" prefix if present and the trailing "/" GetRegistryURL() string // GetRegistryType returns the list of artifacts that needs to be pulled GetArtifacts() []ArtifactReader @@ -29,24 +30,30 @@ type State struct { Artifacts []Artifact `json:"artifacts"` } -func NewState(artifact *Artifact) StateReader { - state := &State{ - Registry: "", - Artifacts: []Artifact{*artifact}, - } +func NewState() StateReader { + state := &State{} return state } func (a *State) GetRegistryURL() string { - return a.Registry + registry := a.Registry + if len(registry) >= 8 && registry[:8] == "https://" { + registry = registry[8:] + } else if len(registry) >= 7 && registry[:7] == "http://" { + registry = registry[7:] + } + if len(registry) > 0 && registry[len(registry)-1] == '/' { + registry = registry[:len(registry)-1] + } + return registry } func (a *State) GetArtifacts() []ArtifactReader { - var artifact_readers []ArtifactReader - for _, artifact := range a.Artifacts { - artifact_readers = append(artifact_readers, &artifact) - } - return artifact_readers + var artifact_readers []ArtifactReader + for _, artifact := range a.Artifacts { + artifact_readers = append(artifact_readers, &artifact) + } + return artifact_readers } func (a *State) GetArtifactByRepository(repo string) (ArtifactReader, error) { @@ -60,30 +67,29 @@ func (a *State) GetArtifactByRepository(repo string) (ArtifactReader, error) { type StateArtifactFetcher interface { // Fetches the state artifact from the registry - FetchStateArtifact() error + FetchStateArtifact() (StateReader, error) } -type URLStateArtifactFetcher struct { +type URLStateFetcher struct { url string group_name string state_artifact_name string state_artifact_reader StateReader } -func NewURLStateArtifactFetcher() StateArtifactFetcher { - url := GetRemoteRegistryURL() +func NewURLStateFetcher() StateArtifactFetcher { + url := config.GetRemoteRegistryURL() // Trim the "https://" or "http://" prefix if present if len(url) >= 8 && url[:8] == "https://" { url = url[8:] } else if len(url) >= 7 && url[:7] == "http://" { url = url[7:] } - artifact := NewArtifact() - state_artifact_reader := NewState(artifact.(*Artifact)) - return &URLStateArtifactFetcher{ + state_artifact_reader := NewState() + return &URLStateFetcher{ url: url, - group_name: GetGroupName(), - state_artifact_name: GetStateArtifactName(), + group_name: config.GetGroupName(), + state_artifact_name: config.GetStateArtifactName(), state_artifact_reader: state_artifact_reader, } } @@ -92,15 +98,15 @@ type FileStateArtifactFetcher struct { filePath string } -func (f *URLStateArtifactFetcher) FetchStateArtifact() error { +func (f *URLStateFetcher) FetchStateArtifact() (StateReader, error) { cwd, err := os.Getwd() if err != nil { - return fmt.Errorf("failed to get current working directory: %v", err) + return nil, fmt.Errorf("failed to get current working directory: %v", err) } - // Creating a file store in the current working directory + // Creating a file store in the current working directory will be deleted later after reading the state artifact fs, err := file.New(fmt.Sprintf("%s/state-artifact", cwd)) if err != nil { - return fmt.Errorf("failed to create file store: %v", err) + return nil, fmt.Errorf("failed to create file store: %v", err) } defer fs.Close() @@ -108,7 +114,7 @@ func (f *URLStateArtifactFetcher) FetchStateArtifact() error { repo, err := remote.NewRepository(fmt.Sprintf("%s/%s/%s", f.url, f.group_name, f.state_artifact_name)) if err != nil { - return fmt.Errorf("failed to create remote repository: %v", err) + return nil, fmt.Errorf("failed to create remote repository: %v", err) } // Setting up the authentication for the remote registry @@ -118,8 +124,8 @@ func (f *URLStateArtifactFetcher) FetchStateArtifact() error { Credential: auth.StaticCredential( f.url, auth.Credential{ - Username: GetHarborUsername(), - Password: GetHarborPassword(), + Username: config.GetHarborUsername(), + Password: config.GetHarborPassword(), }, ), } @@ -127,9 +133,11 @@ func (f *URLStateArtifactFetcher) FetchStateArtifact() error { tag := "latest" _, err = oras.Copy(ctx, repo, tag, fs, tag, oras.DefaultCopyOptions) if err != nil { - return fmt.Errorf("failed to copy from remote repository to file store: %v", err) + return nil, fmt.Errorf("failed to copy from remote repository to file store: %v", err) } stateArtifactDir := filepath.Join(cwd, "state-artifact") + + var state_reader StateReader // Find the state artifact file in the state-artifact directory that is created temporarily err = filepath.Walk(stateArtifactDir, func(path string, info os.FileInfo, err error) error { if err != nil { @@ -143,25 +151,24 @@ func (f *URLStateArtifactFetcher) FetchStateArtifact() error { fmt.Printf("Contents of %s:\n", info.Name()) fmt.Println(string(content)) - state_artifact_reader, err := FromJSON(content, f.state_artifact_reader.(*State)) + state_reader, err = FromJSON(content, f.state_artifact_reader.(*State)) if err != nil { return fmt.Errorf("failed to parse the state artifact file: %v", err) } - fmt.Println(state_artifact_reader) } return nil }) if err != nil { - return fmt.Errorf("failed to read the state artifact file: %v", err) + return nil, fmt.Errorf("failed to read the state artifact file: %v", err) } // Clean up everything inside the state-artifact folder err = os.RemoveAll(stateArtifactDir) if err != nil { - return fmt.Errorf("failed to remove state-artifact directory: %v", err) + return nil, fmt.Errorf("failed to remove state-artifact directory: %v", err) } - return nil + return state_reader, nil } // FromJSON parses the input JSON data into a StateArtifactReader diff --git a/internal/utils/utils.go b/internal/utils/utils.go index bac9c69..89cbf19 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -137,3 +137,13 @@ func SetUrlConfig(input string) { os.Setenv("IMAGE", registryParts[3]) config.SetImage(registryParts[3]) } + +func GetRepositoryAndImageNameFromArtifact(repository string) (string, string, error){ + parts := strings.Split(repository, "/") + if len(parts) < 2 { + return "", "", fmt.Errorf("invalid repository format") + } + repo := parts[0] + image := parts[1] + return repo, image, nil +} diff --git a/main.go b/main.go index 12df1db..7fb3af1 100644 --- a/main.go +++ b/main.go @@ -12,6 +12,7 @@ import ( "container-registry.com/harbor-satellite/internal/replicate" "container-registry.com/harbor-satellite/internal/satellite" "container-registry.com/harbor-satellite/internal/server" + "container-registry.com/harbor-satellite/internal/state" "container-registry.com/harbor-satellite/internal/store" "container-registry.com/harbor-satellite/internal/utils" "container-registry.com/harbor-satellite/logger" @@ -58,7 +59,7 @@ func run() error { } ctx, storer := store.NewInMemoryStore(ctx, fetcher) - replicator := replicate.NewReplicator(ctx) + replicator := replicate.NewReplicator() satelliteService := satellite.NewSatellite(ctx, storer, replicator) g.Go(func() error { @@ -129,8 +130,10 @@ func processInput(ctx context.Context, log *zerolog.Logger) (store.ImageFetcher, log.Info().Msg("Input is a valid URL") config.SetRemoteRegistryURL(input) - state_arifact_fetcher := config.NewURLStateArtifactFetcher() - if err := state_arifact_fetcher.FetchStateArtifact(); err != nil { + fmt.Println(config.GetRemoteRegistryURL()) + state_artifact_fetcher := state.NewURLStateFetcher() + _, err := state_artifact_fetcher.FetchStateArtifact() + if err != nil { log.Error().Err(err).Msg("Error fetching state artifact") return nil, err } From c291e6a90e133053d41099a287a76f70200553aa Mon Sep 17 00:00:00 2001 From: Mehul-Kumar-27 Date: Sun, 29 Sep 2024 22:12:42 +0530 Subject: [PATCH 03/36] adding schedulers and process to satellite --- config.toml | 2 + go.mod | 1 + go.sum | 2 + image-list/images.json | 27 ++-- internal/config/config.go | 6 + internal/replicate/replicate.go | 275 -------------------------------- internal/satellite/satellite.go | 81 +++------- internal/scheduler/process.go | 21 +++ internal/scheduler/scheduler.go | 99 ++++++++++++ internal/server/server.go | 9 +- internal/state/artifact.go | 6 + internal/state/replicator.go | 94 +++++++++++ internal/state/state.go | 52 +++++- internal/state/state_process.go | 84 ++++++++++ internal/utils/utils.go | 85 ++++------ logger/logger.go | 80 ++++++++-- main.go | 76 +++++---- 17 files changed, 536 insertions(+), 464 deletions(-) delete mode 100644 internal/replicate/replicate.go create mode 100644 internal/scheduler/process.go create mode 100644 internal/scheduler/scheduler.go create mode 100644 internal/state/replicator.go create mode 100644 internal/state/state_process.go diff --git a/config.toml b/config.toml index 1d0161d..bd7f9d5 100644 --- a/config.toml +++ b/config.toml @@ -8,6 +8,8 @@ own_registry_port = "8585" # URL of remote registry OR local file path # url_or_file = "https://demo.goharbor.io/v2/myproject/album-server" url_or_file = "https://demo.goharbor.io" +## for testing for local file +# url_or_file = "./image-list/images.json" # Default path for Zot registry config.json zotConfigPath = "./registry/config.json" diff --git a/go.mod b/go.mod index 4e8c686..dbffd9f 100644 --- a/go.mod +++ b/go.mod @@ -328,6 +328,7 @@ require ( github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.4.7 // indirect + github.com/robfig/cron/v3 v3.0.1 github.com/rubenv/sql-migrate v1.5.2 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect diff --git a/go.sum b/go.sum index 0b366eb..1f90f4e 100644 --- a/go.sum +++ b/go.sum @@ -1289,6 +1289,8 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qq github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= diff --git a/image-list/images.json b/image-list/images.json index 271496a..cd8ac06 100644 --- a/image-list/images.json +++ b/image-list/images.json @@ -1,16 +1,15 @@ { - "registryUrl": "https://demo.goharbor.io/v2/", - "repositories": [ - { - "repository": "myproject", - "images": [ - { - "name": "album-server@sha256:39879890008f12c25ea14125aa8e9ec8ef3e167f0b0ed88057e955a8fa32c430" - }, - { - "name": "album-server:busybox" - } - ] - } + "registry": "http://demo.goharbor.io/", + "artifacts": [ + { + "repository": "satellite-test-alpine/alpine", + "tag": "latest", + "hash": "sha256:9cee2b38" + }, + { + "repository": "satellite-test-postgres/postgres", + "tag": "latest", + "hash": "sha256:9cee2b38" + } ] -} \ No newline at end of file +} diff --git a/internal/config/config.go b/internal/config/config.go index 3915322..d4a9946 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -31,6 +31,7 @@ type Config struct { remote_registry_url string group_name string state_artifact_name string + state_fetch_period string } func GetLogLevel() string { @@ -141,6 +142,10 @@ func GetStateArtifactName() string { return AppConfig.state_artifact_name } +func GetStateFetchPeriod() string { + return AppConfig.state_fetch_period +} + func LoadConfig() (*Config, error) { viper.SetConfigName("config") viper.SetConfigType("toml") @@ -174,6 +179,7 @@ func LoadConfig() (*Config, error) { use_unsecure: use_unsecure, group_name: os.Getenv("GROUP_NAME"), state_artifact_name: os.Getenv("STATE_ARTIFACT_NAME"), + state_fetch_period: os.Getenv("STATE_FETCH_PERIOD"), }, nil } diff --git a/internal/replicate/replicate.go b/internal/replicate/replicate.go deleted file mode 100644 index bc323e6..0000000 --- a/internal/replicate/replicate.go +++ /dev/null @@ -1,275 +0,0 @@ -package replicate - -import ( - "context" - "encoding/json" - "fmt" - "os" - "path/filepath" - "strings" - - "container-registry.com/harbor-satellite/internal/config" - "container-registry.com/harbor-satellite/internal/state" - "container-registry.com/harbor-satellite/internal/store" - "container-registry.com/harbor-satellite/internal/utils" - "container-registry.com/harbor-satellite/logger" - "github.com/google/go-containerregistry/pkg/authn" - "github.com/google/go-containerregistry/pkg/crane" -) - -type Replicator interface { - // Replicate copies images from the source registry to the local registry. - Replicate(ctx context.Context) error - DeleteExtraImages(ctx context.Context, imgs []store.Image) error -} - -type BasicReplicator struct { - username string - password string - use_unsecure bool - zot_url string - state_reader state.StateReader -} - -type ImageInfo struct { - Name string `json:"name"` -} - -type Repository struct { - Repository string `json:"repository"` - Images []ImageInfo `json:"images"` -} - -type RegistryInfo struct { - RegistryUrl string `json:"registryUrl"` - Repositories []Repository `json:"repositories"` -} - -func NewReplicator(state_reader state.StateReader) Replicator { - return &BasicReplicator{ - username: config.GetHarborUsername(), - password: config.GetHarborPassword(), - use_unsecure: config.UseUnsecure(), - zot_url: config.GetZotURL(), - state_reader: state_reader, - } -} - -func (r *BasicReplicator) Replicate(ctx context.Context) error { - log := logger.FromContext(ctx) - auth := authn.FromConfig(authn.AuthConfig{ - Username: r.username, - Password: r.password, - }) - - options := []crane.Option{crane.WithAuth(auth)} - if r.use_unsecure { - options = append(options, crane.Insecure) - } - source_registry := r.state_reader.GetRegistryURL() - for _, artifact := range r.state_reader.GetArtifacts() { - // Extract the image name from the repository of the artifact - repo, image, err := utils.GetRepositoryAndImageNameFromArtifact(artifact.GetRepository()) - if err != nil { - log.Error().Msgf("Error getting repository and image name: %v", err) - return err - } - log.Info().Msgf("Pulling image %s from repository %s at registry %s", image, repo, source_registry) - // Pull the image at the given repository at the source registry - srcImage, err := crane.Pull(fmt.Sprintf("%s/%s/%s", source_registry, repo, image), options...) - if err != nil { - logger.FromContext(ctx).Error().Msgf("Failed to pull image: %v", err) - return err - } - - // Push the image to the local registry - err = crane.Push(srcImage, fmt.Sprintf("%s/%s", r.zot_url, image), options...) - if err != nil { - logger.FromContext(ctx).Error().Msgf("Failed to push image: %v", err) - return err - } - log.Info().Msgf("Image %s pushed successfully", image) - } - // Delete ./local-oci-layout directory - // This is required because it is a temporary directory used by crane to pull and push images to and from - // And crane does not automatically clean it - if err := os.RemoveAll("./local-oci-layout"); err != nil { - log.Error().Msgf("Failed to remove directory: %v", err) - return fmt.Errorf("failed to remove directory: %w", err) - } - return nil -} - -func (r *BasicReplicator) Rep(ctx context.Context, image string) error { - - source := getPullSource(ctx, image) - - if source != "" { - CopyImage(ctx, source) - } - return nil -} - -func stripPrefix(imageName string) string { - if idx := strings.Index(imageName, ":"); idx != -1 { - return imageName[idx+1:] - } - return imageName -} - -func (r *BasicReplicator) DeleteExtraImages(ctx context.Context, imgs []store.Image) error { - log := logger.FromContext(ctx) - zotUrl := os.Getenv("ZOT_URL") - registry := os.Getenv("REGISTRY") - repository := os.Getenv("REPOSITORY") - image := os.Getenv("IMAGE") - - localRegistry := fmt.Sprintf("%s/%s/%s/%s", zotUrl, registry, repository, image) - log.Info().Msgf("Local registry: %s", localRegistry) - - // Get the list of images from the local registry - localImages, err := crane.ListTags(localRegistry) - if err != nil { - log.Error().Msgf("failed to list tags: %v", err) - return err - } - - // Create a map for quick lookup of the provided image list - imageMap := make(map[string]struct{}) - for _, img := range imgs { - // Strip the "album-server:" prefix from the image name - strippedName := stripPrefix(img.Name) - imageMap[strippedName] = struct{}{} - } - - // Iterate over the local images and delete those not in the provided image list - for _, localImage := range localImages { - if _, exists := imageMap[localImage]; !exists { - // Image is not in the provided list, delete it - log.Info().Msgf("Deleting image: %s", localImage) - err := crane.Delete(fmt.Sprintf("%s:%s", localRegistry, localImage)) - if err != nil { - log.Error().Msgf("failed to delete image: %v", err) - return err - } - log.Info().Msgf("Image deleted: %s", localImage) - } - } - - return nil -} - -func getPullSource(ctx context.Context, image string) string { - log := logger.FromContext(ctx) - input := os.Getenv("USER_INPUT") - scheme := os.Getenv("SCHEME") - if strings.HasPrefix(scheme, "http://") || strings.HasPrefix(scheme, "https://") { - url := os.Getenv("REGISTRY") + "/" + os.Getenv("REPOSITORY") + "/" + image - return url - } else { - registryInfo, err := getFileInfo(ctx, input) - if err != nil { - log.Error().Msgf("Error getting file info: %v", err) - return "" - } - registryURL := registryInfo.RegistryUrl - registryURL = strings.TrimPrefix(registryURL, "https://") - registryURL = strings.TrimSuffix(registryURL, "/v2/") - - // TODO: Handle multiple repositories - repositoryName := registryInfo.Repositories[0].Repository - - return registryURL + "/" + repositoryName + "/" + image - } -} - -func getFileInfo(ctx context.Context, input string) (*RegistryInfo, error) { - log := logger.FromContext(ctx) - // Get the current working directory - workingDir, err := os.Getwd() - if err != nil { - log.Error().Msgf("Error getting current directory: %v", err) - return nil, err - } - - // Construct the full path by joining the working directory and the input path - fullPath := filepath.Join(workingDir, input) - - // Read the file - jsonData, err := os.ReadFile(fullPath) - if err != nil { - log.Error().Msgf("Error reading file: %v", err) - return nil, err - } - - var registryInfo RegistryInfo - err = json.Unmarshal(jsonData, ®istryInfo) - if err != nil { - log.Error().Msgf("Error unmarshalling JSON data: %v", err) - return nil, err - } - - return ®istryInfo, nil -} - -func CopyImage(ctx context.Context, imageName string) error { - log := logger.FromContext(ctx) - log.Info().Msgf("Copying image: %s", imageName) - zotUrl := os.Getenv("ZOT_URL") - if zotUrl == "" { - log.Error().Msg("ZOT_URL environment variable is not set") - return fmt.Errorf("ZOT_URL environment variable is not set") - } - - // Build the destination reference - destRef := fmt.Sprintf("%s/%s", zotUrl, imageName) - log.Info().Msgf("Destination reference: %s", destRef) - - // Get credentials from environment variables - username := os.Getenv("HARBOR_USERNAME") - password := os.Getenv("HARBOR_PASSWORD") - if username == "" || password == "" { - log.Error().Msg("HARBOR_USERNAME or HARBOR_PASSWORD environment variable is not set") - return fmt.Errorf("HARBOR_USERNAME or HARBOR_PASSWORD environment variable is not set") - } - - auth := authn.FromConfig(authn.AuthConfig{ - Username: username, - Password: password, - }) - options := []crane.Option{crane.WithAuth(auth)} - if config.UseUnsecure() { - options = append(options, crane.Insecure) - } - // Pull the image with authentication - srcImage, err := crane.Pull(imageName, options...) - if err != nil { - log.Error().Msgf("Failed to pull image: %v", err) - return fmt.Errorf("failed to pull image: %w", err) - } else { - log.Info().Msg("Image pulled successfully") - } - - // Push the image to the destination registry - push_options := []crane.Option{} - if config.UseUnsecure() { - push_options = append(push_options, crane.Insecure) - } - err = crane.Push(srcImage, destRef, push_options...) - if err != nil { - log.Error().Msgf("Failed to push image: %v", err) - return fmt.Errorf("failed to push image: %w", err) - } else { - log.Info().Msg("Image pushed successfully") - } - - // Delete ./local-oci-layout directory - // This is required because it is a temporary directory used by crane to pull and push images to and from - // And crane does not automatically clean it - if err := os.RemoveAll("./local-oci-layout"); err != nil { - log.Error().Msgf("Failed to remove directory: %v", err) - return fmt.Errorf("failed to remove directory: %w", err) - } - - return nil -} diff --git a/internal/satellite/satellite.go b/internal/satellite/satellite.go index 71c2a16..8ca2849 100644 --- a/internal/satellite/satellite.go +++ b/internal/satellite/satellite.go @@ -2,75 +2,44 @@ package satellite import ( "context" - "time" - "container-registry.com/harbor-satellite/internal/replicate" - "container-registry.com/harbor-satellite/internal/store" + "container-registry.com/harbor-satellite/internal/config" + "container-registry.com/harbor-satellite/internal/scheduler" + "container-registry.com/harbor-satellite/internal/state" + "container-registry.com/harbor-satellite/internal/utils" "container-registry.com/harbor-satellite/logger" ) type Satellite struct { - storer store.Storer - replicator replicate.Replicator + stateReader state.StateReader + stateArtifactFetcher state.StateFetcher + schedulerKey scheduler.SchedulerKey } -func NewSatellite(ctx context.Context, storer store.Storer, replicator replicate.Replicator) *Satellite { +func NewSatellite(ctx context.Context, stateArtifactFetcher state.StateFetcher, schedulerKey scheduler.SchedulerKey) *Satellite { return &Satellite{ - storer: storer, - replicator: replicator, + stateArtifactFetcher: stateArtifactFetcher, + schedulerKey: schedulerKey, } } func (s *Satellite) Run(ctx context.Context) error { log := logger.FromContext(ctx) - - // Execute the initial operation immediately without waiting for the ticker - imgs, err := s.storer.List(ctx) + log.Info().Msg("Starting Satellite") + var cronExpr string + state_fetch_period := config.GetStateFetchPeriod() + cronExpr, err := utils.FormatDuration(state_fetch_period) if err != nil { - log.Error().Err(err).Msg("Error listing images") - return err - } - if len(imgs) == 0 { - log.Info().Msg("No images to replicate") - } else { - for _, img := range imgs { - err = s.replicator.Replicate(ctx) - if err != nil { - log.Error().Err(err).Msg("Error replicating image") - return err - } - } - s.replicator.DeleteExtraImages(ctx, imgs) - } - log.Info().Msg("--------------------------------\n") - - // Temporarily set to faster tick rate for testing purposes - ticker := time.NewTicker(3 * time.Second) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - return nil - case <-ticker.C: - imgs, err := s.storer.List(ctx) - if err != nil { - log.Error().Err(err).Msg("Error listing images") - return err - } - if len(imgs) == 0 { - log.Info().Msg("No images to replicate") - } else { - for _, img := range imgs { - err = s.replicator.Replicate(ctx, img.Name) - if err != nil { - log.Error().Err(err).Msg("Error replicating image") - return err - } - } - s.replicator.DeleteExtraImages(ctx, imgs) - } - } - log.Info().Msg("--------------------------------\n") + log.Warn().Msgf("Error formatting duration in seconds: %v", err) + log.Warn().Msgf("Using default duration: %v", state.DefaultFetchAndReplicateStateTimePeriod) + cronExpr = state.DefaultFetchAndReplicateStateTimePeriod } + // Get the scheduler from the context + scheduler := ctx.Value(s.schedulerKey).(scheduler.Scheduler) + // Creating a process to fetch and replicate the state + fetchAndReplicateStateProcess := state.NewFetchAndReplicateStateProcess(scheduler.NextID(), cronExpr, s.stateArtifactFetcher) + // Add the process to the scheduler + scheduler.Schedule(&fetchAndReplicateStateProcess) + + return nil } diff --git a/internal/scheduler/process.go b/internal/scheduler/process.go new file mode 100644 index 0000000..a306bc9 --- /dev/null +++ b/internal/scheduler/process.go @@ -0,0 +1,21 @@ +package scheduler + +import "context" + +// Process represents a process that can be scheduled +type Process interface { + // Execute runs the process + Execute(ctx context.Context) error + + // GetID returns the unique GetID of the process + GetID() uint64 + + // GetName returns the name of the process + GetName() string + + // GetCronExpr returns the cron expression for the process + GetCronExpr() string + + // IsRunning returns true if the process is running + IsRunning() bool +} diff --git a/internal/scheduler/scheduler.go b/internal/scheduler/scheduler.go new file mode 100644 index 0000000..2418310 --- /dev/null +++ b/internal/scheduler/scheduler.go @@ -0,0 +1,99 @@ +package scheduler + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + + "container-registry.com/harbor-satellite/logger" + "github.com/robfig/cron/v3" +) + +type SchedulerKey string + +const BasicSchedulerKey SchedulerKey = "basic-scheduler" + +type Scheduler interface { + // GetSchedulerKey would return the key of the scheduler which is unique and for a particular scheduler and is used to get the scheduler from the context + GetSchedulerKey() SchedulerKey + // Schedule would add a process to the scheduler + Schedule(process Process) error + // Start would start the scheduler + Start() error + // Stop would stop the scheduler + Stop() error + // NextID would return the next unique ID + NextID() uint64 +} + +type BasicScheduler struct { + name SchedulerKey + cron *cron.Cron + processes map[string]Process + locks map[string]*sync.Mutex + stopped bool + counter uint64 + mu sync.Mutex + ctx context.Context +} + +func NewBasicScheduler(ctx *context.Context) Scheduler { + return &BasicScheduler{ + cron: cron.New(), + processes: make(map[string]Process), + locks: make(map[string]*sync.Mutex), + mu: sync.Mutex{}, + name: BasicSchedulerKey, + ctx: *ctx, + } +} + +func (s *BasicScheduler) GetSchedulerKey() SchedulerKey { + return s.name +} + +func (s *BasicScheduler) NextID() uint64 { + return atomic.AddUint64(&s.counter, 1) +} + +func (s *BasicScheduler) Schedule(process Process) error { + log := logger.FromContext(s.ctx) + log.Info().Msgf("Scheduling process %s", process.GetName()) + s.mu.Lock() + defer s.mu.Unlock() + for _, processes := range s.processes { + if process.GetName() == processes.GetName() { + return fmt.Errorf("process with Name %s already exists", process.GetName()) + } + } + s.processes[process.GetName()] = process + // Add the process to the scheduler + _, err := s.cron.AddFunc(process.GetCronExpr(), func() { + s.executeProcess(process) + }) + if err != nil { + return fmt.Errorf("error adding process to scheduler: %w", err) + } + log.Info().Msgf("Process %s scheduled with cron expression %s", process.GetName(), process.GetCronExpr()) + return nil +} + +func (s *BasicScheduler) Start() error { + s.cron.Start() + return nil +} + +func (s *BasicScheduler) Stop() error { + s.stopped = true + s.cron.Stop() + return nil +} + +func (s *BasicScheduler) executeProcess(process Process) error { + if s.stopped { + return fmt.Errorf("scheduler is stopped") + } + // Execute the process + return process.Execute(s.ctx) +} diff --git a/internal/server/server.go b/internal/server/server.go index 7da7785..019ed01 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -3,6 +3,7 @@ package server import ( "context" "errors" + "fmt" "net/http" "container-registry.com/harbor-satellite/internal/config" @@ -62,7 +63,11 @@ func (a *App) SetupServer(g *errgroup.Group) { }) g.Go(func() error { <-a.ctx.Done() - a.Logger.Info().Msg("Shutting down server") - return a.Shutdown(a.ctx) + a.Logger.Warn().Msg("Shutting down server") + err := a.Shutdown(a.ctx) + if err != nil { + return fmt.Errorf("error shutting down server: %w", err) + } + return fmt.Errorf("satellite shutting down") }) } diff --git a/internal/state/artifact.go b/internal/state/artifact.go index 8989307..40771a7 100644 --- a/internal/state/artifact.go +++ b/internal/state/artifact.go @@ -8,6 +8,8 @@ type ArtifactReader interface { GetTag() string // GetHash returns the hash of the artifact GetHash() string + // HasChanged returns true if the artifact has changed + HasChanged(newArtifact ArtifactReader) bool } // Artifact represents an artifact object in the registry @@ -32,3 +34,7 @@ func (a *Artifact) GetTag() string { func (a *Artifact) GetHash() string { return a.Hash } + +func (a *Artifact) HasChanged(newArtifact ArtifactReader) bool { + return a.GetHash() != newArtifact.GetHash() +} diff --git a/internal/state/replicator.go b/internal/state/replicator.go new file mode 100644 index 0000000..cd1b3fc --- /dev/null +++ b/internal/state/replicator.go @@ -0,0 +1,94 @@ +package state + +import ( + "context" + "fmt" + "os" + + "container-registry.com/harbor-satellite/internal/config" + "container-registry.com/harbor-satellite/internal/utils" + "container-registry.com/harbor-satellite/logger" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/crane" +) + +type Replicator interface { + // Replicate copies images from the source registry to the local registry. + Replicate(ctx context.Context) error +} + +type BasicReplicator struct { + username string + password string + use_unsecure bool + zot_url string + state_reader StateReader +} + +type ImageInfo struct { + Name string `json:"name"` +} + +type Repository struct { + Repository string `json:"repository"` + Images []ImageInfo `json:"images"` +} + +type RegistryInfo struct { + RegistryUrl string `json:"registryUrl"` + Repositories []Repository `json:"repositories"` +} + +func BasicNewReplicator(state_reader StateReader) Replicator { + return &BasicReplicator{ + username: config.GetHarborUsername(), + password: config.GetHarborPassword(), + use_unsecure: config.UseUnsecure(), + zot_url: config.GetZotURL(), + state_reader: state_reader, + } +} + +func (r *BasicReplicator) Replicate(ctx context.Context) error { + log := logger.FromContext(ctx) + auth := authn.FromConfig(authn.AuthConfig{ + Username: r.username, + Password: r.password, + }) + + options := []crane.Option{crane.WithAuth(auth)} + if r.use_unsecure { + options = append(options, crane.Insecure) + } + source_registry := r.state_reader.GetRegistryURL() + for _, artifact := range r.state_reader.GetArtifacts() { + // Extract the image name from the repository of the artifact + repo, image, err := utils.GetRepositoryAndImageNameFromArtifact(artifact.GetRepository()) + if err != nil { + log.Error().Msgf("Error getting repository and image name: %v", err) + return err + } + log.Info().Msgf("Pulling image %s from repository %s at registry %s", image, repo, source_registry) + // Pull the image at the given repository at the source registry + _, err = crane.Pull(fmt.Sprintf("%s/%s/%s", source_registry, repo, image), options...) + if err != nil { + logger.FromContext(ctx).Error().Msgf("Failed to pull image: %v", err) + return err + } + // Push the image to the local registry + // err = crane.Push(srcImage, fmt.Sprintf("%s/%s", r.zot_url, image), options...) + // if err != nil { + // logger.FromContext(ctx).Error().Msgf("Failed to push image: %v", err) + // return err + // } + log.Info().Msgf("Image %s pushed successfully", image) + } + // Delete ./local-oci-layout directory + // This is required because it is a temporary directory used by crane to pull and push images to and from + // And crane does not automatically clean it + if err := os.RemoveAll("./local-oci-layout"); err != nil { + log.Error().Msgf("Failed to remove directory: %v", err) + return fmt.Errorf("failed to remove directory: %w", err) + } + return nil +} diff --git a/internal/state/state.go b/internal/state/state.go index c97a598..f8d063e 100644 --- a/internal/state/state.go +++ b/internal/state/state.go @@ -23,6 +23,8 @@ type StateReader interface { GetArtifacts() []ArtifactReader // GetArtifactByRepository takes in the repository name and returns the artifact associated with it GetArtifactByRepository(repo string) (ArtifactReader, error) + // Compare the state artifact with the new state artifact + HasStateChanged(newState StateReader) bool } type State struct { @@ -65,7 +67,24 @@ func (a *State) GetArtifactByRepository(repo string) (ArtifactReader, error) { return &Artifact{}, fmt.Errorf("artifact not found in the list") } -type StateArtifactFetcher interface { +func (a *State) HasStateChanged(newState StateReader) bool { + if a.GetRegistryURL() != newState.GetRegistryURL() { + return true + } + artifacts := a.GetArtifacts() + newArtifacts := newState.GetArtifacts() + if len(artifacts) != len(newArtifacts) { + return true + } + for i, artifact := range artifacts { + if artifact.HasChanged(newArtifacts[i]) { + return true + } + } + return false +} + +type StateFetcher interface { // Fetches the state artifact from the registry FetchStateArtifact() (StateReader, error) } @@ -77,7 +96,7 @@ type URLStateFetcher struct { state_artifact_reader StateReader } -func NewURLStateFetcher() StateArtifactFetcher { +func NewURLStateFetcher() StateFetcher { url := config.GetRemoteRegistryURL() // Trim the "https://" or "http://" prefix if present if len(url) >= 8 && url[:8] == "https://" { @@ -95,7 +114,34 @@ func NewURLStateFetcher() StateArtifactFetcher { } type FileStateArtifactFetcher struct { - filePath string + filePath string + group_name string + state_artifact_name string + state_artifact_reader StateReader +} + +func NewFileStateFetcher() StateFetcher { + filePath := config.GetInput() + state_artifact_reader := NewState() + return &FileStateArtifactFetcher{ + filePath: filePath, + group_name: config.GetGroupName(), + state_artifact_name: config.GetStateArtifactName(), + state_artifact_reader: state_artifact_reader, + } +} + +func (f *FileStateArtifactFetcher) FetchStateArtifact() (StateReader, error) { + /// Read the state artifact file from the file path + content, err := os.ReadFile(f.filePath) + if err != nil { + return nil, fmt.Errorf("failed to read the state artifact file: %v", err) + } + state_reader, err := FromJSON(content, f.state_artifact_reader.(*State)) + if err != nil { + return nil, fmt.Errorf("failed to parse the state artifact file: %v", err) + } + return state_reader, nil } func (f *URLStateFetcher) FetchStateArtifact() (StateReader, error) { diff --git a/internal/state/state_process.go b/internal/state/state_process.go new file mode 100644 index 0000000..4c348e8 --- /dev/null +++ b/internal/state/state_process.go @@ -0,0 +1,84 @@ +package state + +import ( + "context" + "fmt" + + "container-registry.com/harbor-satellite/logger" +) + +const FetchAndReplicateStateProcessName string = "fetch-replicate-state-process" + +const DefaultFetchAndReplicateStateTimePeriod string = "00h00m05s" + +type FetchAndReplicateStateProcess struct { + id uint64 + name string + stateArtifactFetcher StateFetcher + cronExpr string + isRunning bool + stateReader StateReader +} + +func NewFetchAndReplicateStateProcess(id uint64, cronExpr string, stateFetcher StateFetcher) FetchAndReplicateStateProcess { + return FetchAndReplicateStateProcess{ + id: id, + name: FetchAndReplicateStateProcessName, + cronExpr: cronExpr, + isRunning: false, + stateArtifactFetcher: stateFetcher, + } +} + +func (f *FetchAndReplicateStateProcess) Execute(ctx context.Context) error { + log := logger.FromContext(ctx) + if f.IsRunning() { + log.Warn().Msg("Process is already running") + return fmt.Errorf("process %s is already running", f.GetName()) + } + log.Info().Msg("Starting process to fetch and replicate state") + f.isRunning = true + defer func() { + f.isRunning = false + }() + + newStateFetched, err := f.stateArtifactFetcher.FetchStateArtifact() + if err != nil { + log.Error().Err(err).Msg("Error fetching state artifact") + return err + } + if !f.HasStateChanged(newStateFetched) { + log.Info().Msg("State has not changed") + return nil + } + + replicator := BasicNewReplicator(newStateFetched) + if err := replicator.Replicate(ctx); err != nil { + log.Error().Err(err).Msg("Error replicating state") + return err + } + return nil +} + +func (f *FetchAndReplicateStateProcess) HasStateChanged(newState StateReader) bool { + if f.stateReader == nil { + return true + } + return f.stateReader.HasStateChanged(newState) +} + +func (f *FetchAndReplicateStateProcess) GetID() uint64 { + return f.id +} + +func (f *FetchAndReplicateStateProcess) GetName() string { + return f.name +} + +func (f *FetchAndReplicateStateProcess) GetCronExpr() string { + return fmt.Sprintf("@every %s", f.cronExpr) +} + +func (f *FetchAndReplicateStateProcess) IsRunning() bool { + return f.isRunning +} diff --git a/internal/utils/utils.go b/internal/utils/utils.go index 89cbf19..9e0323d 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -1,7 +1,6 @@ package utils import ( - "encoding/json" "errors" "fmt" "net" @@ -12,7 +11,6 @@ import ( "strings" "container-registry.com/harbor-satellite/internal/config" - "container-registry.com/harbor-satellite/internal/images" "container-registry.com/harbor-satellite/registry" ) @@ -80,70 +78,41 @@ func HasInvalidPathChars(input string) bool { return strings.ContainsAny(input, "\\:*?\"<>|") } -// ParseImagesJsonFile parses the images.json file and decodes it into the ImageList struct -func ParseImagesJsonFile(absPath string, imagesList *images.ImageList) error { - file, err := os.Open(absPath) - if err != nil { - return err - } - defer file.Close() - if err := json.NewDecoder(file).Decode(imagesList); err != nil { - return err +func GetRepositoryAndImageNameFromArtifact(repository string) (string, string, error) { + parts := strings.Split(repository, "/") + if len(parts) < 2 { + return "", "", fmt.Errorf("invalid repository format") } - return nil + repo := parts[0] + image := parts[1] + return repo, image, nil } -// Set registry environment variables -func SetRegistryEnvVars(imageList images.ImageList) error { - if !IsValidURL(imageList.RegistryURL) { - return fmt.Errorf("invalid registry url format in images.json") - } - registryURL := imageList.RegistryURL - registryParts := strings.Split(registryURL, "/") - if len(registryParts) < 3 { - return fmt.Errorf("invalid registryUrl format in images.json") +func FormatDuration(input string) (string , error) { + seconds, err := strconv.Atoi(input) // Convert input string to an integer + if err != nil { + return "", errors.New("invalid input: not a valid number") } - - os.Setenv("REGISTRY", registryParts[2]) - config.SetRegistry(registryParts[2]) - - if len(imageList.Repositories) > 0 { - os.Setenv("REPOSITORY", imageList.Repositories[0].Repository) - config.SetRepository(imageList.Repositories[0].Repository) - } else { - return fmt.Errorf("no repositories found in images.json") + if seconds < 0 { + return "", errors.New("invalid input: seconds cannot be negative") } - return nil -} + hours := seconds / 3600 + minutes := (seconds % 3600) / 60 + secondsRemaining := seconds % 60 -// SetUrlConfig sets the URL configuration for the input URL and sets the environment variables -func SetUrlConfig(input string) { - os.Setenv("USER_INPUT", input) - config.SetUserInput(input) - parts := strings.SplitN(input, "://", 2) - scheme := parts[0] + "://" - os.Setenv("SCHEME", scheme) - config.SetScheme(scheme) - registryAndPath := parts[1] - registryParts := strings.Split(registryAndPath, "/") - os.Setenv("REGISTRY", registryParts[0]) - config.SetRegistry(registryParts[0]) - os.Setenv("API_VERSION", registryParts[1]) - config.SetAPIVersion(registryParts[1]) - os.Setenv("REPOSITORY", registryParts[2]) - config.SetRepository(registryParts[2]) - os.Setenv("IMAGE", registryParts[3]) - config.SetImage(registryParts[3]) -} + var result string -func GetRepositoryAndImageNameFromArtifact(repository string) (string, string, error){ - parts := strings.Split(repository, "/") - if len(parts) < 2 { - return "", "", fmt.Errorf("invalid repository format") + if hours > 0 { + result += strconv.Itoa(hours) + "h" } - repo := parts[0] - image := parts[1] - return repo, image, nil + if minutes > 0 { + result += strconv.Itoa(minutes) + "m" + } + if secondsRemaining > 0 || result == "" { + result += strconv.Itoa(secondsRemaining) + "s" + } + + return result, nil } diff --git a/logger/logger.go b/logger/logger.go index 78664dc..69dfadb 100644 --- a/logger/logger.go +++ b/logger/logger.go @@ -2,7 +2,9 @@ package logger import ( "context" + "fmt" "os" + "strings" "github.com/rs/zerolog" ) @@ -11,27 +13,46 @@ type contextKey string const loggerKey contextKey = "logger" -// AddLoggerToContext creates a new context with a zerolog logger for stdout adn stderr and sets the global log level. +// AddLoggerToContext creates a new context with a zerolog logger for stdout and stderr and sets the global log level. func AddLoggerToContext(ctx context.Context, logLevel string) context.Context { // Set log level to configured value - switch logLevel { - case "debug": - zerolog.SetGlobalLevel(zerolog.DebugLevel) - case "info": - zerolog.SetGlobalLevel(zerolog.InfoLevel) - case "warn": - zerolog.SetGlobalLevel(zerolog.WarnLevel) - case "error": - zerolog.SetGlobalLevel(zerolog.ErrorLevel) - case "fatal": - zerolog.SetGlobalLevel(zerolog.FatalLevel) - case "panic": - zerolog.SetGlobalLevel(zerolog.PanicLevel) - default: - zerolog.SetGlobalLevel(zerolog.InfoLevel) + level := getLogLevel(logLevel) + zerolog.SetGlobalLevel(level) + + // Create a custom console writer + output := zerolog.ConsoleWriter{Out: os.Stderr, TimeFormat: "2006-01-02 15:04:05"} + + // Customize the output for each log level + output.FormatLevel = func(i interface{}) string { + var l string + if ll, ok := i.(string); ok { + switch ll { + case "debug": + l = colorize(ll, 36) // cyan + case "info": + l = colorize(ll, 34) // blue + case "warn": + l = colorize(ll, 33) // yellow + case "error": + l = colorize(ll, 31) // red + case "fatal": + l = colorize(ll, 35) // magenta + case "panic": + l = colorize(ll, 41) // white on red background + default: + l = colorize(ll, 37) // white + } + } else { + if i == nil { + l = colorize("???", 37) // white + } else { + l = strings.ToUpper(fmt.Sprintf("%s", i))[0:3] + } + } + return fmt.Sprintf("| %s |", l) } - logger := zerolog.New(os.Stderr).With().Timestamp().Logger() + logger := zerolog.New(output).With().Timestamp().Logger() ctx = context.WithValue(ctx, loggerKey, &logger) return ctx @@ -48,3 +69,28 @@ func FromContext(ctx context.Context) *zerolog.Logger { } return logger } + +// Helper function to get the log level +func getLogLevel(logLevel string) zerolog.Level { + switch logLevel { + case "debug": + return zerolog.DebugLevel + case "info": + return zerolog.InfoLevel + case "warn": + return zerolog.WarnLevel + case "error": + return zerolog.ErrorLevel + case "fatal": + return zerolog.FatalLevel + case "panic": + return zerolog.PanicLevel + default: + return zerolog.InfoLevel + } +} + +// Helper function to colorize text +func colorize(s string, color int) string { + return fmt.Sprintf("\x1b[%dm%s\x1b[0m", color, s) +} diff --git a/main.go b/main.go index 7fb3af1..95218e2 100644 --- a/main.go +++ b/main.go @@ -8,12 +8,10 @@ import ( "syscall" "container-registry.com/harbor-satellite/internal/config" - "container-registry.com/harbor-satellite/internal/images" - "container-registry.com/harbor-satellite/internal/replicate" "container-registry.com/harbor-satellite/internal/satellite" + "container-registry.com/harbor-satellite/internal/scheduler" "container-registry.com/harbor-satellite/internal/server" "container-registry.com/harbor-satellite/internal/state" - "container-registry.com/harbor-satellite/internal/store" "container-registry.com/harbor-satellite/internal/utils" "container-registry.com/harbor-satellite/logger" "golang.org/x/sync/errgroup" @@ -40,7 +38,6 @@ func run() error { g, ctx := errgroup.WithContext(ctx) ctx = logger.AddLoggerToContext(ctx, config.GetLogLevel()) log := logger.FromContext(ctx) - log.Info().Msg("Satellite starting") // Set up router and app app := setupServerApp(ctx, log) @@ -51,22 +48,26 @@ func run() error { if err := handleRegistrySetup(g, log, cancel); err != nil { return err } - - // Process Input (file or URL) - fetcher, err := processInput(ctx, log) + scheduler := scheduler.NewBasicScheduler(&ctx) + ctx = context.WithValue(ctx, scheduler.GetSchedulerKey(), scheduler) + err := scheduler.Start() if err != nil { + log.Error().Err(err).Msg("Error starting scheduler") return err } + // Process Input (file or URL) + stateArtifactFetcher, err := processInput(ctx, log) + if err != nil || stateArtifactFetcher == nil { + return fmt.Errorf("error processing input: %w", err) + } - ctx, storer := store.NewInMemoryStore(ctx, fetcher) - replicator := replicate.NewReplicator() - satelliteService := satellite.NewSatellite(ctx, storer, replicator) + satelliteService := satellite.NewSatellite(ctx, stateArtifactFetcher, scheduler.GetSchedulerKey()) g.Go(func() error { return satelliteService.Run(ctx) }) - log.Info().Msg("Satellite running") + log.Info().Msg("Startup complete 🚀") return g.Wait() } @@ -118,28 +119,39 @@ func handleRegistrySetup(g *errgroup.Group, log *zerolog.Logger, cancel context. return nil } -func processInput(ctx context.Context, log *zerolog.Logger) (store.ImageFetcher, error) { +func processInput(ctx context.Context, log *zerolog.Logger) (state.StateFetcher, error) { input := config.GetInput() - if !utils.IsValidURL(input) { - log.Info().Msg("Input is not a valid URL, checking if it is a file path") - if err := validateFilePath(config.GetInput(), log); err != nil { - return nil, err - } - return setupFileFetcher(ctx, log) + + if utils.IsValidURL(input) { + return processURLInput(input, log) } + log.Info().Msg("Input is not a valid URL, checking if it is a file path") + if err := validateFilePath(input, log); err != nil { + return nil, err + } + + return processFileInput(log) +} + +func processURLInput(input string, log *zerolog.Logger) (state.StateFetcher, error) { log.Info().Msg("Input is a valid URL") config.SetRemoteRegistryURL(input) - fmt.Println(config.GetRemoteRegistryURL()) - state_artifact_fetcher := state.NewURLStateFetcher() - _, err := state_artifact_fetcher.FetchStateArtifact() + + stateArtifactFetcher := state.NewURLStateFetcher() + + return stateArtifactFetcher, nil +} + +func processFileInput(log *zerolog.Logger) (state.StateFetcher, error) { + stateArtifactFetcher := state.NewFileStateFetcher() + stateReader, err := stateArtifactFetcher.FetchStateArtifact() if err != nil { - log.Error().Err(err).Msg("Error fetching state artifact") + log.Error().Err(err).Msg("Error fetching state artifact from file") return nil, err } - fetcher := store.RemoteImageListFetcher(ctx, input) - // utils.SetUrlConfig(input) - return fetcher, nil + config.SetRemoteRegistryURL(stateReader.GetRegistryURL()) + return stateArtifactFetcher, nil } func validateFilePath(path string, log *zerolog.Logger) error { @@ -153,17 +165,3 @@ func validateFilePath(path string, log *zerolog.Logger) error { } return nil } - -func setupFileFetcher(ctx context.Context, log *zerolog.Logger) (store.ImageFetcher, error) { - fetcher := store.FileImageListFetcher(ctx, config.GetInput()) - var imagesList images.ImageList - if err := utils.ParseImagesJsonFile(config.GetInput(), &imagesList); err != nil { - log.Error().Err(err).Msg("Error parsing images.json file") - return nil, err - } - if err := utils.SetRegistryEnvVars(imagesList); err != nil { - log.Error().Err(err).Msg("Error setting registry environment variables") - return nil, err - } - return fetcher, nil -} From fe299f85694f99c9fbd9c7462d3ba6d5793d4bf2 Mon Sep 17 00:00:00 2001 From: Mehul-Kumar-27 Date: Sun, 29 Sep 2024 22:33:05 +0530 Subject: [PATCH 04/36] adding simple notifier to fetch state process --- internal/notifier/email_notifier.go | 1 + internal/notifier/notifier.go | 28 ++++++++++++++++++++++++++++ internal/satellite/satellite.go | 5 ++++- internal/state/state_process.go | 8 +++++++- 4 files changed, 40 insertions(+), 2 deletions(-) create mode 100644 internal/notifier/email_notifier.go create mode 100644 internal/notifier/notifier.go diff --git a/internal/notifier/email_notifier.go b/internal/notifier/email_notifier.go new file mode 100644 index 0000000..ed45f23 --- /dev/null +++ b/internal/notifier/email_notifier.go @@ -0,0 +1 @@ +package notifier diff --git a/internal/notifier/notifier.go b/internal/notifier/notifier.go new file mode 100644 index 0000000..cf4b2ff --- /dev/null +++ b/internal/notifier/notifier.go @@ -0,0 +1,28 @@ +package notifier + +import ( + "context" + + "container-registry.com/harbor-satellite/logger" +) + +type Notifier interface { + // Notify sends a notification + Notify() error +} + +type SimpleNotifier struct{ + ctx context.Context +} + +func NewSimpleNotifier(ctx context.Context) Notifier { + return &SimpleNotifier{ + ctx: ctx, + } +} + +func (n *SimpleNotifier) Notify() error { + log := logger.FromContext(n.ctx) + log.Info().Msg("This is a simple notifier") + return nil +} diff --git a/internal/satellite/satellite.go b/internal/satellite/satellite.go index 8ca2849..7601a6b 100644 --- a/internal/satellite/satellite.go +++ b/internal/satellite/satellite.go @@ -4,6 +4,7 @@ import ( "context" "container-registry.com/harbor-satellite/internal/config" + "container-registry.com/harbor-satellite/internal/notifier" "container-registry.com/harbor-satellite/internal/scheduler" "container-registry.com/harbor-satellite/internal/state" "container-registry.com/harbor-satellite/internal/utils" @@ -36,8 +37,10 @@ func (s *Satellite) Run(ctx context.Context) error { } // Get the scheduler from the context scheduler := ctx.Value(s.schedulerKey).(scheduler.Scheduler) + // Create a simple notifier and add it to the process + notifier := notifier.NewSimpleNotifier(ctx) // Creating a process to fetch and replicate the state - fetchAndReplicateStateProcess := state.NewFetchAndReplicateStateProcess(scheduler.NextID(), cronExpr, s.stateArtifactFetcher) + fetchAndReplicateStateProcess := state.NewFetchAndReplicateStateProcess(scheduler.NextID(), cronExpr, s.stateArtifactFetcher, notifier) // Add the process to the scheduler scheduler.Schedule(&fetchAndReplicateStateProcess) diff --git a/internal/state/state_process.go b/internal/state/state_process.go index 4c348e8..e9c4c44 100644 --- a/internal/state/state_process.go +++ b/internal/state/state_process.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + "container-registry.com/harbor-satellite/internal/notifier" "container-registry.com/harbor-satellite/logger" ) @@ -18,15 +19,17 @@ type FetchAndReplicateStateProcess struct { cronExpr string isRunning bool stateReader StateReader + notifier notifier.Notifier } -func NewFetchAndReplicateStateProcess(id uint64, cronExpr string, stateFetcher StateFetcher) FetchAndReplicateStateProcess { +func NewFetchAndReplicateStateProcess(id uint64, cronExpr string, stateFetcher StateFetcher, notifier notifier.Notifier) FetchAndReplicateStateProcess { return FetchAndReplicateStateProcess{ id: id, name: FetchAndReplicateStateProcessName, cronExpr: cronExpr, isRunning: false, stateArtifactFetcher: stateFetcher, + notifier: notifier, } } @@ -51,6 +54,9 @@ func (f *FetchAndReplicateStateProcess) Execute(ctx context.Context) error { log.Info().Msg("State has not changed") return nil } + if err := f.notifier.Notify(); err != nil { + log.Error().Err(err).Msg("Error sending notification") + } replicator := BasicNewReplicator(newStateFetched) if err := replicator.Replicate(ctx); err != nil { From b6b5dea9388702965bdc0061781b4561a747b2b5 Mon Sep 17 00:00:00 2001 From: Mehul-Kumar-27 Date: Sun, 29 Sep 2024 22:45:04 +0530 Subject: [PATCH 05/36] added description to the scheduler --- internal/scheduler/scheduler.go | 10 +++++++++- internal/state/replicator.go | 12 ++++++------ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/internal/scheduler/scheduler.go b/internal/scheduler/scheduler.go index 2418310..552f7a0 100644 --- a/internal/scheduler/scheduler.go +++ b/internal/scheduler/scheduler.go @@ -28,13 +28,21 @@ type Scheduler interface { } type BasicScheduler struct { + // name is the key of the scheduler name SchedulerKey + // cron is the cron scheduler cron *cron.Cron + // processes is a map of processes which are attached to the scheduler processes map[string]Process + // locks is a map of locks for each process which is used to schedule if the process are interdependent locks map[string]*sync.Mutex + // stopped is a flag to check if the scheduler is stopped stopped bool + // counter is the counter for the unique ID of the process counter uint64 + // mu is the mutex for the scheduler mu sync.Mutex + // ctx is the context of the scheduler ctx context.Context } @@ -67,7 +75,6 @@ func (s *BasicScheduler) Schedule(process Process) error { return fmt.Errorf("process with Name %s already exists", process.GetName()) } } - s.processes[process.GetName()] = process // Add the process to the scheduler _, err := s.cron.AddFunc(process.GetCronExpr(), func() { s.executeProcess(process) @@ -75,6 +82,7 @@ func (s *BasicScheduler) Schedule(process Process) error { if err != nil { return fmt.Errorf("error adding process to scheduler: %w", err) } + s.processes[process.GetName()] = process log.Info().Msgf("Process %s scheduled with cron expression %s", process.GetName(), process.GetCronExpr()) return nil } diff --git a/internal/state/replicator.go b/internal/state/replicator.go index cd1b3fc..a0d33a1 100644 --- a/internal/state/replicator.go +++ b/internal/state/replicator.go @@ -70,17 +70,17 @@ func (r *BasicReplicator) Replicate(ctx context.Context) error { } log.Info().Msgf("Pulling image %s from repository %s at registry %s", image, repo, source_registry) // Pull the image at the given repository at the source registry - _, err = crane.Pull(fmt.Sprintf("%s/%s/%s", source_registry, repo, image), options...) + srcImage, err := crane.Pull(fmt.Sprintf("%s/%s/%s", source_registry, repo, image), options...) if err != nil { logger.FromContext(ctx).Error().Msgf("Failed to pull image: %v", err) return err } // Push the image to the local registry - // err = crane.Push(srcImage, fmt.Sprintf("%s/%s", r.zot_url, image), options...) - // if err != nil { - // logger.FromContext(ctx).Error().Msgf("Failed to push image: %v", err) - // return err - // } + err = crane.Push(srcImage, fmt.Sprintf("%s/%s", r.zot_url, image), options...) + if err != nil { + logger.FromContext(ctx).Error().Msgf("Failed to push image: %v", err) + return err + } log.Info().Msgf("Image %s pushed successfully", image) } // Delete ./local-oci-layout directory From 91222d40b51b152fd0c4f2b8dd3aed37205797d7 Mon Sep 17 00:00:00 2001 From: Mehul-Kumar-27 Date: Mon, 30 Sep 2024 03:02:55 +0530 Subject: [PATCH 06/36] coderabbit fixes and changes to fetcher and schedulers --- internal/satellite/satellite.go | 2 +- internal/scheduler/scheduler.go | 10 +- internal/state/artifact.go | 8 +- internal/state/fetcher.go | 158 ++++++++++++++++++++++++++ internal/state/replicator.go | 53 ++++----- internal/state/state.go | 189 +++----------------------------- internal/state/state_process.go | 15 ++- internal/utils/utils.go | 2 +- logger/logger.go | 6 +- main.go | 2 +- 10 files changed, 221 insertions(+), 224 deletions(-) create mode 100644 internal/state/fetcher.go diff --git a/internal/satellite/satellite.go b/internal/satellite/satellite.go index 7601a6b..ef6bdf1 100644 --- a/internal/satellite/satellite.go +++ b/internal/satellite/satellite.go @@ -42,7 +42,7 @@ func (s *Satellite) Run(ctx context.Context) error { // Creating a process to fetch and replicate the state fetchAndReplicateStateProcess := state.NewFetchAndReplicateStateProcess(scheduler.NextID(), cronExpr, s.stateArtifactFetcher, notifier) // Add the process to the scheduler - scheduler.Schedule(&fetchAndReplicateStateProcess) + scheduler.Schedule(fetchAndReplicateStateProcess) return nil } diff --git a/internal/scheduler/scheduler.go b/internal/scheduler/scheduler.go index 552f7a0..f12c20e 100644 --- a/internal/scheduler/scheduler.go +++ b/internal/scheduler/scheduler.go @@ -46,14 +46,14 @@ type BasicScheduler struct { ctx context.Context } -func NewBasicScheduler(ctx *context.Context) Scheduler { +func NewBasicScheduler(ctx context.Context) Scheduler { return &BasicScheduler{ cron: cron.New(), processes: make(map[string]Process), locks: make(map[string]*sync.Mutex), mu: sync.Mutex{}, name: BasicSchedulerKey, - ctx: *ctx, + ctx: ctx, } } @@ -70,10 +70,8 @@ func (s *BasicScheduler) Schedule(process Process) error { log.Info().Msgf("Scheduling process %s", process.GetName()) s.mu.Lock() defer s.mu.Unlock() - for _, processes := range s.processes { - if process.GetName() == processes.GetName() { - return fmt.Errorf("process with Name %s already exists", process.GetName()) - } + if _, exists := s.processes[process.GetName()]; exists { + return fmt.Errorf("process %s already exists", process.GetName()) } // Add the process to the scheduler _, err := s.cron.AddFunc(process.GetCronExpr(), func() { diff --git a/internal/state/artifact.go b/internal/state/artifact.go index 40771a7..541db4d 100644 --- a/internal/state/artifact.go +++ b/internal/state/artifact.go @@ -19,8 +19,12 @@ type Artifact struct { Hash string `json:"hash"` } -func NewArtifact() ArtifactReader { - return &Artifact{} +func NewArtifact(repository, tag, hash string) ArtifactReader { + return &Artifact{ + Repository: repository, + Tag: tag, + Hash: hash, + } } func (a *Artifact) GetRepository() string { diff --git a/internal/state/fetcher.go b/internal/state/fetcher.go new file mode 100644 index 0000000..60a22f9 --- /dev/null +++ b/internal/state/fetcher.go @@ -0,0 +1,158 @@ +package state + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + "container-registry.com/harbor-satellite/internal/config" + "oras.land/oras-go/v2" + "oras.land/oras-go/v2/content/file" + "oras.land/oras-go/v2/registry/remote" + "oras.land/oras-go/v2/registry/remote/auth" + "oras.land/oras-go/v2/registry/remote/retry" +) + + +type StateFetcher interface { + // Fetches the state artifact from the registry + FetchStateArtifact() (StateReader, error) +} + +type URLStateFetcher struct { + url string + group_name string + state_artifact_name string + state_artifact_reader StateReader +} + +func NewURLStateFetcher() StateFetcher { + url := config.GetRemoteRegistryURL() + // Trim the "https://" or "http://" prefix if present + url = strings.TrimPrefix(url, "https://") + url = strings.TrimPrefix(url, "http://") + state_artifact_reader := NewState() + return &URLStateFetcher{ + url: url, + group_name: config.GetGroupName(), + state_artifact_name: config.GetStateArtifactName(), + state_artifact_reader: state_artifact_reader, + } +} + +type FileStateArtifactFetcher struct { + filePath string + group_name string + state_artifact_name string + state_artifact_reader StateReader +} + +func NewFileStateFetcher() StateFetcher { + filePath := config.GetInput() + state_artifact_reader := NewState() + return &FileStateArtifactFetcher{ + filePath: filePath, + group_name: config.GetGroupName(), + state_artifact_name: config.GetStateArtifactName(), + state_artifact_reader: state_artifact_reader, + } +} + +func (f *FileStateArtifactFetcher) FetchStateArtifact() (StateReader, error) { + /// Read the state artifact file from the file path + content, err := os.ReadFile(f.filePath) + if err != nil { + return nil, fmt.Errorf("failed to read the state artifact file: %v", err) + } + state_reader, err := FromJSON(content, f.state_artifact_reader) + if err != nil { + return nil, fmt.Errorf("failed to parse the state artifact file: %v", err) + } + return state_reader, nil +} + +func (f *URLStateFetcher) FetchStateArtifact() (StateReader, error) { + cwd, err := os.Getwd() + if err != nil { + return nil, fmt.Errorf("failed to get current working directory: %v", err) + } + // Creating a file store in the current working directory will be deleted later after reading the state artifact + fs, err := file.New(fmt.Sprintf("%s/state-artifact", cwd)) + if err != nil { + return nil, fmt.Errorf("failed to create file store: %v", err) + } + defer fs.Close() + + ctx := context.Background() + + repo, err := remote.NewRepository(fmt.Sprintf("%s/%s/%s", f.url, f.group_name, f.state_artifact_name)) + if err != nil { + return nil, fmt.Errorf("failed to create remote repository: %v", err) + } + + // Setting up the authentication for the remote registry + repo.Client = &auth.Client{ + Client: retry.DefaultClient, + Cache: auth.NewCache(), + Credential: auth.StaticCredential( + f.url, + auth.Credential{ + Username: config.GetHarborUsername(), + Password: config.GetHarborPassword(), + }, + ), + } + // Copy from the remote repository to the file store + tag := "latest" + _, err = oras.Copy(ctx, repo, tag, fs, tag, oras.DefaultCopyOptions) + if err != nil { + return nil, fmt.Errorf("failed to copy from remote repository to file store: %v", err) + } + stateArtifactDir := filepath.Join(cwd, "state-artifact") + + var state_reader StateReader + // Find the state artifact file in the state-artifact directory that is created temporarily + err = filepath.Walk(stateArtifactDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if filepath.Ext(info.Name()) == ".json" { + content, err := os.ReadFile(path) + if err != nil { + return err + } + state_reader, err = FromJSON(content, f.state_artifact_reader) + if err != nil { + return fmt.Errorf("failed to parse the state artifact file: %v", err) + } + return nil + } + return nil + }) + + if err != nil { + return nil, fmt.Errorf("failed to read the state artifact file: %v", err) + } + // Clean up everything inside the state-artifact folder + err = os.RemoveAll(stateArtifactDir) + if err != nil { + return nil, fmt.Errorf("failed to remove state-artifact directory: %v", err) + } + return state_reader, nil +} + +// FromJSON parses the input JSON data into a StateArtifactReader +func FromJSON(data []byte, reg StateReader) (StateReader, error) { + if err := json.Unmarshal(data, ®); err != nil { + fmt.Print("Error in unmarshalling") + return nil, err + } + // Validation + if reg.GetRegistryURL()== "" { + return nil, fmt.Errorf("registry URL is required") + } + return reg, nil +} diff --git a/internal/state/replicator.go b/internal/state/replicator.go index a0d33a1..6dec2f0 100644 --- a/internal/state/replicator.go +++ b/internal/state/replicator.go @@ -18,34 +18,20 @@ type Replicator interface { } type BasicReplicator struct { - username string - password string - use_unsecure bool - zot_url string - state_reader StateReader + username string + password string + useUnsecure bool + zotURL string + stateReader StateReader } -type ImageInfo struct { - Name string `json:"name"` -} - -type Repository struct { - Repository string `json:"repository"` - Images []ImageInfo `json:"images"` -} - -type RegistryInfo struct { - RegistryUrl string `json:"registryUrl"` - Repositories []Repository `json:"repositories"` -} - -func BasicNewReplicator(state_reader StateReader) Replicator { +func NewBasicReplicator(state_reader StateReader) Replicator { return &BasicReplicator{ - username: config.GetHarborUsername(), - password: config.GetHarborPassword(), - use_unsecure: config.UseUnsecure(), - zot_url: config.GetZotURL(), - state_reader: state_reader, + username: config.GetHarborUsername(), + password: config.GetHarborPassword(), + useUnsecure: config.UseUnsecure(), + zotURL: config.GetZotURL(), + stateReader: state_reader, } } @@ -57,28 +43,29 @@ func (r *BasicReplicator) Replicate(ctx context.Context) error { }) options := []crane.Option{crane.WithAuth(auth)} - if r.use_unsecure { + if r.useUnsecure { options = append(options, crane.Insecure) } - source_registry := r.state_reader.GetRegistryURL() - for _, artifact := range r.state_reader.GetArtifacts() { + sourceRegistry := r.stateReader.GetRegistryURL() + + for _, artifact := range r.stateReader.GetArtifacts() { // Extract the image name from the repository of the artifact repo, image, err := utils.GetRepositoryAndImageNameFromArtifact(artifact.GetRepository()) if err != nil { log.Error().Msgf("Error getting repository and image name: %v", err) return err } - log.Info().Msgf("Pulling image %s from repository %s at registry %s", image, repo, source_registry) + log.Info().Msgf("Pulling image %s from repository %s at registry %s", image, repo, sourceRegistry) // Pull the image at the given repository at the source registry - srcImage, err := crane.Pull(fmt.Sprintf("%s/%s/%s", source_registry, repo, image), options...) + srcImage, err := crane.Pull(fmt.Sprintf("%s/%s/%s", sourceRegistry, repo, image), options...) if err != nil { - logger.FromContext(ctx).Error().Msgf("Failed to pull image: %v", err) + log.Error().Msgf("Failed to pull image: %v", err) return err } // Push the image to the local registry - err = crane.Push(srcImage, fmt.Sprintf("%s/%s", r.zot_url, image), options...) + err = crane.Push(srcImage, fmt.Sprintf("%s/%s", r.zotURL, image), options...) if err != nil { - logger.FromContext(ctx).Error().Msgf("Failed to push image: %v", err) + log.Error().Msgf("Failed to push image: %v", err) return err } log.Info().Msgf("Image %s pushed successfully", image) diff --git a/internal/state/state.go b/internal/state/state.go index f8d063e..3d9bdeb 100644 --- a/internal/state/state.go +++ b/internal/state/state.go @@ -1,25 +1,15 @@ package state import ( - "context" - "encoding/json" "fmt" - "os" - "path/filepath" - - "container-registry.com/harbor-satellite/internal/config" - "oras.land/oras-go/v2" - "oras.land/oras-go/v2/content/file" - "oras.land/oras-go/v2/registry/remote" - "oras.land/oras-go/v2/registry/remote/auth" - "oras.land/oras-go/v2/registry/remote/retry" + "strings" ) // Registry defines an interface for registry operations type StateReader interface { // GetRegistryURL returns the URL of the registry after removing the "https://" or "http://" prefix if present and the trailing "/" GetRegistryURL() string - // GetRegistryType returns the list of artifacts that needs to be pulled + // GetArtifacts returns the list of artifacts that needs to be pulled GetArtifacts() []ArtifactReader // GetArtifactByRepository takes in the repository name and returns the artifact associated with it GetArtifactByRepository(repo string) (ArtifactReader, error) @@ -39,32 +29,27 @@ func NewState() StateReader { func (a *State) GetRegistryURL() string { registry := a.Registry - if len(registry) >= 8 && registry[:8] == "https://" { - registry = registry[8:] - } else if len(registry) >= 7 && registry[:7] == "http://" { - registry = registry[7:] - } - if len(registry) > 0 && registry[len(registry)-1] == '/' { - registry = registry[:len(registry)-1] - } + registry = strings.TrimPrefix(registry, "https://") + registry = strings.TrimPrefix(registry, "http://") + registry = strings.TrimSuffix(registry, "/") return registry } func (a *State) GetArtifacts() []ArtifactReader { - var artifact_readers []ArtifactReader - for _, artifact := range a.Artifacts { - artifact_readers = append(artifact_readers, &artifact) + var artifacts_reader []ArtifactReader + for i := range a.Artifacts { + artifacts_reader = append(artifacts_reader, &a.Artifacts[i]) } - return artifact_readers + return artifacts_reader } func (a *State) GetArtifactByRepository(repo string) (ArtifactReader, error) { - for _, artifact := range a.Artifacts { - if artifact.GetRepository() == repo { - return &artifact, nil + for i := range a.Artifacts { + if a.Artifacts[i].GetRepository() == repo { + return &a.Artifacts[i], nil } } - return &Artifact{}, fmt.Errorf("artifact not found in the list") + return nil, fmt.Errorf("artifact not found in the list") } func (a *State) HasStateChanged(newState StateReader) bool { @@ -83,151 +68,3 @@ func (a *State) HasStateChanged(newState StateReader) bool { } return false } - -type StateFetcher interface { - // Fetches the state artifact from the registry - FetchStateArtifact() (StateReader, error) -} - -type URLStateFetcher struct { - url string - group_name string - state_artifact_name string - state_artifact_reader StateReader -} - -func NewURLStateFetcher() StateFetcher { - url := config.GetRemoteRegistryURL() - // Trim the "https://" or "http://" prefix if present - if len(url) >= 8 && url[:8] == "https://" { - url = url[8:] - } else if len(url) >= 7 && url[:7] == "http://" { - url = url[7:] - } - state_artifact_reader := NewState() - return &URLStateFetcher{ - url: url, - group_name: config.GetGroupName(), - state_artifact_name: config.GetStateArtifactName(), - state_artifact_reader: state_artifact_reader, - } -} - -type FileStateArtifactFetcher struct { - filePath string - group_name string - state_artifact_name string - state_artifact_reader StateReader -} - -func NewFileStateFetcher() StateFetcher { - filePath := config.GetInput() - state_artifact_reader := NewState() - return &FileStateArtifactFetcher{ - filePath: filePath, - group_name: config.GetGroupName(), - state_artifact_name: config.GetStateArtifactName(), - state_artifact_reader: state_artifact_reader, - } -} - -func (f *FileStateArtifactFetcher) FetchStateArtifact() (StateReader, error) { - /// Read the state artifact file from the file path - content, err := os.ReadFile(f.filePath) - if err != nil { - return nil, fmt.Errorf("failed to read the state artifact file: %v", err) - } - state_reader, err := FromJSON(content, f.state_artifact_reader.(*State)) - if err != nil { - return nil, fmt.Errorf("failed to parse the state artifact file: %v", err) - } - return state_reader, nil -} - -func (f *URLStateFetcher) FetchStateArtifact() (StateReader, error) { - cwd, err := os.Getwd() - if err != nil { - return nil, fmt.Errorf("failed to get current working directory: %v", err) - } - // Creating a file store in the current working directory will be deleted later after reading the state artifact - fs, err := file.New(fmt.Sprintf("%s/state-artifact", cwd)) - if err != nil { - return nil, fmt.Errorf("failed to create file store: %v", err) - } - defer fs.Close() - - ctx := context.Background() - - repo, err := remote.NewRepository(fmt.Sprintf("%s/%s/%s", f.url, f.group_name, f.state_artifact_name)) - if err != nil { - return nil, fmt.Errorf("failed to create remote repository: %v", err) - } - - // Setting up the authentication for the remote registry - repo.Client = &auth.Client{ - Client: retry.DefaultClient, - Cache: auth.NewCache(), - Credential: auth.StaticCredential( - f.url, - auth.Credential{ - Username: config.GetHarborUsername(), - Password: config.GetHarborPassword(), - }, - ), - } - // Copy from the remote repository to the file store - tag := "latest" - _, err = oras.Copy(ctx, repo, tag, fs, tag, oras.DefaultCopyOptions) - if err != nil { - return nil, fmt.Errorf("failed to copy from remote repository to file store: %v", err) - } - stateArtifactDir := filepath.Join(cwd, "state-artifact") - - var state_reader StateReader - // Find the state artifact file in the state-artifact directory that is created temporarily - err = filepath.Walk(stateArtifactDir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if filepath.Ext(info.Name()) == ".json" { - content, err := os.ReadFile(path) - if err != nil { - return err - } - fmt.Printf("Contents of %s:\n", info.Name()) - fmt.Println(string(content)) - - state_reader, err = FromJSON(content, f.state_artifact_reader.(*State)) - if err != nil { - return fmt.Errorf("failed to parse the state artifact file: %v", err) - } - - } - return nil - }) - - if err != nil { - return nil, fmt.Errorf("failed to read the state artifact file: %v", err) - } - // Clean up everything inside the state-artifact folder - err = os.RemoveAll(stateArtifactDir) - if err != nil { - return nil, fmt.Errorf("failed to remove state-artifact directory: %v", err) - } - return state_reader, nil -} - -// FromJSON parses the input JSON data into a StateArtifactReader -func FromJSON(data []byte, reg *State) (StateReader, error) { - - if err := json.Unmarshal(data, ®); err != nil { - fmt.Print("Error in unmarshalling") - return nil, err - } - fmt.Print(reg) - // Validation - if reg.Registry == "" { - return nil, fmt.Errorf("registry URL is required") - } - return reg, nil -} diff --git a/internal/state/state_process.go b/internal/state/state_process.go index e9c4c44..ecc758d 100644 --- a/internal/state/state_process.go +++ b/internal/state/state_process.go @@ -3,6 +3,7 @@ package state import ( "context" "fmt" + "sync" "container-registry.com/harbor-satellite/internal/notifier" "container-registry.com/harbor-satellite/logger" @@ -20,29 +21,36 @@ type FetchAndReplicateStateProcess struct { isRunning bool stateReader StateReader notifier notifier.Notifier + mu *sync.Mutex } -func NewFetchAndReplicateStateProcess(id uint64, cronExpr string, stateFetcher StateFetcher, notifier notifier.Notifier) FetchAndReplicateStateProcess { - return FetchAndReplicateStateProcess{ +func NewFetchAndReplicateStateProcess(id uint64, cronExpr string, stateFetcher StateFetcher, notifier notifier.Notifier) *FetchAndReplicateStateProcess { + return &FetchAndReplicateStateProcess{ id: id, name: FetchAndReplicateStateProcessName, cronExpr: cronExpr, isRunning: false, stateArtifactFetcher: stateFetcher, notifier: notifier, + mu: &sync.Mutex{}, } } func (f *FetchAndReplicateStateProcess) Execute(ctx context.Context) error { log := logger.FromContext(ctx) + f.mu.Lock() if f.IsRunning() { + f.mu.Unlock() log.Warn().Msg("Process is already running") return fmt.Errorf("process %s is already running", f.GetName()) } log.Info().Msg("Starting process to fetch and replicate state") f.isRunning = true + f.mu.Unlock() defer func() { + f.mu.Lock() f.isRunning = false + f.mu.Unlock() }() newStateFetched, err := f.stateArtifactFetcher.FetchStateArtifact() @@ -58,11 +66,12 @@ func (f *FetchAndReplicateStateProcess) Execute(ctx context.Context) error { log.Error().Err(err).Msg("Error sending notification") } - replicator := BasicNewReplicator(newStateFetched) + replicator := NewBasicReplicator(newStateFetched) if err := replicator.Replicate(ctx); err != nil { log.Error().Err(err).Msg("Error replicating state") return err } + f.stateReader = newStateFetched return nil } diff --git a/internal/utils/utils.go b/internal/utils/utils.go index 9e0323d..b427564 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -82,7 +82,7 @@ func HasInvalidPathChars(input string) bool { func GetRepositoryAndImageNameFromArtifact(repository string) (string, string, error) { parts := strings.Split(repository, "/") if len(parts) < 2 { - return "", "", fmt.Errorf("invalid repository format") + return "", "", fmt.Errorf("invalid repository format: %s. Expected format: repo/image", repository) } repo := parts[0] image := parts[1] diff --git a/logger/logger.go b/logger/logger.go index 69dfadb..55993e5 100644 --- a/logger/logger.go +++ b/logger/logger.go @@ -46,7 +46,11 @@ func AddLoggerToContext(ctx context.Context, logLevel string) context.Context { if i == nil { l = colorize("???", 37) // white } else { - l = strings.ToUpper(fmt.Sprintf("%s", i))[0:3] + lStr := strings.ToUpper(fmt.Sprintf("%s", i)) + if len(lStr) > 3 { + lStr = lStr[:3] + } + l = lStr } } return fmt.Sprintf("| %s |", l) diff --git a/main.go b/main.go index 95218e2..480a748 100644 --- a/main.go +++ b/main.go @@ -48,7 +48,7 @@ func run() error { if err := handleRegistrySetup(g, log, cancel); err != nil { return err } - scheduler := scheduler.NewBasicScheduler(&ctx) + scheduler := scheduler.NewBasicScheduler(ctx) ctx = context.WithValue(ctx, scheduler.GetSchedulerKey(), scheduler) err := scheduler.Start() if err != nil { From 09419ce8f919410983420f6045b4aec41c7f70ad Mon Sep 17 00:00:00 2001 From: Mehul-Kumar-27 Date: Thu, 3 Oct 2024 00:23:45 +0530 Subject: [PATCH 07/36] adding new format of the state file --- internal/state/artifact.go | 69 ++++++++++++++++++++++++++++---------- 1 file changed, 52 insertions(+), 17 deletions(-) diff --git a/internal/state/artifact.go b/internal/state/artifact.go index 541db4d..e9f152e 100644 --- a/internal/state/artifact.go +++ b/internal/state/artifact.go @@ -1,29 +1,34 @@ package state +import "reflect" + // ArtifactReader defines an interface for reading artifact data type ArtifactReader interface { - // GetRepository returns the repository of the artifact GetRepository() string - // GetTag returns the tag of the artifact - GetTag() string - // GetHash returns the hash of the artifact - GetHash() string - // HasChanged returns true if the artifact has changed + GetTags() []string + GetDigest() string + GetType() string + IsDeleted() bool HasChanged(newArtifact ArtifactReader) bool } // Artifact represents an artifact object in the registry type Artifact struct { - Repository string `json:"repository"` - Tag string `json:"tag"` - Hash string `json:"hash"` + Deleted bool `json:"deleted"` + Repository string `json:"repository"` + Tags []string `json:"tag"` + Digest string `json:"digest"` + Type string `json:"type"` } -func NewArtifact(repository, tag, hash string) ArtifactReader { +// NewArtifact creates a new Artifact object +func NewArtifact(deleted bool, repository string, tags []string, digest, artifactType string) ArtifactReader { return &Artifact{ + Deleted: deleted, Repository: repository, - Tag: tag, - Hash: hash, + Tags: tags, + Digest: digest, + Type: artifactType, } } @@ -31,14 +36,44 @@ func (a *Artifact) GetRepository() string { return a.Repository } -func (a *Artifact) GetTag() string { - return a.Tag +func (a *Artifact) GetTags() []string { + return a.Tags +} + +func (a *Artifact) GetDigest() string { + return a.Digest } -func (a *Artifact) GetHash() string { - return a.Hash +func (a *Artifact) GetType() string { + return a.Type } +func (a *Artifact) IsDeleted() bool { + return a.Deleted +} + +// HasChanged compares the current artifact with another to determine if there are any changes func (a *Artifact) HasChanged(newArtifact ArtifactReader) bool { - return a.GetHash() != newArtifact.GetHash() + // Compare the digest (hash) + if a.GetDigest() != newArtifact.GetDigest() { + return true + } + + // Compare the repository + if a.GetRepository() != newArtifact.GetRepository() { + return true + } + + // Compare the tags (order-agnostic comparison) + if !reflect.DeepEqual(a.GetTags(), newArtifact.GetTags()) { + return true + } + + // Compare the deletion status + if a.IsDeleted() != newArtifact.IsDeleted() { + return true + } + + // No changes detected + return false } From c60f7a4d50192c5f3fa7b4059a61de58bd3c736a Mon Sep 17 00:00:00 2001 From: Mehul-Kumar-27 Date: Tue, 8 Oct 2024 18:22:20 +0530 Subject: [PATCH 08/36] adding config to process new state artifact file --- .env | 4 +- config.toml | 2 +- image-list/images.json | 32 ++++++---- internal/state/fetcher.go | 111 ++++++++++++++++------------------- internal/state/replicator.go | 49 ++++++++++------ internal/utils/utils.go | 8 +++ 6 files changed, 114 insertions(+), 92 deletions(-) diff --git a/.env b/.env index 73ed245..f39fbfd 100644 --- a/.env +++ b/.env @@ -4,5 +4,5 @@ ZOT_URL="127.0.0.1:8585" TOKEN="" ENV=dev USE_UNSECURE=true -GROUP_NAME=test-satellite-group -STATE_ARTIFACT_NAME=state-artifact +GROUP_NAME=satellite-test-group-state +STATE_ARTIFACT_NAME=state diff --git a/config.toml b/config.toml index bd7f9d5..30614fc 100644 --- a/config.toml +++ b/config.toml @@ -7,7 +7,7 @@ own_registry_port = "8585" # URL of remote registry OR local file path # url_or_file = "https://demo.goharbor.io/v2/myproject/album-server" -url_or_file = "https://demo.goharbor.io" +url_or_file = "https://registry.bupd.xyz" ## for testing for local file # url_or_file = "./image-list/images.json" diff --git a/image-list/images.json b/image-list/images.json index cd8ac06..86cf321 100644 --- a/image-list/images.json +++ b/image-list/images.json @@ -1,15 +1,25 @@ { - "registry": "http://demo.goharbor.io/", - "artifacts": [ - { - "repository": "satellite-test-alpine/alpine", - "tag": "latest", - "hash": "sha256:9cee2b38" + "registry": "Satellite", + "artifacts": [ + { + "repository": "satellite-test-group-state/alpine", + "tag": [ + "latest" + ], + "labels": null, + "type": "IMAGE", + "digest": "sha256:9cee2b382fe2412cd77d5d437d15a93da8de373813621f2e4d406e3df0cf0e7c", + "deleted": false }, - { - "repository": "satellite-test-postgres/postgres", - "tag": "latest", - "hash": "sha256:9cee2b38" - } + { + "repository": "satellite-test-group-state/postgres", + "tag": [ + "latest" + ], + "labels": null, + "type": "IMAGE", + "digest": "sha256:dde924f70bc972261013327c480adf402ea71487b5750e40569a0b74fa90c74a", + "deleted": false + } ] } diff --git a/internal/state/fetcher.go b/internal/state/fetcher.go index 60a22f9..d028087 100644 --- a/internal/state/fetcher.go +++ b/internal/state/fetcher.go @@ -1,22 +1,20 @@ package state import ( - "context" + "archive/tar" + "bytes" "encoding/json" "fmt" + "io" "os" - "path/filepath" "strings" "container-registry.com/harbor-satellite/internal/config" - "oras.land/oras-go/v2" - "oras.land/oras-go/v2/content/file" - "oras.land/oras-go/v2/registry/remote" - "oras.land/oras-go/v2/registry/remote/auth" - "oras.land/oras-go/v2/registry/remote/retry" + "container-registry.com/harbor-satellite/internal/utils" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/crane" ) - type StateFetcher interface { // Fetches the state artifact from the registry FetchStateArtifact() (StateReader, error) @@ -75,73 +73,68 @@ func (f *FileStateArtifactFetcher) FetchStateArtifact() (StateReader, error) { } func (f *URLStateFetcher) FetchStateArtifact() (StateReader, error) { - cwd, err := os.Getwd() - if err != nil { - return nil, fmt.Errorf("failed to get current working directory: %v", err) - } - // Creating a file store in the current working directory will be deleted later after reading the state artifact - fs, err := file.New(fmt.Sprintf("%s/state-artifact", cwd)) - if err != nil { - return nil, fmt.Errorf("failed to create file store: %v", err) + + auth := authn.FromConfig(authn.AuthConfig{ + Username: config.GetHarborUsername(), + Password: config.GetHarborPassword(), + }) + + options := []crane.Option{crane.WithAuth(auth)} + if config.UseUnsecure() { + options = append(options, crane.Insecure) } - defer fs.Close() - ctx := context.Background() + sourceRegistry := utils.FormatRegistryUrl(config.GetRemoteRegistryURL()) + group := config.GetGroupName() + stateArtifactName := config.GetStateArtifactName() + var tag string = "latest" + fmt.Printf("Pulling state artifact from %s/%s/%s:%s\n", sourceRegistry, group, stateArtifactName, tag) + fmt.Printf("Auth: %v\n", auth) - repo, err := remote.NewRepository(fmt.Sprintf("%s/%s/%s", f.url, f.group_name, f.state_artifact_name)) + // pull the state artifact from the central registry + img, err := crane.Pull(fmt.Sprintf("%s/%s/%s:%s", sourceRegistry, group, stateArtifactName, tag), options...) if err != nil { - return nil, fmt.Errorf("failed to create remote repository: %v", err) + return nil, fmt.Errorf("failed to pull the state artifact: %v", err) } - // Setting up the authentication for the remote registry - repo.Client = &auth.Client{ - Client: retry.DefaultClient, - Cache: auth.NewCache(), - Credential: auth.StaticCredential( - f.url, - auth.Credential{ - Username: config.GetHarborUsername(), - Password: config.GetHarborPassword(), - }, - ), - } - // Copy from the remote repository to the file store - tag := "latest" - _, err = oras.Copy(ctx, repo, tag, fs, tag, oras.DefaultCopyOptions) - if err != nil { - return nil, fmt.Errorf("failed to copy from remote repository to file store: %v", err) + tarContent := new(bytes.Buffer) + if err := crane.Export(img, tarContent); err != nil { + return nil, fmt.Errorf("failed to export the state artifact: %v", err) } - stateArtifactDir := filepath.Join(cwd, "state-artifact") - var state_reader StateReader - // Find the state artifact file in the state-artifact directory that is created temporarily - err = filepath.Walk(stateArtifactDir, func(path string, info os.FileInfo, err error) error { + // parse the state artifact + tr := tar.NewReader(tarContent) + var artifactsJSON []byte + + for { + hdr, err := tr.Next() + if err == io.EOF { + break // End of tar archive + } if err != nil { - return err + return nil, fmt.Errorf("failed to read the tar archive: %v", err) } - if filepath.Ext(info.Name()) == ".json" { - content, err := os.ReadFile(path) - if err != nil { - return err - } - state_reader, err = FromJSON(content, f.state_artifact_reader) + + if hdr.Name == "artifacts.json" { + // Found `artifacts.json`, read the content + artifactsJSON, err = io.ReadAll(tr) if err != nil { - return fmt.Errorf("failed to parse the state artifact file: %v", err) + return nil, fmt.Errorf("failed to read the artifacts.json file: %v", err) } - return nil + break } - return nil - }) + } - if err != nil { - return nil, fmt.Errorf("failed to read the state artifact file: %v", err) + if artifactsJSON == nil { + return nil, fmt.Errorf("artifacts.json not found in the state artifact") } - // Clean up everything inside the state-artifact folder - err = os.RemoveAll(stateArtifactDir) + + err = json.Unmarshal(artifactsJSON, &f.state_artifact_reader) if err != nil { - return nil, fmt.Errorf("failed to remove state-artifact directory: %v", err) + return nil, fmt.Errorf("failed to parse the artifacts.json file: %v", err) } - return state_reader, nil + + return f.state_artifact_reader, nil } // FromJSON parses the input JSON data into a StateArtifactReader @@ -151,7 +144,7 @@ func FromJSON(data []byte, reg StateReader) (StateReader, error) { return nil, err } // Validation - if reg.GetRegistryURL()== "" { + if reg.GetRegistryURL() == "" { return nil, fmt.Errorf("registry URL is required") } return reg, nil diff --git a/internal/state/replicator.go b/internal/state/replicator.go index 6dec2f0..9ac1312 100644 --- a/internal/state/replicator.go +++ b/internal/state/replicator.go @@ -10,6 +10,8 @@ import ( "container-registry.com/harbor-satellite/logger" "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/crane" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/types" ) type Replicator interface { @@ -34,7 +36,7 @@ func NewBasicReplicator(state_reader StateReader) Replicator { stateReader: state_reader, } } - +// Replicate replicates images from the source registry to the Zot registry. func (r *BasicReplicator) Replicate(ctx context.Context) error { log := logger.FromContext(ctx) auth := authn.FromConfig(authn.AuthConfig{ @@ -46,33 +48,42 @@ func (r *BasicReplicator) Replicate(ctx context.Context) error { if r.useUnsecure { options = append(options, crane.Insecure) } - sourceRegistry := r.stateReader.GetRegistryURL() + sourceRegistry := utils.FormatRegistryUrl(config.GetRemoteRegistryURL()) for _, artifact := range r.stateReader.GetArtifacts() { - // Extract the image name from the repository of the artifact + // Extract the image name and repository from the artifact repo, image, err := utils.GetRepositoryAndImageNameFromArtifact(artifact.GetRepository()) if err != nil { log.Error().Msgf("Error getting repository and image name: %v", err) return err } - log.Info().Msgf("Pulling image %s from repository %s at registry %s", image, repo, sourceRegistry) - // Pull the image at the given repository at the source registry - srcImage, err := crane.Pull(fmt.Sprintf("%s/%s/%s", sourceRegistry, repo, image), options...) - if err != nil { - log.Error().Msgf("Failed to pull image: %v", err) - return err - } - // Push the image to the local registry - err = crane.Push(srcImage, fmt.Sprintf("%s/%s", r.zotURL, image), options...) - if err != nil { - log.Error().Msgf("Failed to push image: %v", err) - return err + allTags := artifact.GetTags() + + // Pull and replicate all tags of the image + for _, tag := range allTags { + log.Info().Msgf("Pulling image %s from repository %s at registry %s with tag %s", image, repo, sourceRegistry, tag) + + // Pull the image from the source registry + srcImage, err := crane.Pull(fmt.Sprintf("%s/%s/%s:%s", sourceRegistry, image, image, tag), options...) + if err != nil { + log.Error().Msgf("Failed to pull image: %v", err) + return err + } + + // Convert Docker manifest to OCI manifest + ociImage := mutate.MediaType(srcImage, types.OCIManifestSchema1) + + // Push the converted OCI image to the Zot registry + err = crane.Push(ociImage, fmt.Sprintf("%s/%s", r.zotURL, image), options...) + if err != nil { + log.Error().Msgf("Failed to push image: %v", err) + return err + } + log.Info().Msgf("Image %s pushed successfully", image) } - log.Info().Msgf("Image %s pushed successfully", image) } - // Delete ./local-oci-layout directory - // This is required because it is a temporary directory used by crane to pull and push images to and from - // And crane does not automatically clean it + + // Clean up the temporary directory if err := os.RemoveAll("./local-oci-layout"); err != nil { log.Error().Msgf("Failed to remove directory: %v", err) return fmt.Errorf("failed to remove directory: %w", err) diff --git a/internal/utils/utils.go b/internal/utils/utils.go index b427564..f77db03 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -116,3 +116,11 @@ func FormatDuration(input string) (string , error) { return result, nil } + +// FormatRegistryUrl formats the registry URL by trimming the "https://" or "http://" prefix if present +func FormatRegistryUrl(url string) string { + // Trim the "https://" or "http://" prefix if present + url = strings.TrimPrefix(url, "https://") + url = strings.TrimPrefix(url, "http://") + return url +} From dec1ba0ee0654262b06a592fc9d5e8967aea8b33 Mon Sep 17 00:00:00 2001 From: Mehul-Kumar-27 Date: Tue, 8 Oct 2024 18:47:31 +0530 Subject: [PATCH 09/36] coderabbit review --- internal/state/artifact.go | 9 +++++ internal/state/fetcher.go | 68 +++++++++++++++--------------------- internal/state/replicator.go | 5 +-- internal/utils/utils.go | 7 ++-- 4 files changed, 43 insertions(+), 46 deletions(-) diff --git a/internal/state/artifact.go b/internal/state/artifact.go index e9f152e..6e9a576 100644 --- a/internal/state/artifact.go +++ b/internal/state/artifact.go @@ -74,6 +74,15 @@ func (a *Artifact) HasChanged(newArtifact ArtifactReader) bool { return true } + if a.GetType() != newArtifact.GetType() { + return true + } + + // Compare the tags (order-agnostic comparison using reflect.DeepEqual) + if !reflect.DeepEqual(a.GetTags(), newArtifact.GetTags()) { + return true + } + // No changes detected return false } diff --git a/internal/state/fetcher.go b/internal/state/fetcher.go index d028087..d155450 100644 --- a/internal/state/fetcher.go +++ b/internal/state/fetcher.go @@ -7,7 +7,6 @@ import ( "fmt" "io" "os" - "strings" "container-registry.com/harbor-satellite/internal/config" "container-registry.com/harbor-satellite/internal/utils" @@ -16,51 +15,50 @@ import ( ) type StateFetcher interface { - // Fetches the state artifact from the registry FetchStateArtifact() (StateReader, error) } -type URLStateFetcher struct { - url string +type baseStateFetcher struct { group_name string state_artifact_name string state_artifact_reader StateReader } +type URLStateFetcher struct { + baseStateFetcher + url string +} + +type FileStateArtifactFetcher struct { + baseStateFetcher + filePath string +} + func NewURLStateFetcher() StateFetcher { url := config.GetRemoteRegistryURL() - // Trim the "https://" or "http://" prefix if present - url = strings.TrimPrefix(url, "https://") - url = strings.TrimPrefix(url, "http://") - state_artifact_reader := NewState() + url = utils.FormatRegistryURL(url) return &URLStateFetcher{ - url: url, - group_name: config.GetGroupName(), - state_artifact_name: config.GetStateArtifactName(), - state_artifact_reader: state_artifact_reader, + baseStateFetcher: baseStateFetcher{ + group_name: config.GetGroupName(), + state_artifact_name: config.GetStateArtifactName(), + state_artifact_reader: NewState(), + }, + url: url, } } -type FileStateArtifactFetcher struct { - filePath string - group_name string - state_artifact_name string - state_artifact_reader StateReader -} - func NewFileStateFetcher() StateFetcher { - filePath := config.GetInput() - state_artifact_reader := NewState() return &FileStateArtifactFetcher{ - filePath: filePath, - group_name: config.GetGroupName(), - state_artifact_name: config.GetStateArtifactName(), - state_artifact_reader: state_artifact_reader, + baseStateFetcher: baseStateFetcher{ + group_name: config.GetGroupName(), + state_artifact_name: config.GetStateArtifactName(), + state_artifact_reader: NewState(), + }, + filePath: config.GetInput(), } } func (f *FileStateArtifactFetcher) FetchStateArtifact() (StateReader, error) { - /// Read the state artifact file from the file path content, err := os.ReadFile(f.filePath) if err != nil { return nil, fmt.Errorf("failed to read the state artifact file: %v", err) @@ -73,7 +71,6 @@ func (f *FileStateArtifactFetcher) FetchStateArtifact() (StateReader, error) { } func (f *URLStateFetcher) FetchStateArtifact() (StateReader, error) { - auth := authn.FromConfig(authn.AuthConfig{ Username: config.GetHarborUsername(), Password: config.GetHarborPassword(), @@ -84,15 +81,10 @@ func (f *URLStateFetcher) FetchStateArtifact() (StateReader, error) { options = append(options, crane.Insecure) } - sourceRegistry := utils.FormatRegistryUrl(config.GetRemoteRegistryURL()) - group := config.GetGroupName() - stateArtifactName := config.GetStateArtifactName() - var tag string = "latest" - fmt.Printf("Pulling state artifact from %s/%s/%s:%s\n", sourceRegistry, group, stateArtifactName, tag) - fmt.Printf("Auth: %v\n", auth) + sourceRegistry := utils.FormatRegistryURL(config.GetRemoteRegistryURL()) + tag := "latest" - // pull the state artifact from the central registry - img, err := crane.Pull(fmt.Sprintf("%s/%s/%s:%s", sourceRegistry, group, stateArtifactName, tag), options...) + img, err := crane.Pull(fmt.Sprintf("%s/%s/%s:%s", sourceRegistry, f.group_name, f.state_artifact_name, tag), options...) if err != nil { return nil, fmt.Errorf("failed to pull the state artifact: %v", err) } @@ -102,21 +94,19 @@ func (f *URLStateFetcher) FetchStateArtifact() (StateReader, error) { return nil, fmt.Errorf("failed to export the state artifact: %v", err) } - // parse the state artifact tr := tar.NewReader(tarContent) var artifactsJSON []byte for { hdr, err := tr.Next() if err == io.EOF { - break // End of tar archive + break } if err != nil { return nil, fmt.Errorf("failed to read the tar archive: %v", err) } if hdr.Name == "artifacts.json" { - // Found `artifacts.json`, read the content artifactsJSON, err = io.ReadAll(tr) if err != nil { return nil, fmt.Errorf("failed to read the artifacts.json file: %v", err) @@ -137,13 +127,11 @@ func (f *URLStateFetcher) FetchStateArtifact() (StateReader, error) { return f.state_artifact_reader, nil } -// FromJSON parses the input JSON data into a StateArtifactReader func FromJSON(data []byte, reg StateReader) (StateReader, error) { if err := json.Unmarshal(data, ®); err != nil { fmt.Print("Error in unmarshalling") return nil, err } - // Validation if reg.GetRegistryURL() == "" { return nil, fmt.Errorf("registry URL is required") } diff --git a/internal/state/replicator.go b/internal/state/replicator.go index 9ac1312..2fe1ec2 100644 --- a/internal/state/replicator.go +++ b/internal/state/replicator.go @@ -36,6 +36,7 @@ func NewBasicReplicator(state_reader StateReader) Replicator { stateReader: state_reader, } } + // Replicate replicates images from the source registry to the Zot registry. func (r *BasicReplicator) Replicate(ctx context.Context) error { log := logger.FromContext(ctx) @@ -48,7 +49,7 @@ func (r *BasicReplicator) Replicate(ctx context.Context) error { if r.useUnsecure { options = append(options, crane.Insecure) } - sourceRegistry := utils.FormatRegistryUrl(config.GetRemoteRegistryURL()) + sourceRegistry := utils.FormatRegistryURL(config.GetRemoteRegistryURL()) for _, artifact := range r.stateReader.GetArtifacts() { // Extract the image name and repository from the artifact @@ -72,7 +73,7 @@ func (r *BasicReplicator) Replicate(ctx context.Context) error { // Convert Docker manifest to OCI manifest ociImage := mutate.MediaType(srcImage, types.OCIManifestSchema1) - + // Push the converted OCI image to the Zot registry err = crane.Push(ociImage, fmt.Sprintf("%s/%s", r.zotURL, image), options...) if err != nil { diff --git a/internal/utils/utils.go b/internal/utils/utils.go index f77db03..92dcefa 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -78,7 +78,6 @@ func HasInvalidPathChars(input string) bool { return strings.ContainsAny(input, "\\:*?\"<>|") } - func GetRepositoryAndImageNameFromArtifact(repository string) (string, string, error) { parts := strings.Split(repository, "/") if len(parts) < 2 { @@ -89,7 +88,7 @@ func GetRepositoryAndImageNameFromArtifact(repository string) (string, string, e return repo, image, nil } -func FormatDuration(input string) (string , error) { +func FormatDuration(input string) (string, error) { seconds, err := strconv.Atoi(input) // Convert input string to an integer if err != nil { return "", errors.New("invalid input: not a valid number") @@ -117,8 +116,8 @@ func FormatDuration(input string) (string , error) { return result, nil } -// FormatRegistryUrl formats the registry URL by trimming the "https://" or "http://" prefix if present -func FormatRegistryUrl(url string) string { +// FormatRegistryURL formats the registry URL by trimming the "https://" or "http://" prefix if present +func FormatRegistryURL(url string) string { // Trim the "https://" or "http://" prefix if present url = strings.TrimPrefix(url, "https://") url = strings.TrimPrefix(url, "http://") From ef0d82afc14ebf384a014180425ebc6a414c061f Mon Sep 17 00:00:00 2001 From: Mehul-Kumar-27 Date: Tue, 8 Oct 2024 19:21:15 +0530 Subject: [PATCH 10/36] added ./zot to gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 1f90f92..32234d8 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,5 @@ dist/ zot/cache.db secrets.txt __debug_bin1949266242 + +/zot From 74fc4b9241462d6c66b538b53750ab4f7fa9ac3a Mon Sep 17 00:00:00 2001 From: Mehul-Kumar-27 Date: Thu, 10 Oct 2024 19:34:57 +0530 Subject: [PATCH 11/36] fixing the replication process --- internal/satellite/satellite.go | 7 ++- internal/state/artifact.go | 30 +++++++-- internal/state/fetcher.go | 24 ++++++-- internal/state/replicator.go | 105 +++++++++++++++++--------------- internal/state/state.go | 28 +++++++++ internal/state/state_process.go | 99 +++++++++++++++++++++++++++--- 6 files changed, 224 insertions(+), 69 deletions(-) diff --git a/internal/satellite/satellite.go b/internal/satellite/satellite.go index ef6bdf1..cc1f415 100644 --- a/internal/satellite/satellite.go +++ b/internal/satellite/satellite.go @@ -35,12 +35,17 @@ func (s *Satellite) Run(ctx context.Context) error { log.Warn().Msgf("Using default duration: %v", state.DefaultFetchAndReplicateStateTimePeriod) cronExpr = state.DefaultFetchAndReplicateStateTimePeriod } + userName := config.GetHarborUsername() + password := config.GetHarborPassword() + zotURL := config.GetZotURL() + sourceRegistry := utils.FormatRegistryURL(config.GetRemoteRegistryURL()) + useUnsecure := config.UseUnsecure() // Get the scheduler from the context scheduler := ctx.Value(s.schedulerKey).(scheduler.Scheduler) // Create a simple notifier and add it to the process notifier := notifier.NewSimpleNotifier(ctx) // Creating a process to fetch and replicate the state - fetchAndReplicateStateProcess := state.NewFetchAndReplicateStateProcess(scheduler.NextID(), cronExpr, s.stateArtifactFetcher, notifier) + fetchAndReplicateStateProcess := state.NewFetchAndReplicateStateProcess(scheduler.NextID(), cronExpr, s.stateArtifactFetcher, notifier, userName, password, zotURL, sourceRegistry, useUnsecure) // Add the process to the scheduler scheduler.Schedule(fetchAndReplicateStateProcess) diff --git a/internal/state/artifact.go b/internal/state/artifact.go index 6e9a576..53bedee 100644 --- a/internal/state/artifact.go +++ b/internal/state/artifact.go @@ -1,6 +1,8 @@ package state -import "reflect" +import ( + "reflect" +) // ArtifactReader defines an interface for reading artifact data type ArtifactReader interface { @@ -10,15 +12,19 @@ type ArtifactReader interface { GetType() string IsDeleted() bool HasChanged(newArtifact ArtifactReader) bool + SetRepository(repository string) + SetName(name string) + GetName() string } // Artifact represents an artifact object in the registry type Artifact struct { - Deleted bool `json:"deleted"` - Repository string `json:"repository"` - Tags []string `json:"tag"` - Digest string `json:"digest"` - Type string `json:"type"` + Deleted bool `json:"deleted,omitempty"` + Repository string `json:"repository,omitempty"` + Tags []string `json:"tag,omitempty"` + Digest string `json:"digest,omitempty"` + Type string `json:"type,omitempty"` + Name string `json:"name,omitempty"` } // NewArtifact creates a new Artifact object @@ -52,6 +58,10 @@ func (a *Artifact) IsDeleted() bool { return a.Deleted } +func (a *Artifact) GetName() string { + return a.Name +} + // HasChanged compares the current artifact with another to determine if there are any changes func (a *Artifact) HasChanged(newArtifact ArtifactReader) bool { // Compare the digest (hash) @@ -86,3 +96,11 @@ func (a *Artifact) HasChanged(newArtifact ArtifactReader) bool { // No changes detected return false } + +func (a *Artifact) SetRepository(repository string) { + a.Repository = repository +} + +func (a *Artifact) SetName(name string) { + a.Name = name +} diff --git a/internal/state/fetcher.go b/internal/state/fetcher.go index d155450..d6b1296 100644 --- a/internal/state/fetcher.go +++ b/internal/state/fetcher.go @@ -63,11 +63,11 @@ func (f *FileStateArtifactFetcher) FetchStateArtifact() (StateReader, error) { if err != nil { return nil, fmt.Errorf("failed to read the state artifact file: %v", err) } - state_reader, err := FromJSON(content, f.state_artifact_reader) + err = json.Unmarshal(content, &f.state_artifact_reader) if err != nil { return nil, fmt.Errorf("failed to parse the state artifact file: %v", err) } - return state_reader, nil + return f.state_artifact_reader, nil } func (f *URLStateFetcher) FetchStateArtifact() (StateReader, error) { @@ -118,13 +118,29 @@ func (f *URLStateFetcher) FetchStateArtifact() (StateReader, error) { if artifactsJSON == nil { return nil, fmt.Errorf("artifacts.json not found in the state artifact") } - err = json.Unmarshal(artifactsJSON, &f.state_artifact_reader) if err != nil { return nil, fmt.Errorf("failed to parse the artifacts.json file: %v", err) } + + state, err := ProcessState(&f.state_artifact_reader) + if err != nil { + return nil, fmt.Errorf("failed to process the state: %v", err) + } + return *state, nil +} - return f.state_artifact_reader, nil +func ProcessState(state *StateReader) (*StateReader, error) { + for _, artifact := range (*state).GetArtifacts() { + repo, image, err := utils.GetRepositoryAndImageNameFromArtifact(artifact.GetRepository()) + if err != nil { + fmt.Printf("Error in getting repository and image name: %v", err) + return nil, err + } + artifact.SetRepository(repo) + artifact.SetName(image) + } + return state, nil } func FromJSON(data []byte, reg StateReader) (StateReader, error) { diff --git a/internal/state/replicator.go b/internal/state/replicator.go index 2fe1ec2..6ed3d62 100644 --- a/internal/state/replicator.go +++ b/internal/state/replicator.go @@ -3,10 +3,7 @@ package state import ( "context" "fmt" - "os" - "container-registry.com/harbor-satellite/internal/config" - "container-registry.com/harbor-satellite/internal/utils" "container-registry.com/harbor-satellite/logger" "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/crane" @@ -16,29 +13,31 @@ import ( type Replicator interface { // Replicate copies images from the source registry to the local registry. - Replicate(ctx context.Context) error + Replicate(ctx context.Context, replicationEntities []ArtifactReader) error + // DeleteReplicationEntity deletes the image from the local registry. + DeleteReplicationEntity(ctx context.Context, replicationEntity []ArtifactReader) error } type BasicReplicator struct { - username string - password string - useUnsecure bool - zotURL string - stateReader StateReader + username string + password string + useUnsecure bool + remoteRegistryURL string + sourceRegistry string } -func NewBasicReplicator(state_reader StateReader) Replicator { +func NewBasicReplicator(username, password, zotURL, sourceRegistry string, useUnsecure bool) Replicator { return &BasicReplicator{ - username: config.GetHarborUsername(), - password: config.GetHarborPassword(), - useUnsecure: config.UseUnsecure(), - zotURL: config.GetZotURL(), - stateReader: state_reader, + username: username, + password: password, + useUnsecure: useUnsecure, + remoteRegistryURL: zotURL, + sourceRegistry: sourceRegistry, } } // Replicate replicates images from the source registry to the Zot registry. -func (r *BasicReplicator) Replicate(ctx context.Context) error { +func (r *BasicReplicator) Replicate(ctx context.Context, replicationEntities []ArtifactReader) error { log := logger.FromContext(ctx) auth := authn.FromConfig(authn.AuthConfig{ Username: r.username, @@ -49,45 +48,55 @@ func (r *BasicReplicator) Replicate(ctx context.Context) error { if r.useUnsecure { options = append(options, crane.Insecure) } - sourceRegistry := utils.FormatRegistryURL(config.GetRemoteRegistryURL()) - for _, artifact := range r.stateReader.GetArtifacts() { - // Extract the image name and repository from the artifact - repo, image, err := utils.GetRepositoryAndImageNameFromArtifact(artifact.GetRepository()) + for _, replicationEntity := range replicationEntities { + + log.Info().Msgf("Pulling image %s from repository %s at registry %s with tag %s", replicationEntity.GetName(), replicationEntity.GetRepository(), r.sourceRegistry, replicationEntity.GetTags()[0]) + + // Pull the image from the source registry + srcImage, err := crane.Pull(fmt.Sprintf("%s/%s/%s:%s", r.sourceRegistry, replicationEntity.GetName(), replicationEntity.GetName(), replicationEntity.GetTags()[0]), options...) if err != nil { - log.Error().Msgf("Error getting repository and image name: %v", err) + log.Error().Msgf("Failed to pull image: %v", err) return err } - allTags := artifact.GetTags() - - // Pull and replicate all tags of the image - for _, tag := range allTags { - log.Info().Msgf("Pulling image %s from repository %s at registry %s with tag %s", image, repo, sourceRegistry, tag) - - // Pull the image from the source registry - srcImage, err := crane.Pull(fmt.Sprintf("%s/%s/%s:%s", sourceRegistry, image, image, tag), options...) - if err != nil { - log.Error().Msgf("Failed to pull image: %v", err) - return err - } - - // Convert Docker manifest to OCI manifest - ociImage := mutate.MediaType(srcImage, types.OCIManifestSchema1) - - // Push the converted OCI image to the Zot registry - err = crane.Push(ociImage, fmt.Sprintf("%s/%s", r.zotURL, image), options...) - if err != nil { - log.Error().Msgf("Failed to push image: %v", err) - return err - } - log.Info().Msgf("Image %s pushed successfully", image) + + // Convert Docker manifest to OCI manifest + ociImage := mutate.MediaType(srcImage, types.OCIManifestSchema1) + + // Push the converted OCI image to the Zot registry + err = crane.Push(ociImage, fmt.Sprintf("%s/%s:%s", r.remoteRegistryURL, replicationEntity.GetName(), replicationEntity.GetTags()[0]), options...) + if err != nil { + log.Error().Msgf("Failed to push image: %v", err) + return err } + log.Info().Msgf("Image %s pushed successfully", replicationEntity.GetName()) + } + return nil +} - // Clean up the temporary directory - if err := os.RemoveAll("./local-oci-layout"); err != nil { - log.Error().Msgf("Failed to remove directory: %v", err) - return fmt.Errorf("failed to remove directory: %w", err) +func (r *BasicReplicator) DeleteReplicationEntity(ctx context.Context, replicationEntity []ArtifactReader) error { + log := logger.FromContext(ctx) + auth := authn.FromConfig(authn.AuthConfig{ + Username: r.username, + Password: r.password, + }) + + options := []crane.Option{crane.WithAuth(auth)} + if r.useUnsecure { + options = append(options, crane.Insecure) + } + + for _, entity := range replicationEntity { + log.Info().Msgf("Deleting image %s from repository %s at registry %s with tag %s", entity.GetName(), entity.GetRepository(), r.remoteRegistryURL, entity.GetTags()[0]) + + err := crane.Delete(fmt.Sprintf("%s/%s/%s:%s", r.remoteRegistryURL, entity.GetName() ,entity.GetName(), entity.GetTags()[0]), options...) + if err != nil { + log.Error().Msgf("Failed to delete image: %v", err) + return err + } + log.Info().Msgf("Image %s deleted successfully", entity.GetName()) } + return nil } diff --git a/internal/state/state.go b/internal/state/state.go index 3d9bdeb..4874fd7 100644 --- a/internal/state/state.go +++ b/internal/state/state.go @@ -15,6 +15,10 @@ type StateReader interface { GetArtifactByRepository(repo string) (ArtifactReader, error) // Compare the state artifact with the new state artifact HasStateChanged(newState StateReader) bool + // RemoveAllArtifacts remove all the artifacts from the state which contains null tags and return the new state reader + RemoveArtifactsWithNullTags(stateWithNullTagsArtifacts StateReader) StateReader + // GetArtifactByName takes in the name of the artifact and returns the artifact associated with it + GetArtifactByNameAndTag(name, tag string) ArtifactReader } type State struct { @@ -68,3 +72,27 @@ func (a *State) HasStateChanged(newState StateReader) bool { } return false } + +func (a *State) RemoveArtifactsWithNullTags(stateWithNullTagsArtifacts StateReader) StateReader { + var newArtifactsWithoutNullTags []Artifact + for _, artifact := range a.Artifacts { + if artifact.Tags != nil || len(artifact.Tags) != 0 { + newArtifactsWithoutNullTags = append(newArtifactsWithoutNullTags, artifact) + } + } + stateWithNullTagsArtifacts.(*State).Artifacts = newArtifactsWithoutNullTags + return stateWithNullTagsArtifacts +} + +func (a *State) GetArtifactByNameAndTag(name, tag string) ArtifactReader { + for i := range a.Artifacts { + if a.Artifacts[i].GetName() == name { + for _, t := range a.Artifacts[i].GetTags() { + if t == tag { + return &a.Artifacts[i] + } + } + } + } + return nil +} diff --git a/internal/state/state_process.go b/internal/state/state_process.go index ecc758d..591d1a1 100644 --- a/internal/state/state_process.go +++ b/internal/state/state_process.go @@ -2,17 +2,28 @@ package state import ( "context" + "encoding/json" "fmt" + "os" "sync" "container-registry.com/harbor-satellite/internal/notifier" "container-registry.com/harbor-satellite/logger" + "github.com/rs/zerolog" ) const FetchAndReplicateStateProcessName string = "fetch-replicate-state-process" const DefaultFetchAndReplicateStateTimePeriod string = "00h00m05s" +type FetchAndReplicateAuthConfig struct { + Username string + Password string + useUnsecure bool + remoteRegistryURL string + sourceRegistry string +} + type FetchAndReplicateStateProcess struct { id uint64 name string @@ -22,9 +33,10 @@ type FetchAndReplicateStateProcess struct { stateReader StateReader notifier notifier.Notifier mu *sync.Mutex + authConfig FetchAndReplicateAuthConfig } -func NewFetchAndReplicateStateProcess(id uint64, cronExpr string, stateFetcher StateFetcher, notifier notifier.Notifier) *FetchAndReplicateStateProcess { +func NewFetchAndReplicateStateProcess(id uint64, cronExpr string, stateFetcher StateFetcher, notifier notifier.Notifier, username, password, remoteRegistryURL, sourceRegistryURL string, useUnsecure bool) *FetchAndReplicateStateProcess { return &FetchAndReplicateStateProcess{ id: id, name: FetchAndReplicateStateProcessName, @@ -33,6 +45,13 @@ func NewFetchAndReplicateStateProcess(id uint64, cronExpr string, stateFetcher S stateArtifactFetcher: stateFetcher, notifier: notifier, mu: &sync.Mutex{}, + authConfig: FetchAndReplicateAuthConfig{ + Username: username, + Password: password, + useUnsecure: useUnsecure, + remoteRegistryURL: remoteRegistryURL, + sourceRegistry: sourceRegistryURL, + }, } } @@ -58,28 +77,71 @@ func (f *FetchAndReplicateStateProcess) Execute(ctx context.Context) error { log.Error().Err(err).Msg("Error fetching state artifact") return err } - if !f.HasStateChanged(newStateFetched) { - log.Info().Msg("State has not changed") - return nil + PrintPrettyJson(newStateFetched, log) + log.Info().Msg("State fetched successfully") + log.Info().Msg("Checking if state has changed") + + deleteEntity, replicateEntity, newState := f.GetChanges(newStateFetched, log) + log.Info().Msgf("Total artifacts to delete: %d", len(deleteEntity)) + for _, entity := range deleteEntity { + log.Info().Msgf("Artifact: %s, Tag: %s", entity.GetName(), entity.GetTags()[0]) + } + log.Info().Msgf("Total artifacts to replicate: %d", len(replicateEntity)) + log.Info().Msg("Artifacts to replicate:") + for _, entity := range replicateEntity { + log.Info().Msgf("Artifact: %s, Tag: %s", entity.GetName(), entity.GetTags()[0]) } if err := f.notifier.Notify(); err != nil { log.Error().Err(err).Msg("Error sending notification") } - replicator := NewBasicReplicator(newStateFetched) - if err := replicator.Replicate(ctx); err != nil { + replicator := NewBasicReplicator(f.authConfig.Username, f.authConfig.Password, f.authConfig.remoteRegistryURL, f.authConfig.sourceRegistry, f.authConfig.useUnsecure) + // Delete the entities from the remote registry + if err := replicator.DeleteReplicationEntity(ctx, deleteEntity); err != nil { + log.Error().Err(err).Msg("Error deleting entities") + return err + } + // Replicate the entities to the remote registry + if err := replicator.Replicate(ctx, replicateEntity); err != nil { log.Error().Err(err).Msg("Error replicating state") return err } - f.stateReader = newStateFetched + f.stateReader = newState return nil } -func (f *FetchAndReplicateStateProcess) HasStateChanged(newState StateReader) bool { +func (f *FetchAndReplicateStateProcess) GetChanges(newState StateReader, log *zerolog.Logger) ([]ArtifactReader, []ArtifactReader, StateReader) { + var entityToDelete []ArtifactReader + var entityToReplicate []ArtifactReader if f.stateReader == nil { - return true + return entityToDelete, newState.GetArtifacts(), newState + } + log.Warn().Msg("Old state reader is not nil") + PrintPrettyJson(f.stateReader, log) + // Remove all the artifacts from the new state reader whose tags are null to make sure if a tags image is updated then it is replicated + newState = f.stateReader.RemoveArtifactsWithNullTags(newState) + + newArtifacts := newState.GetArtifacts() + + for _, artifact := range newArtifacts { + log.Info().Msgf("Checking artifact: %s", artifact.GetName()) + // Check if this artifact is present in the old state reader or not + oldArtifact := f.stateReader.GetArtifactByNameAndTag(artifact.GetName(), artifact.GetTags()[0]) + if oldArtifact == nil { + log.Info().Msgf("Artifact: %s not present in the old state reader", artifact.GetName()) + // This artifact is not present in the old state reader, so we need to replicate it + entityToReplicate = append(entityToReplicate, artifact) + continue + } + // This artifact is present in the old state reader, so we need to check if it has changed or not by comparing the digest + if artifact.GetDigest() != oldArtifact.GetDigest() { + // This artifact has changed, so we need to replicate it + entityToReplicate = append(entityToReplicate, artifact) + // We also need to delete from the remote registry + entityToDelete = append(entityToDelete, oldArtifact) + } } - return f.stateReader.HasStateChanged(newState) + return entityToDelete, entityToReplicate, newState } func (f *FetchAndReplicateStateProcess) GetID() uint64 { @@ -97,3 +159,20 @@ func (f *FetchAndReplicateStateProcess) GetCronExpr() string { func (f *FetchAndReplicateStateProcess) IsRunning() bool { return f.isRunning } + +func (f *FetchAndReplicateStateProcess) RemoveNullTagArtifacts(state StateReader) StateReader { + newStateReader := state.RemoveArtifactsWithNullTags(state) + return newStateReader +} + +func PrintPrettyJson(info interface{}, log *zerolog.Logger) error { + log.Warn().Msg("Printing pretty JSON") + stateJSON, err := json.MarshalIndent(info, "", " ") + if err != nil { + log.Error().Err(err).Msg("Error marshalling state to JSON") + return err + } + log.Info().Msgf("Fetched state: %s", stateJSON) + os.Exit(0) + return nil +} From 3d0e209377987b4b08b9c210629159561ff6c3f3 Mon Sep 17 00:00:00 2001 From: Mehul-Kumar-27 Date: Sat, 12 Oct 2024 23:54:25 +0530 Subject: [PATCH 12/36] fixing the replication and deletion process --- internal/state/artifact.go | 5 +- internal/state/fetcher.go | 49 +++------- internal/state/replicator.go | 4 +- internal/state/state.go | 26 +++--- internal/state/state_process.go | 157 ++++++++++++++++++++------------ main.go | 6 -- 6 files changed, 131 insertions(+), 116 deletions(-) diff --git a/internal/state/artifact.go b/internal/state/artifact.go index 53bedee..81ff6c8 100644 --- a/internal/state/artifact.go +++ b/internal/state/artifact.go @@ -19,11 +19,12 @@ type ArtifactReader interface { // Artifact represents an artifact object in the registry type Artifact struct { - Deleted bool `json:"deleted,omitempty"` Repository string `json:"repository,omitempty"` Tags []string `json:"tag,omitempty"` - Digest string `json:"digest,omitempty"` + Labels []string `json:"labels"` Type string `json:"type,omitempty"` + Digest string `json:"digest,omitempty"` + Deleted bool `json:"deleted"` Name string `json:"name,omitempty"` } diff --git a/internal/state/fetcher.go b/internal/state/fetcher.go index d6b1296..d2f9710 100644 --- a/internal/state/fetcher.go +++ b/internal/state/fetcher.go @@ -15,7 +15,7 @@ import ( ) type StateFetcher interface { - FetchStateArtifact() (StateReader, error) + FetchStateArtifact(state interface{}) error } type baseStateFetcher struct { @@ -58,19 +58,19 @@ func NewFileStateFetcher() StateFetcher { } } -func (f *FileStateArtifactFetcher) FetchStateArtifact() (StateReader, error) { +func (f *FileStateArtifactFetcher) FetchStateArtifact(state interface{}) error { content, err := os.ReadFile(f.filePath) if err != nil { - return nil, fmt.Errorf("failed to read the state artifact file: %v", err) + return fmt.Errorf("failed to read the state artifact file: %v", err) } - err = json.Unmarshal(content, &f.state_artifact_reader) + err = json.Unmarshal(content, state) if err != nil { - return nil, fmt.Errorf("failed to parse the state artifact file: %v", err) + return fmt.Errorf("failed to parse the state artifact file: %v", err) } - return f.state_artifact_reader, nil + return nil } -func (f *URLStateFetcher) FetchStateArtifact() (StateReader, error) { +func (f *URLStateFetcher) FetchStateArtifact(state interface{}) error { auth := authn.FromConfig(authn.AuthConfig{ Username: config.GetHarborUsername(), Password: config.GetHarborPassword(), @@ -86,12 +86,12 @@ func (f *URLStateFetcher) FetchStateArtifact() (StateReader, error) { img, err := crane.Pull(fmt.Sprintf("%s/%s/%s:%s", sourceRegistry, f.group_name, f.state_artifact_name, tag), options...) if err != nil { - return nil, fmt.Errorf("failed to pull the state artifact: %v", err) + return fmt.Errorf("failed to pull the state artifact: %v", err) } tarContent := new(bytes.Buffer) if err := crane.Export(img, tarContent); err != nil { - return nil, fmt.Errorf("failed to export the state artifact: %v", err) + return fmt.Errorf("failed to export the state artifact: %v", err) } tr := tar.NewReader(tarContent) @@ -103,44 +103,25 @@ func (f *URLStateFetcher) FetchStateArtifact() (StateReader, error) { break } if err != nil { - return nil, fmt.Errorf("failed to read the tar archive: %v", err) + return fmt.Errorf("failed to read the tar archive: %v", err) } if hdr.Name == "artifacts.json" { artifactsJSON, err = io.ReadAll(tr) if err != nil { - return nil, fmt.Errorf("failed to read the artifacts.json file: %v", err) + return fmt.Errorf("failed to read the artifacts.json file: %v", err) } break } } - if artifactsJSON == nil { - return nil, fmt.Errorf("artifacts.json not found in the state artifact") - } - err = json.Unmarshal(artifactsJSON, &f.state_artifact_reader) - if err != nil { - return nil, fmt.Errorf("failed to parse the artifacts.json file: %v", err) + return fmt.Errorf("artifacts.json not found in the state artifact") } - - state, err := ProcessState(&f.state_artifact_reader) + err = json.Unmarshal(artifactsJSON, &state) if err != nil { - return nil, fmt.Errorf("failed to process the state: %v", err) - } - return *state, nil -} - -func ProcessState(state *StateReader) (*StateReader, error) { - for _, artifact := range (*state).GetArtifacts() { - repo, image, err := utils.GetRepositoryAndImageNameFromArtifact(artifact.GetRepository()) - if err != nil { - fmt.Printf("Error in getting repository and image name: %v", err) - return nil, err - } - artifact.SetRepository(repo) - artifact.SetName(image) + return fmt.Errorf("failed to parse the artifacts.json file: %v", err) } - return state, nil + return nil } func FromJSON(data []byte, reg StateReader) (StateReader, error) { diff --git a/internal/state/replicator.go b/internal/state/replicator.go index 6ed3d62..24a97bd 100644 --- a/internal/state/replicator.go +++ b/internal/state/replicator.go @@ -64,7 +64,7 @@ func (r *BasicReplicator) Replicate(ctx context.Context, replicationEntities []A ociImage := mutate.MediaType(srcImage, types.OCIManifestSchema1) // Push the converted OCI image to the Zot registry - err = crane.Push(ociImage, fmt.Sprintf("%s/%s:%s", r.remoteRegistryURL, replicationEntity.GetName(), replicationEntity.GetTags()[0]), options...) + err = crane.Push(ociImage, fmt.Sprintf("%s/%s/%s:%s", r.remoteRegistryURL, replicationEntity.GetName(), replicationEntity.GetName(), replicationEntity.GetTags()[0]), options...) if err != nil { log.Error().Msgf("Failed to push image: %v", err) return err @@ -90,7 +90,7 @@ func (r *BasicReplicator) DeleteReplicationEntity(ctx context.Context, replicati for _, entity := range replicationEntity { log.Info().Msgf("Deleting image %s from repository %s at registry %s with tag %s", entity.GetName(), entity.GetRepository(), r.remoteRegistryURL, entity.GetTags()[0]) - err := crane.Delete(fmt.Sprintf("%s/%s/%s:%s", r.remoteRegistryURL, entity.GetName() ,entity.GetName(), entity.GetTags()[0]), options...) + err := crane.Delete(fmt.Sprintf("%s/%s/%s:%s", r.remoteRegistryURL, entity.GetName(), entity.GetName(), entity.GetTags()[0]), options...) if err != nil { log.Error().Msgf("Failed to delete image: %v", err) return err diff --git a/internal/state/state.go b/internal/state/state.go index 4874fd7..b191785 100644 --- a/internal/state/state.go +++ b/internal/state/state.go @@ -15,10 +15,10 @@ type StateReader interface { GetArtifactByRepository(repo string) (ArtifactReader, error) // Compare the state artifact with the new state artifact HasStateChanged(newState StateReader) bool - // RemoveAllArtifacts remove all the artifacts from the state which contains null tags and return the new state reader - RemoveArtifactsWithNullTags(stateWithNullTagsArtifacts StateReader) StateReader // GetArtifactByName takes in the name of the artifact and returns the artifact associated with it GetArtifactByNameAndTag(name, tag string) ArtifactReader + // SetArtifacts sets the artifacts in the state + SetArtifacts(artifacts []ArtifactReader) } type State struct { @@ -73,17 +73,6 @@ func (a *State) HasStateChanged(newState StateReader) bool { return false } -func (a *State) RemoveArtifactsWithNullTags(stateWithNullTagsArtifacts StateReader) StateReader { - var newArtifactsWithoutNullTags []Artifact - for _, artifact := range a.Artifacts { - if artifact.Tags != nil || len(artifact.Tags) != 0 { - newArtifactsWithoutNullTags = append(newArtifactsWithoutNullTags, artifact) - } - } - stateWithNullTagsArtifacts.(*State).Artifacts = newArtifactsWithoutNullTags - return stateWithNullTagsArtifacts -} - func (a *State) GetArtifactByNameAndTag(name, tag string) ArtifactReader { for i := range a.Artifacts { if a.Artifacts[i].GetName() == name { @@ -96,3 +85,14 @@ func (a *State) GetArtifactByNameAndTag(name, tag string) ArtifactReader { } return nil } + +func (a *State) SetArtifacts(artifacts []ArtifactReader) { + // Clear existing artifacts + a.Artifacts = []Artifact{} + + // Set new artifacts + a.Artifacts = make([]Artifact, len(artifacts)) + for i, artifact := range artifacts { + a.Artifacts[i] = *artifact.(*Artifact) + } +} diff --git a/internal/state/state_process.go b/internal/state/state_process.go index 591d1a1..f44f4c0 100644 --- a/internal/state/state_process.go +++ b/internal/state/state_process.go @@ -4,17 +4,17 @@ import ( "context" "encoding/json" "fmt" - "os" "sync" "container-registry.com/harbor-satellite/internal/notifier" + "container-registry.com/harbor-satellite/internal/utils" "container-registry.com/harbor-satellite/logger" "github.com/rs/zerolog" ) const FetchAndReplicateStateProcessName string = "fetch-replicate-state-process" -const DefaultFetchAndReplicateStateTimePeriod string = "00h00m05s" +const DefaultFetchAndReplicateStateTimePeriod string = "00h00m010s" type FetchAndReplicateAuthConfig struct { Username string @@ -57,44 +57,21 @@ func NewFetchAndReplicateStateProcess(id uint64, cronExpr string, stateFetcher S func (f *FetchAndReplicateStateProcess) Execute(ctx context.Context) error { log := logger.FromContext(ctx) - f.mu.Lock() - if f.IsRunning() { - f.mu.Unlock() - log.Warn().Msg("Process is already running") - return fmt.Errorf("process %s is already running", f.GetName()) + if !f.start() { + log.Warn().Msg("Process already running") + return fmt.Errorf("process %s already running", f.GetName()) } - log.Info().Msg("Starting process to fetch and replicate state") - f.isRunning = true - f.mu.Unlock() - defer func() { - f.mu.Lock() - f.isRunning = false - f.mu.Unlock() - }() - - newStateFetched, err := f.stateArtifactFetcher.FetchStateArtifact() + defer f.stop() + newStateFetched, err := f.FetchAndProcessState(log) if err != nil { - log.Error().Err(err).Msg("Error fetching state artifact") return err } - PrintPrettyJson(newStateFetched, log) log.Info().Msg("State fetched successfully") - log.Info().Msg("Checking if state has changed") - deleteEntity, replicateEntity, newState := f.GetChanges(newStateFetched, log) - log.Info().Msgf("Total artifacts to delete: %d", len(deleteEntity)) - for _, entity := range deleteEntity { - log.Info().Msgf("Artifact: %s, Tag: %s", entity.GetName(), entity.GetTags()[0]) - } - log.Info().Msgf("Total artifacts to replicate: %d", len(replicateEntity)) - log.Info().Msg("Artifacts to replicate:") - for _, entity := range replicateEntity { - log.Info().Msgf("Artifact: %s, Tag: %s", entity.GetName(), entity.GetTags()[0]) - } + f.LogChanges(deleteEntity, replicateEntity, log) if err := f.notifier.Notify(); err != nil { log.Error().Err(err).Msg("Error sending notification") } - replicator := NewBasicReplicator(f.authConfig.Username, f.authConfig.Password, f.authConfig.remoteRegistryURL, f.authConfig.sourceRegistry, f.authConfig.useUnsecure) // Delete the entities from the remote registry if err := replicator.DeleteReplicationEntity(ctx, deleteEntity); err != nil { @@ -111,39 +88,51 @@ func (f *FetchAndReplicateStateProcess) Execute(ctx context.Context) error { } func (f *FetchAndReplicateStateProcess) GetChanges(newState StateReader, log *zerolog.Logger) ([]ArtifactReader, []ArtifactReader, StateReader) { + log.Info().Msg("Getting changes") + var entityToDelete []ArtifactReader var entityToReplicate []ArtifactReader + if f.stateReader == nil { + log.Warn().Msg("Old state is nil") return entityToDelete, newState.GetArtifacts(), newState } - log.Warn().Msg("Old state reader is not nil") - PrintPrettyJson(f.stateReader, log) - // Remove all the artifacts from the new state reader whose tags are null to make sure if a tags image is updated then it is replicated - newState = f.stateReader.RemoveArtifactsWithNullTags(newState) - - newArtifacts := newState.GetArtifacts() - - for _, artifact := range newArtifacts { - log.Info().Msgf("Checking artifact: %s", artifact.GetName()) - // Check if this artifact is present in the old state reader or not - oldArtifact := f.stateReader.GetArtifactByNameAndTag(artifact.GetName(), artifact.GetTags()[0]) - if oldArtifact == nil { - log.Info().Msgf("Artifact: %s not present in the old state reader", artifact.GetName()) - // This artifact is not present in the old state reader, so we need to replicate it - entityToReplicate = append(entityToReplicate, artifact) - continue - } - // This artifact is present in the old state reader, so we need to check if it has changed or not by comparing the digest - if artifact.GetDigest() != oldArtifact.GetDigest() { - // This artifact has changed, so we need to replicate it - entityToReplicate = append(entityToReplicate, artifact) - // We also need to delete from the remote registry + + // Remove artifacts with null tags from the new state + newState = f.RemoveNullTagArtifacts(newState) + + // Create maps for quick lookups + oldArtifactsMap := make(map[string]ArtifactReader) + for _, oldArtifact := range f.stateReader.GetArtifacts() { + tag := oldArtifact.GetTags()[0] + oldArtifactsMap[oldArtifact.GetName()+"|"+tag] = oldArtifact + } + + // Check new artifacts and update lists + for _, newArtifact := range newState.GetArtifacts() { + nameTagKey := newArtifact.GetName() + "|" + newArtifact.GetTags()[0] + oldArtifact, exists := oldArtifactsMap[nameTagKey] + + if !exists { + // New artifact doesn't exist in old state, add to replication list + entityToReplicate = append(entityToReplicate, newArtifact) + } else if newArtifact.GetDigest() != oldArtifact.GetDigest() { + // Artifact exists but has changed, add to both lists + entityToReplicate = append(entityToReplicate, newArtifact) entityToDelete = append(entityToDelete, oldArtifact) } + + // Remove processed old artifact from map + delete(oldArtifactsMap, nameTagKey) } + + // Remaining artifacts in oldArtifactsMap should be deleted + for _, oldArtifact := range oldArtifactsMap { + entityToDelete = append(entityToDelete, oldArtifact) + } + return entityToDelete, entityToReplicate, newState } - func (f *FetchAndReplicateStateProcess) GetID() uint64 { return f.id } @@ -160,19 +149,69 @@ func (f *FetchAndReplicateStateProcess) IsRunning() bool { return f.isRunning } +func (f *FetchAndReplicateStateProcess) start() bool { + f.mu.Lock() + defer f.mu.Unlock() + if f.isRunning { + return false + } + f.isRunning = true + return true +} + +func (f *FetchAndReplicateStateProcess) stop() { + f.mu.Lock() + defer f.mu.Unlock() + f.isRunning = false +} + func (f *FetchAndReplicateStateProcess) RemoveNullTagArtifacts(state StateReader) StateReader { - newStateReader := state.RemoveArtifactsWithNullTags(state) - return newStateReader + var artifactsWithoutNullTags []ArtifactReader + for _, artifact := range state.GetArtifacts() { + if artifact.GetTags() != nil && len(artifact.GetTags()) != 0 { + artifactsWithoutNullTags = append(artifactsWithoutNullTags, artifact) + } + } + state.SetArtifacts(artifactsWithoutNullTags) + return state } -func PrintPrettyJson(info interface{}, log *zerolog.Logger) error { +func PrintPrettyJson(info interface{}, log *zerolog.Logger, message string) error { log.Warn().Msg("Printing pretty JSON") stateJSON, err := json.MarshalIndent(info, "", " ") if err != nil { log.Error().Err(err).Msg("Error marshalling state to JSON") return err } - log.Info().Msgf("Fetched state: %s", stateJSON) - os.Exit(0) + log.Info().Msgf("%s: %s", message, stateJSON) return nil } + +func ProcessState(state *StateReader) (*StateReader, error) { + for _, artifact := range (*state).GetArtifacts() { + repo, image, err := utils.GetRepositoryAndImageNameFromArtifact(artifact.GetRepository()) + if err != nil { + fmt.Printf("Error in getting repository and image name: %v", err) + return nil, err + } + artifact.SetRepository(repo) + artifact.SetName(image) + } + return state, nil +} + +func (f *FetchAndReplicateStateProcess) FetchAndProcessState(log *zerolog.Logger) (StateReader, error) { + state := NewState() + err := f.stateArtifactFetcher.FetchStateArtifact(&state) + if err != nil { + log.Error().Err(err).Msg("Error fetching state artifact") + return nil, err + } + ProcessState(&state) + return state, nil +} + +func (f *FetchAndReplicateStateProcess) LogChanges(deleteEntity, replicateEntity []ArtifactReader, log *zerolog.Logger) { + log.Warn().Msgf("Total artifacts to delete: %d", len(deleteEntity)) + log.Warn().Msgf("Total artifacts to replicate: %d", len(replicateEntity)) +} diff --git a/main.go b/main.go index 480a748..37e46a9 100644 --- a/main.go +++ b/main.go @@ -145,12 +145,6 @@ func processURLInput(input string, log *zerolog.Logger) (state.StateFetcher, err func processFileInput(log *zerolog.Logger) (state.StateFetcher, error) { stateArtifactFetcher := state.NewFileStateFetcher() - stateReader, err := stateArtifactFetcher.FetchStateArtifact() - if err != nil { - log.Error().Err(err).Msg("Error fetching state artifact from file") - return nil, err - } - config.SetRemoteRegistryURL(stateReader.GetRegistryURL()) return stateArtifactFetcher, nil } From 6e54a144f1ecc139c058c150735a211b8250d445 Mon Sep 17 00:00:00 2001 From: Mehul-Kumar-27 <202151092@iiitvadodara.ac.in> Date: Mon, 14 Oct 2024 02:20:19 +0530 Subject: [PATCH 13/36] fixing paning while removing the null tags --- dagger.json | 2 +- go.mod | 2 +- internal/state/state_process.go | 7 +++---- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/dagger.json b/dagger.json index b79d3a0..3328762 100644 --- a/dagger.json +++ b/dagger.json @@ -2,5 +2,5 @@ "name": "harbor-satellite", "sdk": "go", "source": "ci", - "engineVersion": "v0.13.3" + "engineVersion": "v0.13.0" } diff --git a/go.mod b/go.mod index 5847501..3df758c 100644 --- a/go.mod +++ b/go.mod @@ -397,7 +397,7 @@ require ( go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 - go.opentelemetry.io/otel/log v0.3.0 + go.opentelemetry.io/otel/log v0.7.0 go.opentelemetry.io/otel/metric v1.27.0 // indirect go.opentelemetry.io/otel/sdk v1.27.0 go.opentelemetry.io/otel/sdk/log v0.3.0 diff --git a/internal/state/state_process.go b/internal/state/state_process.go index f44f4c0..14bcc42 100644 --- a/internal/state/state_process.go +++ b/internal/state/state_process.go @@ -89,6 +89,8 @@ func (f *FetchAndReplicateStateProcess) Execute(ctx context.Context) error { func (f *FetchAndReplicateStateProcess) GetChanges(newState StateReader, log *zerolog.Logger) ([]ArtifactReader, []ArtifactReader, StateReader) { log.Info().Msg("Getting changes") + // Remove artifacts with null tags from the new state + newState = f.RemoveNullTagArtifacts(newState) var entityToDelete []ArtifactReader var entityToReplicate []ArtifactReader @@ -97,10 +99,7 @@ func (f *FetchAndReplicateStateProcess) GetChanges(newState StateReader, log *ze log.Warn().Msg("Old state is nil") return entityToDelete, newState.GetArtifacts(), newState } - - // Remove artifacts with null tags from the new state - newState = f.RemoveNullTagArtifacts(newState) - + // Create maps for quick lookups oldArtifactsMap := make(map[string]ArtifactReader) for _, oldArtifact := range f.stateReader.GetArtifacts() { From 8904e99c92c8f586093fd87b54acd748350b115a Mon Sep 17 00:00:00 2001 From: Mehul-Kumar-27 <202151092@iiitvadodara.ac.in> Date: Thu, 17 Oct 2024 02:02:02 +0530 Subject: [PATCH 14/36] using repository name instead of the image name while uploading the image to the zot --- internal/state/replicator.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/state/replicator.go b/internal/state/replicator.go index 24a97bd..879bc93 100644 --- a/internal/state/replicator.go +++ b/internal/state/replicator.go @@ -54,7 +54,7 @@ func (r *BasicReplicator) Replicate(ctx context.Context, replicationEntities []A log.Info().Msgf("Pulling image %s from repository %s at registry %s with tag %s", replicationEntity.GetName(), replicationEntity.GetRepository(), r.sourceRegistry, replicationEntity.GetTags()[0]) // Pull the image from the source registry - srcImage, err := crane.Pull(fmt.Sprintf("%s/%s/%s:%s", r.sourceRegistry, replicationEntity.GetName(), replicationEntity.GetName(), replicationEntity.GetTags()[0]), options...) + srcImage, err := crane.Pull(fmt.Sprintf("%s/%s/%s:%s", r.sourceRegistry, replicationEntity.GetName(), replicationEntity.GetRepository(), replicationEntity.GetTags()[0]), options...) if err != nil { log.Error().Msgf("Failed to pull image: %v", err) return err @@ -64,7 +64,7 @@ func (r *BasicReplicator) Replicate(ctx context.Context, replicationEntities []A ociImage := mutate.MediaType(srcImage, types.OCIManifestSchema1) // Push the converted OCI image to the Zot registry - err = crane.Push(ociImage, fmt.Sprintf("%s/%s/%s:%s", r.remoteRegistryURL, replicationEntity.GetName(), replicationEntity.GetName(), replicationEntity.GetTags()[0]), options...) + err = crane.Push(ociImage, fmt.Sprintf("%s/%s/%s:%s", r.remoteRegistryURL, replicationEntity.GetName(), replicationEntity.GetRepository(), replicationEntity.GetTags()[0]), options...) if err != nil { log.Error().Msgf("Failed to push image: %v", err) return err @@ -90,7 +90,7 @@ func (r *BasicReplicator) DeleteReplicationEntity(ctx context.Context, replicati for _, entity := range replicationEntity { log.Info().Msgf("Deleting image %s from repository %s at registry %s with tag %s", entity.GetName(), entity.GetRepository(), r.remoteRegistryURL, entity.GetTags()[0]) - err := crane.Delete(fmt.Sprintf("%s/%s/%s:%s", r.remoteRegistryURL, entity.GetName(), entity.GetName(), entity.GetTags()[0]), options...) + err := crane.Delete(fmt.Sprintf("%s/%s/%s:%s", r.remoteRegistryURL, entity.GetName(), entity.GetRepository(), entity.GetTags()[0]), options...) if err != nil { log.Error().Msgf("Failed to delete image: %v", err) return err From d14af7e3e3451f00f94717e2f36679bb36746b6c Mon Sep 17 00:00:00 2001 From: Mehul-Kumar-27 Date: Sun, 20 Oct 2024 14:10:20 +0530 Subject: [PATCH 15/36] adding container runtime config --- cmd/container_runtime/container_runtime.go | 1 + cmd/root.go | 172 +++++++++++++++++++++ go.mod | 2 +- go.sum | 3 + main.go | 152 +----------------- 5 files changed, 180 insertions(+), 150 deletions(-) create mode 100644 cmd/container_runtime/container_runtime.go create mode 100644 cmd/root.go diff --git a/cmd/container_runtime/container_runtime.go b/cmd/container_runtime/container_runtime.go new file mode 100644 index 0000000..7ccdf5f --- /dev/null +++ b/cmd/container_runtime/container_runtime.go @@ -0,0 +1 @@ +package runtime diff --git a/cmd/root.go b/cmd/root.go new file mode 100644 index 0000000..525ac3d --- /dev/null +++ b/cmd/root.go @@ -0,0 +1,172 @@ +package cmd + +import ( + "context" + "fmt" + "os/signal" + "syscall" + + "container-registry.com/harbor-satellite/internal/config" + "container-registry.com/harbor-satellite/internal/satellite" + "container-registry.com/harbor-satellite/internal/scheduler" + "container-registry.com/harbor-satellite/internal/server" + "container-registry.com/harbor-satellite/internal/state" + "container-registry.com/harbor-satellite/internal/utils" + "container-registry.com/harbor-satellite/logger" + "github.com/rs/zerolog" + "github.com/spf13/cobra" + "golang.org/x/sync/errgroup" +) + +func NewRootCommand() *cobra.Command { + rootCmd := &cobra.Command{ + Use: "harbor-satellite", + Short: "Harbor Satellite is a tool to replicate images from source registry to Harbor registry", + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + return config.InitConfig() + }, + RunE: func(cmd *cobra.Command, args []string) error { + return run() + }, + } + + return rootCmd +} + +func Execute() error { + return NewRootCommand().Execute() +} + +func run() error { + // Initialize Config and Logger + if err := initConfig(); err != nil { + return err + } + + ctx, cancel := setupContext() + defer cancel() + + g, ctx := errgroup.WithContext(ctx) + ctx = logger.AddLoggerToContext(ctx, config.GetLogLevel()) + log := logger.FromContext(ctx) + + // Set up router and app + app := setupServerApp(ctx, log) + app.SetupRoutes() + app.SetupServer(g) + + // Handle registry setup + if err := handleRegistrySetup(g, log, cancel); err != nil { + return err + } + scheduler := scheduler.NewBasicScheduler(ctx) + ctx = context.WithValue(ctx, scheduler.GetSchedulerKey(), scheduler) + err := scheduler.Start() + if err != nil { + log.Error().Err(err).Msg("Error starting scheduler") + return err + } + // Process Input (file or URL) + stateArtifactFetcher, err := processInput(ctx, log) + if err != nil || stateArtifactFetcher == nil { + return fmt.Errorf("error processing input: %w", err) + } + + satelliteService := satellite.NewSatellite(ctx, stateArtifactFetcher, scheduler.GetSchedulerKey()) + + g.Go(func() error { + return satelliteService.Run(ctx) + }) + + log.Info().Msg("Startup complete 🚀") + return g.Wait() +} + +func initConfig() error { + if err := config.InitConfig(); err != nil { + return fmt.Errorf("error initializing config: %w", err) + } + return nil +} + +func setupContext() (context.Context, context.CancelFunc) { + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) + return ctx, cancel +} + +func setupServerApp(ctx context.Context, log *zerolog.Logger) *server.App { + router := server.NewDefaultRouter("/api/v1") + router.Use(server.LoggingMiddleware) + + return server.NewApp( + router, + ctx, + log, + config.AppConfig, + &server.MetricsRegistrar{}, + &server.DebugRegistrar{}, + &satellite.SatelliteRegistrar{}, + ) +} + +func handleRegistrySetup(g *errgroup.Group, log *zerolog.Logger, cancel context.CancelFunc) error { + if config.GetOwnRegistry() { + if err := utils.HandleOwnRegistry(); err != nil { + log.Error().Err(err).Msg("Error handling own registry") + return err + } + } else { + log.Info().Msg("Launching default registry") + g.Go(func() error { + if err := utils.LaunchDefaultZotRegistry(); err != nil { + log.Error().Err(err).Msg("Error launching default registry") + cancel() + return err + } + cancel() + return nil + }) + } + return nil +} + +func processInput(ctx context.Context, log *zerolog.Logger) (state.StateFetcher, error) { + input := config.GetInput() + + if utils.IsValidURL(input) { + return processURLInput(input, log) + } + + log.Info().Msg("Input is not a valid URL, checking if it is a file path") + if err := validateFilePath(input, log); err != nil { + return nil, err + } + + return processFileInput(log) +} + +func processURLInput(input string, log *zerolog.Logger) (state.StateFetcher, error) { + log.Info().Msg("Input is a valid URL") + config.SetRemoteRegistryURL(input) + + stateArtifactFetcher := state.NewURLStateFetcher() + + return stateArtifactFetcher, nil +} + +func processFileInput(log *zerolog.Logger) (state.StateFetcher, error) { + stateArtifactFetcher := state.NewFileStateFetcher() + return stateArtifactFetcher, nil +} + +func validateFilePath(path string, log *zerolog.Logger) error { + if utils.HasInvalidPathChars(path) { + log.Error().Msg("Path contains invalid characters") + return fmt.Errorf("invalid file path: %s", path) + } + if err := utils.GetAbsFilePath(path); err != nil { + log.Error().Err(err).Msg("No file found") + return fmt.Errorf("no file found: %s", path) + } + return nil +} diff --git a/go.mod b/go.mod index 3df758c..d5a3b40 100644 --- a/go.mod +++ b/go.mod @@ -315,7 +315,7 @@ require ( github.com/owenrumney/go-sarif/v2 v2.3.1 // indirect github.com/owenrumney/squealer v1.2.2 // indirect github.com/package-url/packageurl-go v0.1.3 // indirect - github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/pelletier/go-toml/v2 v2.2.3 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect diff --git a/go.sum b/go.sum index 9705caf..1edfe5a 100644 --- a/go.sum +++ b/go.sum @@ -1236,8 +1236,11 @@ github.com/package-url/packageurl-go v0.1.3 h1:4juMED3hHiz0set3Vq3KeQ75KD1avthoX github.com/package-url/packageurl-go v0.1.3/go.mod h1:nKAWB8E6uk1MHqiS/lQb9pYBGH2+mdJ2PJc2s50dQY0= github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= +github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 h1:Ii+DKncOVM8Cu1Hc+ETb5K+23HdAMvESYE3ZJ5b5cMI= diff --git a/main.go b/main.go index 37e46a9..ba941f4 100644 --- a/main.go +++ b/main.go @@ -1,161 +1,15 @@ package main import ( - "context" "fmt" "os" - "os/signal" - "syscall" - "container-registry.com/harbor-satellite/internal/config" - "container-registry.com/harbor-satellite/internal/satellite" - "container-registry.com/harbor-satellite/internal/scheduler" - "container-registry.com/harbor-satellite/internal/server" - "container-registry.com/harbor-satellite/internal/state" - "container-registry.com/harbor-satellite/internal/utils" - "container-registry.com/harbor-satellite/logger" - "golang.org/x/sync/errgroup" - - "github.com/rs/zerolog" + "container-registry.com/harbor-satellite/cmd" ) func main() { - if err := run(); err != nil { - fmt.Fprintf(os.Stderr, "Error: %v\n", err) - os.Exit(1) - } -} - -func run() error { - // Initialize Config and Logger - if err := initConfig(); err != nil { - return err - } - - ctx, cancel := setupContext() - defer cancel() - - g, ctx := errgroup.WithContext(ctx) - ctx = logger.AddLoggerToContext(ctx, config.GetLogLevel()) - log := logger.FromContext(ctx) - - // Set up router and app - app := setupServerApp(ctx, log) - app.SetupRoutes() - app.SetupServer(g) - - // Handle registry setup - if err := handleRegistrySetup(g, log, cancel); err != nil { - return err - } - scheduler := scheduler.NewBasicScheduler(ctx) - ctx = context.WithValue(ctx, scheduler.GetSchedulerKey(), scheduler) - err := scheduler.Start() + err := cmd.Execute() if err != nil { - log.Error().Err(err).Msg("Error starting scheduler") - return err - } - // Process Input (file or URL) - stateArtifactFetcher, err := processInput(ctx, log) - if err != nil || stateArtifactFetcher == nil { - return fmt.Errorf("error processing input: %w", err) - } - - satelliteService := satellite.NewSatellite(ctx, stateArtifactFetcher, scheduler.GetSchedulerKey()) - - g.Go(func() error { - return satelliteService.Run(ctx) - }) - - log.Info().Msg("Startup complete 🚀") - return g.Wait() -} - -func initConfig() error { - if err := config.InitConfig(); err != nil { - return fmt.Errorf("error initializing config: %w", err) - } - return nil -} - -func setupContext() (context.Context, context.CancelFunc) { - ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) - return ctx, cancel -} - -func setupServerApp(ctx context.Context, log *zerolog.Logger) *server.App { - router := server.NewDefaultRouter("/api/v1") - router.Use(server.LoggingMiddleware) - - return server.NewApp( - router, - ctx, - log, - config.AppConfig, - &server.MetricsRegistrar{}, - &server.DebugRegistrar{}, - &satellite.SatelliteRegistrar{}, - ) -} - -func handleRegistrySetup(g *errgroup.Group, log *zerolog.Logger, cancel context.CancelFunc) error { - if config.GetOwnRegistry() { - if err := utils.HandleOwnRegistry(); err != nil { - log.Error().Err(err).Msg("Error handling own registry") - return err - } - } else { - log.Info().Msg("Launching default registry") - g.Go(func() error { - if err := utils.LaunchDefaultZotRegistry(); err != nil { - log.Error().Err(err).Msg("Error launching default registry") - cancel() - return err - } - cancel() - return nil - }) - } - return nil -} - -func processInput(ctx context.Context, log *zerolog.Logger) (state.StateFetcher, error) { - input := config.GetInput() - - if utils.IsValidURL(input) { - return processURLInput(input, log) - } - - log.Info().Msg("Input is not a valid URL, checking if it is a file path") - if err := validateFilePath(input, log); err != nil { - return nil, err - } - - return processFileInput(log) -} - -func processURLInput(input string, log *zerolog.Logger) (state.StateFetcher, error) { - log.Info().Msg("Input is a valid URL") - config.SetRemoteRegistryURL(input) - - stateArtifactFetcher := state.NewURLStateFetcher() - - return stateArtifactFetcher, nil -} - -func processFileInput(log *zerolog.Logger) (state.StateFetcher, error) { - stateArtifactFetcher := state.NewFileStateFetcher() - return stateArtifactFetcher, nil -} - -func validateFilePath(path string, log *zerolog.Logger) error { - if utils.HasInvalidPathChars(path) { - log.Error().Msg("Path contains invalid characters") - return fmt.Errorf("invalid file path: %s", path) - } - if err := utils.GetAbsFilePath(path); err != nil { - log.Error().Err(err).Msg("No file found") - return fmt.Errorf("no file found: %s", path) + fmt.Fprintf(os.Stderr, "Error: %v\n", err) } - return nil } From 7254c1beffd3856df096b3061d897a6deffbfbbb Mon Sep 17 00:00:00 2001 From: Mehul-Kumar-27 <202151092@iiitvadodara.ac.in> Date: Tue, 22 Oct 2024 01:47:50 +0530 Subject: [PATCH 16/36] containerd function and changing the harbor satellite to a cobra cli application --- ci/utils.go | 1 - cmd/container_runtime/container_runtime.go | 1 - cmd/container_runtime/containerd.go | 24 ++++++++++ cmd/container_runtime/generate.go | 18 +++++++ cmd/container_runtime/read_config.go | 47 +++++++++++++++++++ cmd/root.go | 30 ++++-------- go.mod | 13 +++-- go.sum | 7 ++- internal/config/config.go | 4 ++ internal/utils/utils.go | 30 ++++++++++++ registry/default_config.go | 46 ++++++++++++++++++ value/io.containerd.metadata.v1.bolt/meta.db | Bin 0 -> 32768 bytes 12 files changed, 192 insertions(+), 29 deletions(-) delete mode 100644 cmd/container_runtime/container_runtime.go create mode 100644 cmd/container_runtime/containerd.go create mode 100644 cmd/container_runtime/generate.go create mode 100644 cmd/container_runtime/read_config.go create mode 100644 registry/default_config.go create mode 100644 value/io.containerd.metadata.v1.bolt/meta.db diff --git a/ci/utils.go b/ci/utils.go index 954ec1a..cde501a 100644 --- a/ci/utils.go +++ b/ci/utils.go @@ -58,7 +58,6 @@ func (m *HarborSatellite) Service( AsService() } - // builds given component from source func (m *HarborSatellite) build(source *dagger.Directory, component string) *dagger.Directory { fmt.Printf("Building %s\n", component) diff --git a/cmd/container_runtime/container_runtime.go b/cmd/container_runtime/container_runtime.go deleted file mode 100644 index 7ccdf5f..0000000 --- a/cmd/container_runtime/container_runtime.go +++ /dev/null @@ -1 +0,0 @@ -package runtime diff --git a/cmd/container_runtime/containerd.go b/cmd/container_runtime/containerd.go new file mode 100644 index 0000000..1010fad --- /dev/null +++ b/cmd/container_runtime/containerd.go @@ -0,0 +1,24 @@ +package runtime + +import ( + + containerd "github.com/containerd/containerd/pkg/cri/config" + "github.com/spf13/cobra" +) + +func NewContainerdCommand() *cobra.Command { + containerd := &cobra.Command{ + Use: "containerd", + Short: "Creates the config file for the containerd runtime to fetch the images from the local repository", + RunE: func(cmd *cobra.Command, args []string) error { + // Step 1: Generate a default config for the containerd + config := containerd.Config{} + // Step 2: We have generate the default plugin config for the containerd config + config.PluginConfig = containerd.DefaultConfig() + // Step 3: Now that we have the default plugin config, we need to generate the necessary registry config + return nil + }, + } + containerd.AddCommand(NewReadConfigCommand("containerd")) + return containerd +} diff --git a/cmd/container_runtime/generate.go b/cmd/container_runtime/generate.go new file mode 100644 index 0000000..9d56528 --- /dev/null +++ b/cmd/container_runtime/generate.go @@ -0,0 +1,18 @@ +package runtime + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +func NewGenerateConfig(runtime string) *cobra.Command { + generateConfig := &cobra.Command{ + Use: "gen", + Short: fmt.Sprintf("Generates the config file for the %s runtime", runtime), + RunE: func(cmd *cobra.Command, args []string) error { + return nil + }, + } + return generateConfig +} diff --git a/cmd/container_runtime/read_config.go b/cmd/container_runtime/read_config.go new file mode 100644 index 0000000..a934c8f --- /dev/null +++ b/cmd/container_runtime/read_config.go @@ -0,0 +1,47 @@ +package runtime + +import ( + "fmt" + "path/filepath" + + "container-registry.com/harbor-satellite/internal/utils" + "container-registry.com/harbor-satellite/logger" + "github.com/spf13/cobra" +) + +var ( + DefaultContainerdConfigPath string = filepath.Join("/", "etc/containerd/config.toml") +) + +func NewReadConfigCommand(runtime string) *cobra.Command { + var defaultPath string + switch runtime { + case "containerd": + defaultPath = DefaultContainerdConfigPath + default: + defaultPath = "" + } + readContainerdConfig := &cobra.Command{ + Use: "read", + Short: fmt.Sprintf("Reads the config file for the %s runtime", runtime), + PersistentPreRun: func(cmd *cobra.Command, args []string) { + utils.SetupContextForCommand(cmd) + }, + RunE: func(cmd *cobra.Command, args []string) error { + //Parse the flags + path, err := cmd.Flags().GetString("path") + if err != nil { + return fmt.Errorf("error reading the path flag: %v", err) + } + log := logger.FromContext(cmd.Context()) + log.Info().Msgf("Reading the containerd config file from path: %s", path) + err = utils.ReadFile(path) + if err != nil { + return fmt.Errorf("error reading the containerd config file: %v", err) + } + return nil + }, + } + readContainerdConfig.Flags().StringP("path", "p", defaultPath, "Path to the containerd config file of the container runtime") + return readContainerdConfig +} diff --git a/cmd/root.go b/cmd/root.go index 525ac3d..e175d1d 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -3,9 +3,8 @@ package cmd import ( "context" "fmt" - "os/signal" - "syscall" + runtime "container-registry.com/harbor-satellite/cmd/container_runtime" "container-registry.com/harbor-satellite/internal/config" "container-registry.com/harbor-satellite/internal/satellite" "container-registry.com/harbor-satellite/internal/scheduler" @@ -22,14 +21,17 @@ func NewRootCommand() *cobra.Command { rootCmd := &cobra.Command{ Use: "harbor-satellite", Short: "Harbor Satellite is a tool to replicate images from source registry to Harbor registry", - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - return config.InitConfig() + PersistentPreRun: func(cmd *cobra.Command, args []string) { + config.InitConfig() }, RunE: func(cmd *cobra.Command, args []string) error { - return run() + ctx := cmd.Context() + ctx, cancel := utils.SetupContext(ctx) + ctx = logger.AddLoggerToContext(ctx, config.GetLogLevel()) + return run(ctx, cancel) }, } - + rootCmd.AddCommand(runtime.NewContainerdCommand()) return rootCmd } @@ -37,17 +39,8 @@ func Execute() error { return NewRootCommand().Execute() } -func run() error { - // Initialize Config and Logger - if err := initConfig(); err != nil { - return err - } - - ctx, cancel := setupContext() - defer cancel() - +func run(ctx context.Context, cancel context.CancelFunc) error { g, ctx := errgroup.WithContext(ctx) - ctx = logger.AddLoggerToContext(ctx, config.GetLogLevel()) log := logger.FromContext(ctx) // Set up router and app @@ -89,11 +82,6 @@ func initConfig() error { return nil } -func setupContext() (context.Context, context.CancelFunc) { - ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) - return ctx, cancel -} - func setupServerApp(ctx context.Context, log *zerolog.Logger) *server.App { router := server.NewDefaultRouter("/api/v1") router.Use(server.LoggingMiddleware) diff --git a/go.mod b/go.mod index d5a3b40..58efc67 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,12 @@ require ( github.com/stretchr/testify v1.9.0 ) -require golang.org/x/oauth2 v0.22.0 // indirect +require ( + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/pelletier/go-toml v1.9.5 // indirect + golang.org/x/oauth2 v0.22.0 // indirect + k8s.io/cri-api v0.27.1 // indirect +) require ( cloud.google.com/go v0.112.1 // indirect @@ -124,7 +129,7 @@ require ( github.com/cheggaaa/pb/v3 v3.1.5 // indirect github.com/cloudflare/circl v1.3.7 // indirect github.com/containerd/cgroups/v3 v3.0.3 // indirect - github.com/containerd/containerd v1.7.18 // indirect + github.com/containerd/containerd v1.7.18 github.com/containerd/continuity v0.4.3 // indirect github.com/containerd/errdefs v0.1.0 // indirect github.com/containerd/fifo v1.1.0 // indirect @@ -315,7 +320,7 @@ require ( github.com/owenrumney/go-sarif/v2 v2.3.1 // indirect github.com/owenrumney/squealer v1.2.2 // indirect github.com/package-url/packageurl-go v0.1.3 // indirect - github.com/pelletier/go-toml/v2 v2.2.3 // indirect + github.com/pelletier/go-toml/v2 v2.2.3 github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect @@ -353,7 +358,7 @@ require ( github.com/spdx/tools-golang v0.5.4 // indirect github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect - github.com/spf13/cobra v1.8.0 // indirect + github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 // indirect github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6 // indirect github.com/stretchr/objx v0.5.2 // indirect diff --git a/go.sum b/go.sum index 1edfe5a..30cf763 100644 --- a/go.sum +++ b/go.sum @@ -451,6 +451,8 @@ github.com/bitnami/go-version v0.0.0-20231130084017-bb00604d650c h1:C4UZIaS+HAw+ github.com/bitnami/go-version v0.0.0-20231130084017-bb00604d650c/go.mod h1:9iglf1GG4oNRJ39bZ5AZrjgAFD2RwQbXw6Qf7Cs47wo= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bmatcuk/doublestar/v4 v4.6.1 h1:FH9SifrbvJhnlQpztAx++wlkk70QBf0iBWDwNy7PA4I= github.com/bmatcuk/doublestar/v4 v4.6.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= @@ -1237,8 +1239,7 @@ github.com/package-url/packageurl-go v0.1.3/go.mod h1:nKAWB8E6uk1MHqiS/lQb9pYBGH github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= -github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= -github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= @@ -2220,6 +2221,8 @@ k8s.io/client-go v0.30.0 h1:sB1AGGlhY/o7KCyCEQ0bPWzYDL0pwOZO4vAtTSh/gJQ= k8s.io/client-go v0.30.0/go.mod h1:g7li5O5256qe6TYdAMyX/otJqMhIiGgTapdLchhmOaY= k8s.io/component-base v0.30.0 h1:cj6bp38g0ainlfYtaOQuRELh5KSYjhKxM+io7AUIk4o= k8s.io/component-base v0.30.0/go.mod h1:V9x/0ePFNaKeKYA3bOvIbrNoluTSG+fSJKjLdjOoeXQ= +k8s.io/cri-api v0.27.1 h1:KWO+U8MfI9drXB/P4oU9VchaWYOlwDglJZVHWMpTT3Q= +k8s.io/cri-api v0.27.1/go.mod h1:+Ts/AVYbIo04S86XbTD73UPp/DkTiYxtsFeOFEu32L0= k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= diff --git a/internal/config/config.go b/internal/config/config.go index d4a9946..4717e2d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -8,6 +8,10 @@ import ( "github.com/spf13/viper" ) +func init(){ + InitConfig() +} + var AppConfig *Config type Config struct { diff --git a/internal/utils/utils.go b/internal/utils/utils.go index 92dcefa..4126b8a 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -1,17 +1,22 @@ package utils import ( + "context" "errors" "fmt" "net" "net/url" "os" + "os/signal" "path/filepath" "strconv" "strings" + "syscall" "container-registry.com/harbor-satellite/internal/config" + "container-registry.com/harbor-satellite/logger" "container-registry.com/harbor-satellite/registry" + "github.com/spf13/cobra" ) // / ValidateRegistryAddress validates the registry address and port and returns the URL @@ -116,6 +121,17 @@ func FormatDuration(input string) (string, error) { return result, nil } +func SetupContext(context context.Context) (context.Context, context.CancelFunc) { + ctx, cancel := signal.NotifyContext(context, syscall.SIGTERM, syscall.SIGINT) + return ctx, cancel +} + +func SetupContextForCommand(cmd *cobra.Command) { + ctx := cmd.Context() + ctx = logger.AddLoggerToContext(ctx, config.GetLogLevel()) + cmd.SetContext(ctx) +} + // FormatRegistryURL formats the registry URL by trimming the "https://" or "http://" prefix if present func FormatRegistryURL(url string) string { // Trim the "https://" or "http://" prefix if present @@ -123,3 +139,17 @@ func FormatRegistryURL(url string) string { url = strings.TrimPrefix(url, "http://") return url } + +func ReadFile(path string) error { + data, err := os.ReadFile(path) + if err != nil { + return err + } + content := string(data) + lines := strings.Split(content, "\n") + fmt.Print("\n") + for i, line := range lines { + fmt.Printf("%5d | %s\n", i+1, line) + } + return nil +} diff --git a/registry/default_config.go b/registry/default_config.go new file mode 100644 index 0000000..8aeef3e --- /dev/null +++ b/registry/default_config.go @@ -0,0 +1,46 @@ +package registry + +import ( + "encoding/json" + "fmt" + "io" + "os" +) + +type DefaultZotConfig struct { + DistSpecVersion string `json:"distSpecVersion"` + Storage struct { + RootDirectory string `json:"rootDirectory"` + } `json:"storage"` + HTTP struct { + Address string `json:"address"` + Port string `json:"port"` + } `json:"http"` + Log struct { + Level string `json:"level"` + } `json:"log"` +} + +// ReadConfig reads a JSON file from the specified path and unmarshals it into a Config struct. +func ReadConfig(filePath string) (*DefaultZotConfig, error) { + file, err := os.Open(filePath) + if err != nil { + return nil, fmt.Errorf("could not open file: %w", err) + } + defer file.Close() + + // Read the file contents + bytes, err := io.ReadAll(file) + if err != nil { + return nil, fmt.Errorf("could not read file: %w", err) + } + + // Unmarshal the JSON into a Config struct + var config DefaultZotConfig + err = json.Unmarshal(bytes, &config) + if err != nil { + return nil, fmt.Errorf("could not unmarshal JSON: %w", err) + } + + return &config, nil +} diff --git a/value/io.containerd.metadata.v1.bolt/meta.db b/value/io.containerd.metadata.v1.bolt/meta.db new file mode 100644 index 0000000000000000000000000000000000000000..1884558ee4286fcdeaf8d5ccc7e4c32dd627dbc4 GIT binary patch literal 32768 zcmeI(tqlS(7y!`Y-ym26iDLxDz#uS?gK!GL8aNn%377%P4$J^8*9sB@K@swr^e1h< zqX+wcEYia(wY+v~KP{Lap-ga82o1PBlyK!5-N0t5&U$O8HPK3(_cZZ!b{ z1PBlyK!5-N0t5&UAn->5`Tjni1N_m}Y=r;;0t5&UAV7cs0RjXF5J&>~-G8&F{{)C- zd Date: Tue, 22 Oct 2024 03:00:55 +0530 Subject: [PATCH 17/36] generating config file for containerd --- cmd/container_runtime/containerd.go | 107 +++++++++++++++++++-- cmd/container_runtime/generate.go | 18 ---- runtime/containerd/config.toml | 144 ++++++++++++++++++++++++++++ 3 files changed, 242 insertions(+), 27 deletions(-) delete mode 100644 cmd/container_runtime/generate.go create mode 100644 runtime/containerd/config.toml diff --git a/cmd/container_runtime/containerd.go b/cmd/container_runtime/containerd.go index 1010fad..a506365 100644 --- a/cmd/container_runtime/containerd.go +++ b/cmd/container_runtime/containerd.go @@ -1,24 +1,113 @@ package runtime import ( + "fmt" + "os" + "path/filepath" + "container-registry.com/harbor-satellite/internal/config" + "container-registry.com/harbor-satellite/internal/utils" + "container-registry.com/harbor-satellite/registry" containerd "github.com/containerd/containerd/pkg/cri/config" + toml "github.com/pelletier/go-toml" "github.com/spf13/cobra" ) +const ( + ContainerDCertPath = "/etc/containerd/certs.d" + DefaultGeneratedTomlName = "config.toml" +) + +var DefaultGenPath string + +func init() { + cwd, err := os.Getwd() + if err != nil { + fmt.Printf("Error getting current working directory: %v\n", err) + DefaultGenPath = "/runtime/containerd" // Fallback in case of error + } else { + DefaultGenPath = filepath.Join(cwd, "runtime/containerd") + } +} + func NewContainerdCommand() *cobra.Command { - containerd := &cobra.Command{ + var generateConfig bool + var defaultZotConfig *registry.DefaultZotConfig + + containerdCmd := &cobra.Command{ Use: "containerd", Short: "Creates the config file for the containerd runtime to fetch the images from the local repository", - RunE: func(cmd *cobra.Command, args []string) error { - // Step 1: Generate a default config for the containerd - config := containerd.Config{} - // Step 2: We have generate the default plugin config for the containerd config - config.PluginConfig = containerd.DefaultConfig() - // Step 3: Now that we have the default plugin config, we need to generate the necessary registry config + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + var err error + if config.GetOwnRegistry() { + _, err = utils.ValidateRegistryAddress(config.GetOwnRegistryAdr(), config.GetOwnRegistryPort()) + if err != nil { + return err + } + defaultZotConfig.HTTP.Address = config.GetOwnRegistryAdr() + defaultZotConfig.HTTP.Port = config.GetOwnRegistryPort() + } else { + defaultZotConfig, err = registry.ReadConfig(config.GetZotConfigPath()) + if err != nil { + return fmt.Errorf("could not read config: %w", err) + } + } return nil }, + RunE: func(cmd *cobra.Command, args []string) error { + if !generateConfig { + fmt.Println("Skipping config generation as -gen flag is not set") + return nil + } + + return generateContainerdConfig(defaultZotConfig) + }, } - containerd.AddCommand(NewReadConfigCommand("containerd")) - return containerd + + containerdCmd.Flags().BoolVarP(&generateConfig, "gen", "g", false, "Generate the containerd config file") + + return containerdCmd +} + +func generateContainerdConfig(defaultZotConfig *registry.DefaultZotConfig) error { + containerdConfig := containerd.Config{} + containerdConfig.PluginConfig = containerd.DefaultConfig() + containerdConfig.PluginConfig.Registry.ConfigPath = ContainerDCertPath + + registryMirror := map[string]containerd.Mirror{ + defaultZotConfig.HTTP.Address: { + Endpoints: []string{defaultZotConfig.HTTP.Address + ":" + defaultZotConfig.HTTP.Port}, + }, + } + + registryConfig := map[string]containerd.RegistryConfig{ + defaultZotConfig.HTTP.Address: { + TLS: &containerd.TLSConfig{ + InsecureSkipVerify: config.UseUnsecure(), + }, + }, + } + + containerdConfig.PluginConfig.Registry.Mirrors = registryMirror + containerdConfig.PluginConfig.Registry.Configs = registryConfig + + generatedConfig, err := toml.Marshal(containerdConfig) + if err != nil { + return fmt.Errorf("could not marshal config: %w", err) + } + + filePath := filepath.Join(DefaultGenPath, DefaultGeneratedTomlName) + file, err := os.Create(filePath) + if err != nil { + return fmt.Errorf("could not create file: %w", err) + } + defer file.Close() + + _, err = file.Write(generatedConfig) + if err != nil { + return fmt.Errorf("could not write to file: %w", err) + } + + fmt.Printf("Config file created successfully at: %s\n", filePath) + return nil } diff --git a/cmd/container_runtime/generate.go b/cmd/container_runtime/generate.go deleted file mode 100644 index 9d56528..0000000 --- a/cmd/container_runtime/generate.go +++ /dev/null @@ -1,18 +0,0 @@ -package runtime - -import ( - "fmt" - - "github.com/spf13/cobra" -) - -func NewGenerateConfig(runtime string) *cobra.Command { - generateConfig := &cobra.Command{ - Use: "gen", - Short: fmt.Sprintf("Generates the config file for the %s runtime", runtime), - RunE: func(cmd *cobra.Command, args []string) error { - return nil - }, - } - return generateConfig -} diff --git a/runtime/containerd/config.toml b/runtime/containerd/config.toml new file mode 100644 index 0000000..40e4cf3 --- /dev/null +++ b/runtime/containerd/config.toml @@ -0,0 +1,144 @@ +ContainerdEndpoint = "" +ContainerdRootDir = "" +RootDir = "" +StateDir = "" +cdi_spec_dirs = ["/etc/cdi", "/var/run/cdi"] +device_ownership_from_security_context = false +disable_apparmor = false +disable_cgroup = false +disable_hugetlb_controller = true +disable_proc_mount = false +disable_tcp_service = true +drain_exec_sync_io_timeout = "0s" +enable_cdi = false +enable_selinux = false +enable_tls_streaming = false +enable_unprivileged_icmp = false +enable_unprivileged_ports = false +ignore_deprecation_warnings = [] +ignore_image_defined_volumes = false +image_pull_progress_timeout = "5m0s" +image_pull_with_sync_fs = false +max_concurrent_downloads = 3 +max_container_log_line_size = 16384 +netns_mounts_under_state_dir = false +restrict_oom_score_adj = false +sandbox_image = "registry.k8s.io/pause:3.8" +selinux_category_range = 1024 +stats_collect_period = 10 +stream_idle_timeout = "4h0m0s" +stream_server_address = "127.0.0.1" +stream_server_port = "0" +systemd_cgroup = false +tolerate_missing_hugetlb_controller = true +unset_seccomp_profile = "" + +[cni] + bin_dir = "/opt/cni/bin" + conf_dir = "/etc/cni/net.d" + conf_template = "" + ip_pref = "" + max_conf_num = 1 + setup_serially = false + +[containerd] + default_runtime_name = "runc" + disable_snapshot_annotations = true + discard_unpacked_layers = false + ignore_blockio_not_enabled_errors = false + ignore_rdt_not_enabled_errors = false + no_pivot = false + snapshotter = "overlayfs" + + [containerd.default_runtime] + base_runtime_spec = "" + cni_conf_dir = "" + cni_max_conf_num = 0 + container_annotations = [] + pod_annotations = [] + privileged_without_host_devices = false + privileged_without_host_devices_all_devices_allowed = false + runtime_engine = "" + runtime_path = "" + runtime_root = "" + runtime_type = "" + sandbox_mode = "" + snapshotter = "" + + [containerd.default_runtime.options] + + [containerd.runtimes] + + [containerd.runtimes.runc] + base_runtime_spec = "" + cni_conf_dir = "" + cni_max_conf_num = 0 + container_annotations = [] + pod_annotations = [] + privileged_without_host_devices = false + privileged_without_host_devices_all_devices_allowed = false + runtime_engine = "" + runtime_path = "" + runtime_root = "" + runtime_type = "io.containerd.runc.v2" + sandbox_mode = "podsandbox" + snapshotter = "" + + [containerd.runtimes.runc.options] + BinaryName = "" + CriuImagePath = "" + CriuPath = "" + CriuWorkPath = "" + IoGid = 0 + IoUid = 0 + NoNewKeyring = false + NoPivotRoot = false + Root = "" + ShimCgroup = "" + SystemdCgroup = false + + [containerd.untrusted_workload_runtime] + base_runtime_spec = "" + cni_conf_dir = "" + cni_max_conf_num = 0 + container_annotations = [] + pod_annotations = [] + privileged_without_host_devices = false + privileged_without_host_devices_all_devices_allowed = false + runtime_engine = "" + runtime_path = "" + runtime_root = "" + runtime_type = "" + sandbox_mode = "" + snapshotter = "" + + [containerd.untrusted_workload_runtime.options] + +[image_decryption] + key_model = "node" + +[registry] + config_path = "/etc/containerd/certs.d" + + [registry.auths] + + [registry.configs] + + [registry.configs."127.0.0.1"] + + [registry.configs."127.0.0.1".tls] + ca_file = "" + cert_file = "" + insecure_skip_verify = true + key_file = "" + + [registry.headers] + + [registry.mirrors] + + [registry.mirrors."127.0.0.1"] + endpoint = ["127.0.0.1:8585"] + +[x509_key_pair_streaming] + tls_cert_file = "" + tls_key_file = "" From 222c666c2422f728e93a825eaae4df595796b858 Mon Sep 17 00:00:00 2001 From: Mehul-Kumar-27 <202151092@iiitvadodara.ac.in> Date: Tue, 22 Oct 2024 03:11:51 +0530 Subject: [PATCH 18/36] adding better logging --- cmd/container_runtime/containerd.go | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/cmd/container_runtime/containerd.go b/cmd/container_runtime/containerd.go index a506365..dbfd83b 100644 --- a/cmd/container_runtime/containerd.go +++ b/cmd/container_runtime/containerd.go @@ -7,9 +7,11 @@ import ( "container-registry.com/harbor-satellite/internal/config" "container-registry.com/harbor-satellite/internal/utils" + "container-registry.com/harbor-satellite/logger" "container-registry.com/harbor-satellite/registry" containerd "github.com/containerd/containerd/pkg/cri/config" toml "github.com/pelletier/go-toml" + "github.com/rs/zerolog" "github.com/spf13/cobra" ) @@ -39,28 +41,35 @@ func NewContainerdCommand() *cobra.Command { Short: "Creates the config file for the containerd runtime to fetch the images from the local repository", PersistentPreRunE: func(cmd *cobra.Command, args []string) error { var err error + utils.SetupContextForCommand(cmd) + log := logger.FromContext(cmd.Context()) if config.GetOwnRegistry() { - _, err = utils.ValidateRegistryAddress(config.GetOwnRegistryAdr(), config.GetOwnRegistryPort()) + log.Info().Msg("Using own registry for config generation") + address, err := utils.ValidateRegistryAddress(config.GetOwnRegistryAdr(), config.GetOwnRegistryPort()) if err != nil { + log.Err(err).Msg("Error validating registry address") return err } + log.Info().Msgf("Registry address validated: %s", address) defaultZotConfig.HTTP.Address = config.GetOwnRegistryAdr() defaultZotConfig.HTTP.Port = config.GetOwnRegistryPort() } else { + log.Info().Msg("Using default registry for config generation") defaultZotConfig, err = registry.ReadConfig(config.GetZotConfigPath()) if err != nil { return fmt.Errorf("could not read config: %w", err) } + log.Info().Msgf("Default config read successfully: %v", defaultZotConfig.HTTP.Address+":"+defaultZotConfig.HTTP.Port) } return nil }, RunE: func(cmd *cobra.Command, args []string) error { + log := logger.FromContext(cmd.Context()) if !generateConfig { - fmt.Println("Skipping config generation as -gen flag is not set") return nil } - - return generateContainerdConfig(defaultZotConfig) + log.Info().Msg("Generating containerd config file for containerd ...") + return generateContainerdConfig(defaultZotConfig, log) }, } @@ -69,7 +78,7 @@ func NewContainerdCommand() *cobra.Command { return containerdCmd } -func generateContainerdConfig(defaultZotConfig *registry.DefaultZotConfig) error { +func generateContainerdConfig(defaultZotConfig *registry.DefaultZotConfig, log *zerolog.Logger) error { containerdConfig := containerd.Config{} containerdConfig.PluginConfig = containerd.DefaultConfig() containerdConfig.PluginConfig.Registry.ConfigPath = ContainerDCertPath @@ -93,21 +102,25 @@ func generateContainerdConfig(defaultZotConfig *registry.DefaultZotConfig) error generatedConfig, err := toml.Marshal(containerdConfig) if err != nil { + log.Error().Err(err).Msg("Error marshalling config") return fmt.Errorf("could not marshal config: %w", err) } filePath := filepath.Join(DefaultGenPath, DefaultGeneratedTomlName) + log.Info().Msgf("Writing config to file: %s", filePath) file, err := os.Create(filePath) if err != nil { + log.Err(err).Msg("Error creating file") return fmt.Errorf("could not create file: %w", err) } defer file.Close() _, err = file.Write(generatedConfig) if err != nil { + log.Err(err).Msg("Error writing to file") return fmt.Errorf("could not write to file: %w", err) } - fmt.Printf("Config file created successfully at: %s\n", filePath) + log.Info().Msgf("Config file generated successfully at: %s", filePath) return nil } From d2180bf0a10d8309c00d19298f68b74f4829b764 Mon Sep 17 00:00:00 2001 From: Mehul-Kumar-27 <202151092@iiitvadodara.ac.in> Date: Tue, 22 Oct 2024 03:13:38 +0530 Subject: [PATCH 19/36] fix --- cmd/container_runtime/containerd.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/container_runtime/containerd.go b/cmd/container_runtime/containerd.go index dbfd83b..7d7c9c6 100644 --- a/cmd/container_runtime/containerd.go +++ b/cmd/container_runtime/containerd.go @@ -69,7 +69,7 @@ func NewContainerdCommand() *cobra.Command { return nil } log.Info().Msg("Generating containerd config file for containerd ...") - return generateContainerdConfig(defaultZotConfig, log) + return GenerateContainerdConfig(defaultZotConfig, log) }, } @@ -78,7 +78,7 @@ func NewContainerdCommand() *cobra.Command { return containerdCmd } -func generateContainerdConfig(defaultZotConfig *registry.DefaultZotConfig, log *zerolog.Logger) error { +func GenerateContainerdConfig(defaultZotConfig *registry.DefaultZotConfig, log *zerolog.Logger) error { containerdConfig := containerd.Config{} containerdConfig.PluginConfig = containerd.DefaultConfig() containerdConfig.PluginConfig.Registry.ConfigPath = ContainerDCertPath From 97f8b01aba493cc9131e74fc939e67bef574ca01 Mon Sep 17 00:00:00 2001 From: Mehul-Kumar-27 <202151092@iiitvadodara.ac.in> Date: Tue, 29 Oct 2024 02:31:03 +0530 Subject: [PATCH 20/36] adding config generation for containerd --- .gitignore | 1 + cmd/container_runtime/containerd.go | 109 ++++++++++++++++++--------- cmd/container_runtime/host.go | 106 ++++++++++++++++++++++++++ cmd/container_runtime/read_config.go | 16 +--- internal/utils/folder.go | 24 ++++++ internal/utils/utils.go | 26 ++++++- registry/default_config.go | 4 + runtime/containerd/config.toml | 70 ++++++----------- 8 files changed, 255 insertions(+), 101 deletions(-) create mode 100644 cmd/container_runtime/host.go create mode 100644 internal/utils/folder.go diff --git a/.gitignore b/.gitignore index 32234d8..9095ac2 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,4 @@ secrets.txt __debug_bin1949266242 /zot +runtime/ diff --git a/cmd/container_runtime/containerd.go b/cmd/container_runtime/containerd.go index 7d7c9c6..09a9672 100644 --- a/cmd/container_runtime/containerd.go +++ b/cmd/container_runtime/containerd.go @@ -1,6 +1,7 @@ package runtime import ( + "context" "fmt" "os" "path/filepath" @@ -16,10 +17,17 @@ import ( ) const ( - ContainerDCertPath = "/etc/containerd/certs.d" - DefaultGeneratedTomlName = "config.toml" + ContainerDCertPath = "/etc/containerd/certs.d" + DefaultGeneratedTomlName = "config.toml" + ContainerdRuntime = "containerd" + DefaultContainerdConfigPath = "/etc/containerd/config.toml" ) +type ContainerdController interface { + Load(ctx context.Context, log *zerolog.Logger) (*registry.DefaultZotConfig, error) + Generate(ctx context.Context, configPath string, log *zerolog.Logger) error +} + var DefaultGenPath string func init() { @@ -35,6 +43,8 @@ func init() { func NewContainerdCommand() *cobra.Command { var generateConfig bool var defaultZotConfig *registry.DefaultZotConfig + var containerdConfigPath string + var containerDCertPath string containerdCmd := &cobra.Command{ Use: "containerd", @@ -61,34 +71,67 @@ func NewContainerdCommand() *cobra.Command { } log.Info().Msgf("Default config read successfully: %v", defaultZotConfig.HTTP.Address+":"+defaultZotConfig.HTTP.Port) } - return nil + return utils.CreateRuntimeDirectory(DefaultGenPath) }, RunE: func(cmd *cobra.Command, args []string) error { log := logger.FromContext(cmd.Context()) - if !generateConfig { - return nil + sourceRegistry := config.GetRemoteRegistryURL() + satelliteHostConfig := NewSatelliteHostConfig(defaultZotConfig.GetLocalRegistryURL(), sourceRegistry) + if generateConfig { + log.Info().Msg("Generating containerd config file for containerd ...") + log.Info().Msgf("Fetching containerd config from path path: %s", containerdConfigPath) + return GenerateContainerdHostConfig(containerDCertPath, DefaultGenPath, log, *satelliteHostConfig) } - log.Info().Msg("Generating containerd config file for containerd ...") - return GenerateContainerdConfig(defaultZotConfig, log) + return nil }, } containerdCmd.Flags().BoolVarP(&generateConfig, "gen", "g", false, "Generate the containerd config file") - + containerdCmd.PersistentFlags().StringVarP(&containerdConfigPath, "path", "p", DefaultContainerdConfigPath, "Path to the containerd config file of the container runtime") + containerdCmd.PersistentFlags().StringVarP(&containerDCertPath, "cert-path", "c", ContainerDCertPath, "Path to the containerd cert directory") + containerdCmd.AddCommand(NewReadConfigCommand(ContainerdRuntime)) return containerdCmd } -func GenerateContainerdConfig(defaultZotConfig *registry.DefaultZotConfig, log *zerolog.Logger) error { - containerdConfig := containerd.Config{} - containerdConfig.PluginConfig = containerd.DefaultConfig() - containerdConfig.PluginConfig.Registry.ConfigPath = ContainerDCertPath - +// GenerateConfig generates the containerd config file for the containerd runtime +// It takes the zot config a logger and the containerd config path +// It reads the containerd config file and adds the local registry to the config file +func GenerateConfig(defaultZotConfig *registry.DefaultZotConfig, log *zerolog.Logger, containerdConfigPath, containerdCertPath string) error { + // First Read the present config file at the configPath + data, err := utils.ReadFile(containerdConfigPath, false) + if err != nil { + log.Err(err).Msg("Error reading config file") + return fmt.Errorf("could not read config file: %w", err) + } + // Now we marshal the data into the containerd config + containerdConfig := &containerd.Config{} + err = toml.Unmarshal(data, containerdConfig) + if err != nil { + log.Err(err).Msg("Error unmarshalling config") + return fmt.Errorf("could not unmarshal config: %w", err) + } + // Steps to configure the containerd config: + // 1. Set the default registry config cert path + // -- This is the path where the certs of the registry are stored + // -- If the user has already has a cert path then we do not set it rather we would now use the + // user path as the default path + if containerdConfig.PluginConfig.Registry.ConfigPath == "" { + containerdConfig.PluginConfig.Registry.ConfigPath = containerdCertPath + } + log.Info().Msgf("Setting the registry cert path to: %s", containerdConfig.PluginConfig.Registry.ConfigPath) + // Now we add the local registry to the containerd config mirrors registryMirror := map[string]containerd.Mirror{ defaultZotConfig.HTTP.Address: { Endpoints: []string{defaultZotConfig.HTTP.Address + ":" + defaultZotConfig.HTTP.Port}, }, } - + if containerdConfig.PluginConfig.Registry.Mirrors == nil { + containerdConfig.PluginConfig.Registry.Mirrors = registryMirror + } else { + for key, value := range registryMirror { + containerdConfig.PluginConfig.Registry.Mirrors[key] = value + } + } registryConfig := map[string]containerd.RegistryConfig{ defaultZotConfig.HTTP.Address: { TLS: &containerd.TLSConfig{ @@ -96,31 +139,27 @@ func GenerateContainerdConfig(defaultZotConfig *registry.DefaultZotConfig, log * }, }, } - - containerdConfig.PluginConfig.Registry.Mirrors = registryMirror - containerdConfig.PluginConfig.Registry.Configs = registryConfig - - generatedConfig, err := toml.Marshal(containerdConfig) - if err != nil { - log.Error().Err(err).Msg("Error marshalling config") - return fmt.Errorf("could not marshal config: %w", err) + // Now we add the local registry to the containerd config registry + if containerdConfig.PluginConfig.Registry.Configs == nil { + containerdConfig.PluginConfig.Registry.Configs = registryConfig + } else { + for key, value := range registryConfig { + containerdConfig.PluginConfig.Registry.Configs[key] = value + } } - - filePath := filepath.Join(DefaultGenPath, DefaultGeneratedTomlName) - log.Info().Msgf("Writing config to file: %s", filePath) - file, err := os.Create(filePath) + // ToDo: Find a way to remove the unwanted configuration added to the config file while marshalling + pathToWrite := filepath.Join(DefaultGenPath, DefaultGeneratedTomlName) + log.Info().Msgf("Writing the containerd config to path: %s", pathToWrite) + // Now we write the config to the file + data, err = toml.Marshal(containerdConfig) if err != nil { - log.Err(err).Msg("Error creating file") - return fmt.Errorf("could not create file: %w", err) + log.Err(err).Msg("Error marshalling config") + return fmt.Errorf("could not marshal config: %w", err) } - defer file.Close() - - _, err = file.Write(generatedConfig) + err = utils.WriteFile(pathToWrite, data) if err != nil { - log.Err(err).Msg("Error writing to file") - return fmt.Errorf("could not write to file: %w", err) + log.Err(err).Msg("Error writing config to file") + return fmt.Errorf("could not write config to file: %w", err) } - - log.Info().Msgf("Config file generated successfully at: %s", filePath) return nil } diff --git a/cmd/container_runtime/host.go b/cmd/container_runtime/host.go new file mode 100644 index 0000000..af55057 --- /dev/null +++ b/cmd/container_runtime/host.go @@ -0,0 +1,106 @@ +package runtime + +import ( + "fmt" + "os" + + "container-registry.com/harbor-satellite/internal/config" + "container-registry.com/harbor-satellite/internal/utils" + "github.com/pelletier/go-toml/v2" + "github.com/rs/zerolog" +) + +const ( + DockerIoConfigPath = "docker" + HostToml = "host_gen.toml" + DefaultTomlConfigPath = "_default" + DockerURL = "https://registry-1.docker.io" +) + +type ContainerdHostConfig struct { + Server string `toml:"server,omitempty"` + Host map[string]HostConfig `toml:"host,omitempty"` +} + +type HostConfig struct { + Capabilities []string `toml:"capabilities,omitempty"` + CA interface{} `toml:"ca,omitempty"` + Client interface{} `toml:"client,omitempty"` + SkipVerify bool `toml:"skip_verify,omitempty"` + Header map[string][]string `toml:"header,omitempty"` + OverridePath bool `toml:"override_path,omitempty"` +} + +type SatelliteHostConfig struct { + LocalRegistry string + SourceRegistry string +} + +func NewSatelliteHostConfig(localRegistry, sourceRegistry string) *SatelliteHostConfig { + return &SatelliteHostConfig{ + LocalRegistry: localRegistry, + SourceRegistry: sourceRegistry, + } +} + +// GenerateContainerdHostConfig generates the host.toml file for containerd docker.io and also create a default config.toml file +func GenerateContainerdHostConfig(containerdCertPath, genPath string, log *zerolog.Logger, satelliteHostConfig SatelliteHostConfig) error { + mirrorGenPath := fmt.Sprintf("%s/%s", genPath, DockerIoConfigPath) + err := utils.CreateRuntimeDirectory(mirrorGenPath) + if err != nil { + log.Err(err).Msgf("Error creating the directory: %s", mirrorGenPath) + return fmt.Errorf("error creating the directory: %v", err) + } + dockerHubHostConfigPath := fmt.Sprintf("%s/%s/%s", containerdCertPath, DockerIoConfigPath, HostToml) + var dockerContainerdHostConfig ContainerdHostConfig + + // Read the `docker.io/host.toml` file if present + data, err := utils.ReadFile(dockerHubHostConfigPath, false) + if err != nil { + if os.IsNotExist(err) { + log.Warn().Msgf("The docker.io/host.toml file does not exist at path: %s", dockerHubHostConfigPath) + } else { + return fmt.Errorf("error reading the docker.io/host.toml file: %v", err) + } + } + err = toml.Unmarshal(data, &dockerContainerdHostConfig) + if err != nil { + log.Err(err).Msgf("Error unmarshalling the docker.io/host.toml file at path: %s", dockerHubHostConfigPath) + return fmt.Errorf("error unmarshalling the docker.io/host.toml file: %v", err) + } + satelliteHostConfigToAdd := ContainerdHostConfig{ + Host: map[string]HostConfig{ + satelliteHostConfig.LocalRegistry: { + Capabilities: []string{"pull", "push", "resolve"}, + SkipVerify: config.UseUnsecure(), + }, + }, + } + + if dockerContainerdHostConfig.Server == "" { + dockerContainerdHostConfig.Server = DockerURL + } + if dockerContainerdHostConfig.Host == nil { + dockerContainerdHostConfig.Host = satelliteHostConfigToAdd.Host + } else { + for key, value := range dockerContainerdHostConfig.Host { + satelliteHostConfigToAdd.Host[key] = value + } + dockerContainerdHostConfig.Host = satelliteHostConfigToAdd.Host + } + + pathTOWrite := fmt.Sprintf("%s/%s", mirrorGenPath, HostToml) + log.Info().Msgf("Writing the host.toml file at path: %s", pathTOWrite) + hostData, err := toml.Marshal(dockerContainerdHostConfig) + if err != nil { + log.Err(err).Msg("Error marshalling the host.toml file") + return fmt.Errorf("error marshalling the host.toml file: %v", err) + } + err = utils.WriteFile(pathTOWrite, hostData) + if err != nil { + log.Err(err).Msg("Error writing the host.toml file") + return fmt.Errorf("error writing the host.toml file: %v", err) + } + log.Info().Msg("Successfully wrote the host.toml file") + return nil +} diff --git a/cmd/container_runtime/read_config.go b/cmd/container_runtime/read_config.go index a934c8f..a815baf 100644 --- a/cmd/container_runtime/read_config.go +++ b/cmd/container_runtime/read_config.go @@ -2,25 +2,14 @@ package runtime import ( "fmt" - "path/filepath" + "container-registry.com/harbor-satellite/internal/utils" "container-registry.com/harbor-satellite/logger" "github.com/spf13/cobra" ) -var ( - DefaultContainerdConfigPath string = filepath.Join("/", "etc/containerd/config.toml") -) - func NewReadConfigCommand(runtime string) *cobra.Command { - var defaultPath string - switch runtime { - case "containerd": - defaultPath = DefaultContainerdConfigPath - default: - defaultPath = "" - } readContainerdConfig := &cobra.Command{ Use: "read", Short: fmt.Sprintf("Reads the config file for the %s runtime", runtime), @@ -35,13 +24,12 @@ func NewReadConfigCommand(runtime string) *cobra.Command { } log := logger.FromContext(cmd.Context()) log.Info().Msgf("Reading the containerd config file from path: %s", path) - err = utils.ReadFile(path) + _, err = utils.ReadFile(path, true) if err != nil { return fmt.Errorf("error reading the containerd config file: %v", err) } return nil }, } - readContainerdConfig.Flags().StringP("path", "p", defaultPath, "Path to the containerd config file of the container runtime") return readContainerdConfig } diff --git a/internal/utils/folder.go b/internal/utils/folder.go new file mode 100644 index 0000000..88b54bb --- /dev/null +++ b/internal/utils/folder.go @@ -0,0 +1,24 @@ +package utils + +import ( + "fmt" + "os" + "path/filepath" +) + +func CreateRuntimeDirectory(dir string) error { + cwd, err := os.Getwd() + if err != nil { + return fmt.Errorf("failed to get current working directory: %v", err) + } + runtimePath := filepath.Join(cwd, dir) + //check if the runtime directory exists + if _, err := os.Stat(runtimePath); os.IsNotExist(err) { + //create the runtime directory + err = os.MkdirAll(dir, 0755) + if err != nil { + return fmt.Errorf("failed to create the runtime directory: %v, please create a folder %v", err, dir) + } + } + return nil +} diff --git a/internal/utils/utils.go b/internal/utils/utils.go index 4126b8a..8b46b2f 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -140,16 +140,36 @@ func FormatRegistryURL(url string) string { return url } -func ReadFile(path string) error { +func ReadFile(path string, shouldPrint bool) ([]byte, error) { data, err := os.ReadFile(path) if err != nil { - return err + return nil, err + } + if shouldPrint { + PrintData(string(data)) } - content := string(data) + return data, nil +} + +// PrintData prints the content of a file line by line +func PrintData(content string) { lines := strings.Split(content, "\n") fmt.Print("\n") for i, line := range lines { fmt.Printf("%5d | %s\n", i+1, line) } +} + +// WriteFile takes the path and the data wand write the data to the file +func WriteFile(path string, data []byte) error { + file, err := os.Create(path) + if err != nil { + return fmt.Errorf("error creating file :%s", err) + } + defer file.Close() + _, err = file.Write(data) + if err != nil { + return fmt.Errorf("error while write to the file :%s", err) + } return nil } diff --git a/registry/default_config.go b/registry/default_config.go index 8aeef3e..0bf8ee4 100644 --- a/registry/default_config.go +++ b/registry/default_config.go @@ -21,6 +21,10 @@ type DefaultZotConfig struct { } `json:"log"` } +func (c *DefaultZotConfig) GetLocalRegistryURL() string { + return fmt.Sprintf("%s:%s", c.HTTP.Address, c.HTTP.Port) +} + // ReadConfig reads a JSON file from the specified path and unmarshals it into a Config struct. func ReadConfig(filePath string) (*DefaultZotConfig, error) { file, err := os.Open(filePath) diff --git a/runtime/containerd/config.toml b/runtime/containerd/config.toml index 40e4cf3..5b005ab 100644 --- a/runtime/containerd/config.toml +++ b/runtime/containerd/config.toml @@ -2,14 +2,14 @@ ContainerdEndpoint = "" ContainerdRootDir = "" RootDir = "" StateDir = "" -cdi_spec_dirs = ["/etc/cdi", "/var/run/cdi"] +cdi_spec_dirs = [] device_ownership_from_security_context = false disable_apparmor = false disable_cgroup = false -disable_hugetlb_controller = true +disable_hugetlb_controller = false disable_proc_mount = false -disable_tcp_service = true -drain_exec_sync_io_timeout = "0s" +disable_tcp_service = false +drain_exec_sync_io_timeout = "" enable_cdi = false enable_selinux = false enable_tls_streaming = false @@ -17,38 +17,38 @@ enable_unprivileged_icmp = false enable_unprivileged_ports = false ignore_deprecation_warnings = [] ignore_image_defined_volumes = false -image_pull_progress_timeout = "5m0s" +image_pull_progress_timeout = "" image_pull_with_sync_fs = false -max_concurrent_downloads = 3 -max_container_log_line_size = 16384 +max_concurrent_downloads = 0 +max_container_log_line_size = 0 netns_mounts_under_state_dir = false restrict_oom_score_adj = false -sandbox_image = "registry.k8s.io/pause:3.8" -selinux_category_range = 1024 -stats_collect_period = 10 -stream_idle_timeout = "4h0m0s" -stream_server_address = "127.0.0.1" -stream_server_port = "0" +sandbox_image = "" +selinux_category_range = 0 +stats_collect_period = 0 +stream_idle_timeout = "" +stream_server_address = "" +stream_server_port = "" systemd_cgroup = false -tolerate_missing_hugetlb_controller = true +tolerate_missing_hugetlb_controller = false unset_seccomp_profile = "" [cni] - bin_dir = "/opt/cni/bin" - conf_dir = "/etc/cni/net.d" + bin_dir = "" + conf_dir = "" conf_template = "" ip_pref = "" - max_conf_num = 1 + max_conf_num = 0 setup_serially = false [containerd] - default_runtime_name = "runc" - disable_snapshot_annotations = true + default_runtime_name = "" + disable_snapshot_annotations = false discard_unpacked_layers = false ignore_blockio_not_enabled_errors = false ignore_rdt_not_enabled_errors = false no_pivot = false - snapshotter = "overlayfs" + snapshotter = "" [containerd.default_runtime] base_runtime_spec = "" @@ -69,34 +69,6 @@ unset_seccomp_profile = "" [containerd.runtimes] - [containerd.runtimes.runc] - base_runtime_spec = "" - cni_conf_dir = "" - cni_max_conf_num = 0 - container_annotations = [] - pod_annotations = [] - privileged_without_host_devices = false - privileged_without_host_devices_all_devices_allowed = false - runtime_engine = "" - runtime_path = "" - runtime_root = "" - runtime_type = "io.containerd.runc.v2" - sandbox_mode = "podsandbox" - snapshotter = "" - - [containerd.runtimes.runc.options] - BinaryName = "" - CriuImagePath = "" - CriuPath = "" - CriuWorkPath = "" - IoGid = 0 - IoUid = 0 - NoNewKeyring = false - NoPivotRoot = false - Root = "" - ShimCgroup = "" - SystemdCgroup = false - [containerd.untrusted_workload_runtime] base_runtime_spec = "" cni_conf_dir = "" @@ -115,7 +87,7 @@ unset_seccomp_profile = "" [containerd.untrusted_workload_runtime.options] [image_decryption] - key_model = "node" + key_model = "" [registry] config_path = "/etc/containerd/certs.d" From 949c60cf938c67cf8e0b8cbe4bbd0a536bb330ee Mon Sep 17 00:00:00 2001 From: Mehul-Kumar-27 <202151092@iiitvadodara.ac.in> Date: Tue, 29 Oct 2024 02:57:28 +0530 Subject: [PATCH 21/36] fixing host gen file --- cmd/container_runtime/host.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/container_runtime/host.go b/cmd/container_runtime/host.go index af55057..717bb74 100644 --- a/cmd/container_runtime/host.go +++ b/cmd/container_runtime/host.go @@ -3,6 +3,7 @@ package runtime import ( "fmt" "os" + "strings" "container-registry.com/harbor-satellite/internal/config" "container-registry.com/harbor-satellite/internal/utils" @@ -92,6 +93,9 @@ func GenerateContainerdHostConfig(containerdCertPath, genPath string, log *zerol pathTOWrite := fmt.Sprintf("%s/%s", mirrorGenPath, HostToml) log.Info().Msgf("Writing the host.toml file at path: %s", pathTOWrite) hostData, err := toml.Marshal(dockerContainerdHostConfig) + hostStr := string(hostData) + hostStr = strings.Replace(hostStr, "[host]\n", "", 1) + hostData = []byte(hostStr) if err != nil { log.Err(err).Msg("Error marshalling the host.toml file") return fmt.Errorf("error marshalling the host.toml file: %v", err) From 1b6b45c98a74c131bab95646cf65ff0f48f015e9 Mon Sep 17 00:00:00 2001 From: Mehul-Kumar-27 <202151092@iiitvadodara.ac.in> Date: Tue, 29 Oct 2024 15:49:57 +0530 Subject: [PATCH 22/36] generating the config for the containerd fixes --- cmd/container_runtime/containerd.go | 59 +++---- cmd/container_runtime/containerd_config.go | 169 +++++++++++++++++++++ registry/default_config.go | 7 +- runtime/containerd/config.toml | 121 +-------------- 4 files changed, 207 insertions(+), 149 deletions(-) create mode 100644 cmd/container_runtime/containerd_config.go diff --git a/cmd/container_runtime/containerd.go b/cmd/container_runtime/containerd.go index 09a9672..671a027 100644 --- a/cmd/container_runtime/containerd.go +++ b/cmd/container_runtime/containerd.go @@ -10,7 +10,6 @@ import ( "container-registry.com/harbor-satellite/internal/utils" "container-registry.com/harbor-satellite/logger" "container-registry.com/harbor-satellite/registry" - containerd "github.com/containerd/containerd/pkg/cri/config" toml "github.com/pelletier/go-toml" "github.com/rs/zerolog" "github.com/spf13/cobra" @@ -21,6 +20,7 @@ const ( DefaultGeneratedTomlName = "config.toml" ContainerdRuntime = "containerd" DefaultContainerdConfigPath = "/etc/containerd/config.toml" + DefaultConfigVersion = 2 ) type ContainerdController interface { @@ -80,7 +80,12 @@ func NewContainerdCommand() *cobra.Command { if generateConfig { log.Info().Msg("Generating containerd config file for containerd ...") log.Info().Msgf("Fetching containerd config from path path: %s", containerdConfigPath) - return GenerateContainerdHostConfig(containerDCertPath, DefaultGenPath, log, *satelliteHostConfig) + err := GenerateContainerdHostConfig(containerDCertPath, DefaultGenPath, log, *satelliteHostConfig) + if err != nil { + log.Err(err).Msg("Error generating containerd config") + return fmt.Errorf("could not generate containerd config: %w", err) + } + return GenerateConfig(defaultZotConfig, log, containerdConfigPath, containerDCertPath) } return nil }, @@ -104,48 +109,32 @@ func GenerateConfig(defaultZotConfig *registry.DefaultZotConfig, log *zerolog.Lo return fmt.Errorf("could not read config file: %w", err) } // Now we marshal the data into the containerd config - containerdConfig := &containerd.Config{} + containerdConfig := &ContainerdConfigToml{} err = toml.Unmarshal(data, containerdConfig) if err != nil { log.Err(err).Msg("Error unmarshalling config") return fmt.Errorf("could not unmarshal config: %w", err) } - // Steps to configure the containerd config: - // 1. Set the default registry config cert path - // -- This is the path where the certs of the registry are stored - // -- If the user has already has a cert path then we do not set it rather we would now use the - // user path as the default path - if containerdConfig.PluginConfig.Registry.ConfigPath == "" { - containerdConfig.PluginConfig.Registry.ConfigPath = containerdCertPath + // Add the certs.d path to the config + if containerdConfig.Plugins.Cri.Registry.ConfigPath == "" { + containerdConfig.Plugins.Cri.Registry.ConfigPath = containerdCertPath } - log.Info().Msgf("Setting the registry cert path to: %s", containerdConfig.PluginConfig.Registry.ConfigPath) - // Now we add the local registry to the containerd config mirrors - registryMirror := map[string]containerd.Mirror{ - defaultZotConfig.HTTP.Address: { - Endpoints: []string{defaultZotConfig.HTTP.Address + ":" + defaultZotConfig.HTTP.Port}, - }, + // Set default version + if containerdConfig.Version == 0 { + containerdConfig.Version = DefaultConfigVersion } - if containerdConfig.PluginConfig.Registry.Mirrors == nil { - containerdConfig.PluginConfig.Registry.Mirrors = registryMirror - } else { - for key, value := range registryMirror { - containerdConfig.PluginConfig.Registry.Mirrors[key] = value + // if config disabled plugins container cri then remove it + if len(containerdConfig.DisabledPlugins) > 0 { + filteredPlugins := make([]string, len(containerdConfig.DisabledPlugins)) + for _, plugin := range containerdConfig.DisabledPlugins { + if plugin != "cri" { + filteredPlugins = append(filteredPlugins, plugin) + } } - } - registryConfig := map[string]containerd.RegistryConfig{ - defaultZotConfig.HTTP.Address: { - TLS: &containerd.TLSConfig{ - InsecureSkipVerify: config.UseUnsecure(), - }, - }, - } - // Now we add the local registry to the containerd config registry - if containerdConfig.PluginConfig.Registry.Configs == nil { - containerdConfig.PluginConfig.Registry.Configs = registryConfig - } else { - for key, value := range registryConfig { - containerdConfig.PluginConfig.Registry.Configs[key] = value + if len(filteredPlugins) == 0 { + containerdConfig.DisabledPlugins = nil } + containerdConfig.DisabledPlugins = filteredPlugins } // ToDo: Find a way to remove the unwanted configuration added to the config file while marshalling pathToWrite := filepath.Join(DefaultGenPath, DefaultGeneratedTomlName) diff --git a/cmd/container_runtime/containerd_config.go b/cmd/container_runtime/containerd_config.go new file mode 100644 index 0000000..2a33a7c --- /dev/null +++ b/cmd/container_runtime/containerd_config.go @@ -0,0 +1,169 @@ +package runtime + +// ContainerdConfigToml provides containerd configuration data for the server +type ContainerdConfigToml struct { + // Version of the config file + Version int `toml:"version"` + // Root is the path to a directory where containerd will store persistent data + Root string `toml:"root"` + // State is the path to a directory where containerd will store transient data + State string `toml:"state"` + // TempDir is the path to a directory where to place containerd temporary files + TempDir string `toml:"temp,omitempty"` + // PluginDir is the directory for dynamic plugins to be stored + // + // Deprecated: Please use proxy or binary external plugins. + PluginDir string `toml:"plugin_dir,omitempty"` + // GRPC configuration settings + GRPC GRPCConfig `toml:"grpc,omitempty"` + // TTRPC configuration settings + TTRPC TTRPCConfig `toml:"ttrpc,omitempty"` + // Debug and profiling settings + Debug Debug `toml:"debug,omitempty"` + // Metrics and monitoring settings + Metrics MetricsConfig `toml:"metrics,omitempty"` + // DisabledPlugins are IDs of plugins to disable. Disabled plugins won't be + // initialized and started. + // DisabledPlugins must use a fully qualified plugin URI. + DisabledPlugins []string `toml:"disabled_plugins,omitempty"` + // RequiredPlugins are IDs of required plugins. Containerd exits if any + // required plugin doesn't exist or fails to be initialized or started. + // RequiredPlugins must use a fully qualified plugin URI. + RequiredPlugins []string `toml:"required_plugins,omitempty"` + // Plugins provides plugin specific configuration for the initialization of a plugin + Plugins PluginsConfig `toml:"plugins,omitempty"` + // OOMScore adjust the containerd's oom score + OOMScore int `toml:"oom_score,omitempty"` + // Cgroup specifies cgroup information for the containerd daemon process + Cgroup CgroupConfig `toml:"cgroup,omitempty"` + // ProxyPlugins configures plugins which are communicated to over GRPC + ProxyPlugins map[string]ProxyPlugin `toml:"proxy_plugins,omitempty"` + // Timeouts specified as a duration + Timeouts map[string]string `toml:"timeouts,omitempty"` + // Imports are additional file path list to config files that can overwrite main config file fields + Imports []string `toml:"imports,omitempty"` + // StreamProcessors configuration + StreamProcessors map[string]StreamProcessor `toml:"stream_processors,omitempty"` +} + +type StreamProcessor struct { + // Accepts specific media-types + Accepts []string `toml:"accepts,omitempty"` + // Returns the media-type + Returns string `toml:"returns,omitempty"` + // Path or name of the binary + Path string `toml:"path"` + // Args to the binary + Args []string `toml:"args,omitempty"` + // Environment variables for the binary + Env []string `toml:"env,omitempty"` +} + +type GRPCConfig struct { + Address string `toml:"address"` + TCPAddress string `toml:"tcp_address,omitempty"` + TCPTLSCA string `toml:"tcp_tls_ca,omitempty"` + TCPTLSCert string `toml:"tcp_tls_cert,omitempty"` + TCPTLSKey string `toml:"tcp_tls_key,omitempty"` + UID int `toml:"uid,omitempty"` + GID int `toml:"gid,omitempty"` + MaxRecvMsgSize int `toml:"max_recv_message_size,omitempty"` + MaxSendMsgSize int `toml:"max_send_message_size,omitempty"` +} + +// TTRPCConfig provides TTRPC configuration for the socket +type TTRPCConfig struct { + Address string `toml:"address"` + UID int `toml:"uid,omitempty"` + GID int `toml:"gid,omitempty"` +} + +// Debug provides debug configuration +type Debug struct { + Address string `toml:"address,omitempty"` + UID int `toml:"uid,omitempty"` + GID int `toml:"gid,omitempty"` + Level string `toml:"level,omitempty"` + // Format represents the logging format. Supported values are 'text' and 'json'. + Format string `toml:"format,omitempty"` +} + +// MetricsConfig provides metrics configuration +type MetricsConfig struct { + Address string `toml:"address,omitempty"` + GRPCHistogram bool `toml:"grpc_histogram,omitempty"` +} + +// CgroupConfig provides cgroup configuration +type CgroupConfig struct { + Path string `toml:"path,omitempty"` +} + +// ProxyPlugin provides a proxy plugin configuration +type ProxyPlugin struct { + Type string `toml:"type"` + Address string `toml:"address"` + Platform string `toml:"platform,omitempty"` + Exports map[string]string `toml:"exports,omitempty"` + Capabilities []string `toml:"capabilities,omitempty"` +} + +type PluginsConfig struct { + Cri CriConfig `toml:"io.containerd.grpc.v1.cri,omitempty"` + Cgroups MonitorConfig `toml:"io.containerd.monitor.v1.cgroups,omitempty"` + LinuxRuntime interface{} `toml:"io.containerd.runtime.v1.linux,omitempty"` + Scheduler GCSchedulerConfig `toml:"io.containerd.gc.v1.scheduler,omitempty"` + Bolt interface{} `toml:"io.containerd.metadata.v1.bolt,omitempty"` + Task RuntimeV2TaskConfig `toml:"io.containerd.runtime.v2.task,omitempty"` + Opt interface{} `toml:"io.containerd.internal.v1.opt,omitempty"` + Restart interface{} `toml:"io.containerd.internal.v1.restart,omitempty"` + Tracing interface{} `toml:"io.containerd.internal.v1.tracing,omitempty"` + Otlp interface{} `toml:"io.containerd.tracing.processor.v1.otlp,omitempty"` + Aufs interface{} `toml:"io.containerd.snapshotter.v1.aufs,omitempty"` + Btrfs interface{} `toml:"io.containerd.snapshotter.v1.btrfs,omitempty"` + Devmapper interface{} `toml:"io.containerd.snapshotter.v1.devmapper,omitempty"` + Native interface{} `toml:"io.containerd.snapshotter.v1.native,omitempty"` + Overlayfs interface{} `toml:"io.containerd.snapshotter.v1.overlayfs,omitempty"` + Zfs interface{} `toml:"io.containerd.snapshotter.v1.zfs,omitempty"` +} + +type MonitorConfig struct { + NoPrometheus bool `toml:"no_prometheus,omitempty"` +} + +type GCSchedulerConfig struct { + PauseThreshold float64 `toml:"pause_threshold,omitempty"` + DeletionThreshold int `toml:"deletion_threshold,omitempty"` + MutationThreshold int `toml:"mutation_threshold,omitempty"` + ScheduleDelay string `toml:"schedule_delay,omitempty"` + StartupDelay string `toml:"startup_delay,omitempty"` +} + +type RuntimeV2TaskConfig struct { + Platforms []string `toml:"platforms,omitempty"` + SchedCore bool `toml:"sched_core,omitempty"` +} + +type CriConfig struct { + Containerd CriContainerdConfig `toml:"containerd,omitempty"` + Registry RegistryConfig `toml:"registry,omitempty"` +} + +type CriContainerdConfig struct { + DefaultRuntimeName string `toml:"default_runtime_name,omitempty"` + Runtimes map[string]RuntimeConfig `toml:"runtimes,omitempty"` +} + +type RuntimeConfig struct { + PrivilegedWithoutHostDevices bool `toml:"privileged_without_host_devices,omitempty"` + RuntimeType string `toml:"runtime_type"` + Options RuntimeOptions `toml:"options,omitempty"` +} + +type RuntimeOptions struct { + BinaryName string `toml:"BinaryName,omitempty"` +} + +type RegistryConfig struct { + ConfigPath string `toml:"config_path,omitempty"` +} \ No newline at end of file diff --git a/registry/default_config.go b/registry/default_config.go index 0bf8ee4..ec5bc2e 100644 --- a/registry/default_config.go +++ b/registry/default_config.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "os" + "strings" ) type DefaultZotConfig struct { @@ -22,7 +23,11 @@ type DefaultZotConfig struct { } func (c *DefaultZotConfig) GetLocalRegistryURL() string { - return fmt.Sprintf("%s:%s", c.HTTP.Address, c.HTTP.Port) + address := c.HTTP.Address + if !strings.HasPrefix(address, "http://") && !strings.HasPrefix(address, "https://") { + address = "http://" + address + } + return fmt.Sprintf("%s:%s", address, c.HTTP.Port) } // ReadConfig reads a JSON file from the specified path and unmarshals it into a Config struct. diff --git a/runtime/containerd/config.toml b/runtime/containerd/config.toml index 5b005ab..a651855 100644 --- a/runtime/containerd/config.toml +++ b/runtime/containerd/config.toml @@ -1,116 +1,11 @@ -ContainerdEndpoint = "" -ContainerdRootDir = "" -RootDir = "" -StateDir = "" -cdi_spec_dirs = [] -device_ownership_from_security_context = false -disable_apparmor = false -disable_cgroup = false -disable_hugetlb_controller = false -disable_proc_mount = false -disable_tcp_service = false -drain_exec_sync_io_timeout = "" -enable_cdi = false -enable_selinux = false -enable_tls_streaming = false -enable_unprivileged_icmp = false -enable_unprivileged_ports = false -ignore_deprecation_warnings = [] -ignore_image_defined_volumes = false -image_pull_progress_timeout = "" -image_pull_with_sync_fs = false -max_concurrent_downloads = 0 -max_container_log_line_size = 0 -netns_mounts_under_state_dir = false -restrict_oom_score_adj = false -sandbox_image = "" -selinux_category_range = 0 -stats_collect_period = 0 -stream_idle_timeout = "" -stream_server_address = "" -stream_server_port = "" -systemd_cgroup = false -tolerate_missing_hugetlb_controller = false -unset_seccomp_profile = "" +disabled_plugins = [""] +root = "" +state = "" +version = 2 -[cni] - bin_dir = "" - conf_dir = "" - conf_template = "" - ip_pref = "" - max_conf_num = 0 - setup_serially = false +[plugins] -[containerd] - default_runtime_name = "" - disable_snapshot_annotations = false - discard_unpacked_layers = false - ignore_blockio_not_enabled_errors = false - ignore_rdt_not_enabled_errors = false - no_pivot = false - snapshotter = "" + [plugins."io.containerd.grpc.v1.cri"] - [containerd.default_runtime] - base_runtime_spec = "" - cni_conf_dir = "" - cni_max_conf_num = 0 - container_annotations = [] - pod_annotations = [] - privileged_without_host_devices = false - privileged_without_host_devices_all_devices_allowed = false - runtime_engine = "" - runtime_path = "" - runtime_root = "" - runtime_type = "" - sandbox_mode = "" - snapshotter = "" - - [containerd.default_runtime.options] - - [containerd.runtimes] - - [containerd.untrusted_workload_runtime] - base_runtime_spec = "" - cni_conf_dir = "" - cni_max_conf_num = 0 - container_annotations = [] - pod_annotations = [] - privileged_without_host_devices = false - privileged_without_host_devices_all_devices_allowed = false - runtime_engine = "" - runtime_path = "" - runtime_root = "" - runtime_type = "" - sandbox_mode = "" - snapshotter = "" - - [containerd.untrusted_workload_runtime.options] - -[image_decryption] - key_model = "" - -[registry] - config_path = "/etc/containerd/certs.d" - - [registry.auths] - - [registry.configs] - - [registry.configs."127.0.0.1"] - - [registry.configs."127.0.0.1".tls] - ca_file = "" - cert_file = "" - insecure_skip_verify = true - key_file = "" - - [registry.headers] - - [registry.mirrors] - - [registry.mirrors."127.0.0.1"] - endpoint = ["127.0.0.1:8585"] - -[x509_key_pair_streaming] - tls_cert_file = "" - tls_key_file = "" + [plugins."io.containerd.grpc.v1.cri".registry] + config_path = "/etc/containerd/certs.d" From 86246687880a0dc747bc16a294aa52214dc0f304 Mon Sep 17 00:00:00 2001 From: Mehul-Kumar-27 <202151092@iiitvadodara.ac.in> Date: Tue, 29 Oct 2024 16:40:50 +0530 Subject: [PATCH 23/36] fixes --- .gitignore | 3 ++- cmd/container_runtime/containerd.go | 9 +++++---- cmd/container_runtime/containerd_config.go | 6 +++--- cmd/container_runtime/host.go | 6 +++--- runtime/containerd/config.toml | 4 ---- 5 files changed, 13 insertions(+), 15 deletions(-) diff --git a/.gitignore b/.gitignore index 9095ac2..0b4ba34 100644 --- a/.gitignore +++ b/.gitignore @@ -31,4 +31,5 @@ secrets.txt __debug_bin1949266242 /zot -runtime/ +/runtime +runtime/containerd/config.toml diff --git a/cmd/container_runtime/containerd.go b/cmd/container_runtime/containerd.go index 671a027..069e95c 100644 --- a/cmd/container_runtime/containerd.go +++ b/cmd/container_runtime/containerd.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "container-registry.com/harbor-satellite/internal/config" "container-registry.com/harbor-satellite/internal/utils" @@ -125,15 +126,12 @@ func GenerateConfig(defaultZotConfig *registry.DefaultZotConfig, log *zerolog.Lo } // if config disabled plugins container cri then remove it if len(containerdConfig.DisabledPlugins) > 0 { - filteredPlugins := make([]string, len(containerdConfig.DisabledPlugins)) + filteredPlugins := make([]string, 0, len(containerdConfig.DisabledPlugins)) for _, plugin := range containerdConfig.DisabledPlugins { if plugin != "cri" { filteredPlugins = append(filteredPlugins, plugin) } } - if len(filteredPlugins) == 0 { - containerdConfig.DisabledPlugins = nil - } containerdConfig.DisabledPlugins = filteredPlugins } // ToDo: Find a way to remove the unwanted configuration added to the config file while marshalling @@ -141,6 +139,9 @@ func GenerateConfig(defaultZotConfig *registry.DefaultZotConfig, log *zerolog.Lo log.Info().Msgf("Writing the containerd config to path: %s", pathToWrite) // Now we write the config to the file data, err = toml.Marshal(containerdConfig) + dataStr := string(data) + dataStr = strings.Replace(dataStr, "[plugins]\n", "", 1) + data = []byte(dataStr) if err != nil { log.Err(err).Msg("Error marshalling config") return fmt.Errorf("could not marshal config: %w", err) diff --git a/cmd/container_runtime/containerd_config.go b/cmd/container_runtime/containerd_config.go index 2a33a7c..0148aa7 100644 --- a/cmd/container_runtime/containerd_config.go +++ b/cmd/container_runtime/containerd_config.go @@ -3,11 +3,11 @@ package runtime // ContainerdConfigToml provides containerd configuration data for the server type ContainerdConfigToml struct { // Version of the config file - Version int `toml:"version"` + Version int `toml:"version,omitempty"` // Root is the path to a directory where containerd will store persistent data - Root string `toml:"root"` + Root string `toml:"root,omitempty"` // State is the path to a directory where containerd will store transient data - State string `toml:"state"` + State string `toml:"state,omitempty"` // TempDir is the path to a directory where to place containerd temporary files TempDir string `toml:"temp,omitempty"` // PluginDir is the directory for dynamic plugins to be stored diff --git a/cmd/container_runtime/host.go b/cmd/container_runtime/host.go index 717bb74..abe35ab 100644 --- a/cmd/container_runtime/host.go +++ b/cmd/container_runtime/host.go @@ -12,7 +12,7 @@ import ( ) const ( - DockerIoConfigPath = "docker" + SatelliteConfigPath = "satellite" HostToml = "host_gen.toml" DefaultTomlConfigPath = "_default" DockerURL = "https://registry-1.docker.io" @@ -46,13 +46,13 @@ func NewSatelliteHostConfig(localRegistry, sourceRegistry string) *SatelliteHost // GenerateContainerdHostConfig generates the host.toml file for containerd docker.io and also create a default config.toml file func GenerateContainerdHostConfig(containerdCertPath, genPath string, log *zerolog.Logger, satelliteHostConfig SatelliteHostConfig) error { - mirrorGenPath := fmt.Sprintf("%s/%s", genPath, DockerIoConfigPath) + mirrorGenPath := fmt.Sprintf("%s/%s", genPath, SatelliteConfigPath) err := utils.CreateRuntimeDirectory(mirrorGenPath) if err != nil { log.Err(err).Msgf("Error creating the directory: %s", mirrorGenPath) return fmt.Errorf("error creating the directory: %v", err) } - dockerHubHostConfigPath := fmt.Sprintf("%s/%s/%s", containerdCertPath, DockerIoConfigPath, HostToml) + dockerHubHostConfigPath := fmt.Sprintf("%s/%s/%s", containerdCertPath, SatelliteConfigPath, HostToml) var dockerContainerdHostConfig ContainerdHostConfig // Read the `docker.io/host.toml` file if present diff --git a/runtime/containerd/config.toml b/runtime/containerd/config.toml index a651855..bf77b87 100644 --- a/runtime/containerd/config.toml +++ b/runtime/containerd/config.toml @@ -1,9 +1,5 @@ -disabled_plugins = [""] -root = "" -state = "" version = 2 -[plugins] [plugins."io.containerd.grpc.v1.cri"] From 771a6e32dddfeee8850651c45345d4f181717a59 Mon Sep 17 00:00:00 2001 From: Mehul-Kumar-27 <202151092@iiitvadodara.ac.in> Date: Tue, 29 Oct 2024 17:13:15 +0530 Subject: [PATCH 24/36] coderabbit fixes --- cmd/container_runtime/containerd.go | 10 +++++-- cmd/container_runtime/host.go | 41 ++++++++++++++-------------- cmd/container_runtime/read_config.go | 3 +- cmd/root.go | 3 ++ internal/config/config.go | 8 ++++-- internal/scheduler/scheduler.go | 7 ++++- internal/state/artifact.go | 4 +++ internal/state/fetcher.go | 20 +++++++------- main.go | 1 + 9 files changed, 61 insertions(+), 36 deletions(-) diff --git a/cmd/container_runtime/containerd.go b/cmd/container_runtime/containerd.go index 069e95c..6ac9159 100644 --- a/cmd/container_runtime/containerd.go +++ b/cmd/container_runtime/containerd.go @@ -35,7 +35,12 @@ func init() { cwd, err := os.Getwd() if err != nil { fmt.Printf("Error getting current working directory: %v\n", err) - DefaultGenPath = "/runtime/containerd" // Fallback in case of error + if _, err := os.Stat(DefaultGenPath); os.IsNotExist(err) { + err := os.MkdirAll(DefaultGenPath, os.ModePerm) + if err != nil { + fmt.Printf("Error creating default directory: %v\n", err) + } + } } else { DefaultGenPath = filepath.Join(cwd, "runtime/containerd") } @@ -53,6 +58,7 @@ func NewContainerdCommand() *cobra.Command { PersistentPreRunE: func(cmd *cobra.Command, args []string) error { var err error utils.SetupContextForCommand(cmd) + config.InitConfig() log := logger.FromContext(cmd.Context()) if config.GetOwnRegistry() { log.Info().Msg("Using own registry for config generation") @@ -67,7 +73,7 @@ func NewContainerdCommand() *cobra.Command { } else { log.Info().Msg("Using default registry for config generation") defaultZotConfig, err = registry.ReadConfig(config.GetZotConfigPath()) - if err != nil { + if err != nil || defaultZotConfig == nil { return fmt.Errorf("could not read config: %w", err) } log.Info().Msgf("Default config read successfully: %v", defaultZotConfig.HTTP.Address+":"+defaultZotConfig.HTTP.Port) diff --git a/cmd/container_runtime/host.go b/cmd/container_runtime/host.go index abe35ab..410001d 100644 --- a/cmd/container_runtime/host.go +++ b/cmd/container_runtime/host.go @@ -3,6 +3,7 @@ package runtime import ( "fmt" "os" + "path/filepath" "strings" "container-registry.com/harbor-satellite/internal/config" @@ -52,22 +53,22 @@ func GenerateContainerdHostConfig(containerdCertPath, genPath string, log *zerol log.Err(err).Msgf("Error creating the directory: %s", mirrorGenPath) return fmt.Errorf("error creating the directory: %v", err) } - dockerHubHostConfigPath := fmt.Sprintf("%s/%s/%s", containerdCertPath, SatelliteConfigPath, HostToml) - var dockerContainerdHostConfig ContainerdHostConfig + satelliteHubHostConfigPath := fmt.Sprintf("%s/%s/%s", containerdCertPath, SatelliteConfigPath, HostToml) + var satelliteContainerdHostConfig ContainerdHostConfig - // Read the `docker.io/host.toml` file if present - data, err := utils.ReadFile(dockerHubHostConfigPath, false) + // Read the `satellite/host.toml` file if present + data, err := utils.ReadFile(satelliteHubHostConfigPath, false) if err != nil { if os.IsNotExist(err) { - log.Warn().Msgf("The docker.io/host.toml file does not exist at path: %s", dockerHubHostConfigPath) + log.Warn().Msgf("The satellite/host.toml file does not exist at path: %s", satelliteHubHostConfigPath) } else { - return fmt.Errorf("error reading the docker.io/host.toml file: %v", err) + return fmt.Errorf("error reading the satellite/host.toml file: %v", err) } } - err = toml.Unmarshal(data, &dockerContainerdHostConfig) + err = toml.Unmarshal(data, &satelliteContainerdHostConfig) if err != nil { - log.Err(err).Msgf("Error unmarshalling the docker.io/host.toml file at path: %s", dockerHubHostConfigPath) - return fmt.Errorf("error unmarshalling the docker.io/host.toml file: %v", err) + log.Err(err).Msgf("Error unmarshalling the satellite/host.toml file at path: %s", satelliteHubHostConfigPath) + return fmt.Errorf("error unmarshalling the satellite/host.toml file: %v", err) } satelliteHostConfigToAdd := ContainerdHostConfig{ Host: map[string]HostConfig{ @@ -78,28 +79,28 @@ func GenerateContainerdHostConfig(containerdCertPath, genPath string, log *zerol }, } - if dockerContainerdHostConfig.Server == "" { - dockerContainerdHostConfig.Server = DockerURL + if satelliteContainerdHostConfig.Server == "" { + satelliteContainerdHostConfig.Server = DockerURL } - if dockerContainerdHostConfig.Host == nil { - dockerContainerdHostConfig.Host = satelliteHostConfigToAdd.Host + if satelliteContainerdHostConfig.Host == nil { + satelliteContainerdHostConfig.Host = satelliteHostConfigToAdd.Host } else { - for key, value := range dockerContainerdHostConfig.Host { + for key, value := range satelliteContainerdHostConfig.Host { satelliteHostConfigToAdd.Host[key] = value } - dockerContainerdHostConfig.Host = satelliteHostConfigToAdd.Host + satelliteContainerdHostConfig.Host = satelliteHostConfigToAdd.Host } - pathTOWrite := fmt.Sprintf("%s/%s", mirrorGenPath, HostToml) + pathTOWrite := filepath.Join(mirrorGenPath, HostToml) log.Info().Msgf("Writing the host.toml file at path: %s", pathTOWrite) - hostData, err := toml.Marshal(dockerContainerdHostConfig) - hostStr := string(hostData) - hostStr = strings.Replace(hostStr, "[host]\n", "", 1) - hostData = []byte(hostStr) + hostData, err := toml.Marshal(satelliteContainerdHostConfig) if err != nil { log.Err(err).Msg("Error marshalling the host.toml file") return fmt.Errorf("error marshalling the host.toml file: %v", err) } + hostStr := string(hostData) + hostStr = strings.Replace(hostStr, "[host]\n", "", 1) + hostData = []byte(hostStr) err = utils.WriteFile(pathTOWrite, hostData) if err != nil { log.Err(err).Msg("Error writing the host.toml file") diff --git a/cmd/container_runtime/read_config.go b/cmd/container_runtime/read_config.go index a815baf..94af5df 100644 --- a/cmd/container_runtime/read_config.go +++ b/cmd/container_runtime/read_config.go @@ -3,7 +3,7 @@ package runtime import ( "fmt" - + "container-registry.com/harbor-satellite/internal/config" "container-registry.com/harbor-satellite/internal/utils" "container-registry.com/harbor-satellite/logger" "github.com/spf13/cobra" @@ -15,6 +15,7 @@ func NewReadConfigCommand(runtime string) *cobra.Command { Short: fmt.Sprintf("Reads the config file for the %s runtime", runtime), PersistentPreRun: func(cmd *cobra.Command, args []string) { utils.SetupContextForCommand(cmd) + config.InitConfig() }, RunE: func(cmd *cobra.Command, args []string) error { //Parse the flags diff --git a/cmd/root.go b/cmd/root.go index e175d1d..06926b4 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -64,6 +64,9 @@ func run(ctx context.Context, cancel context.CancelFunc) error { if err != nil || stateArtifactFetcher == nil { return fmt.Errorf("error processing input: %w", err) } + if stateArtifactFetcher == nil { + return fmt.Errorf("state artifact fetcher is nil") + } satelliteService := satellite.NewSatellite(ctx, stateArtifactFetcher, scheduler.GetSchedulerKey()) diff --git a/internal/config/config.go b/internal/config/config.go index 4717e2d..a7149e2 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -8,8 +8,12 @@ import ( "github.com/spf13/viper" ) -func init(){ - InitConfig() +func init() { + err := InitConfig() + if err != nil { + fmt.Printf("Error initializing config: %v\n", err) + os.Exit(1) + } } var AppConfig *Config diff --git a/internal/scheduler/scheduler.go b/internal/scheduler/scheduler.go index f12c20e..e0f9f93 100644 --- a/internal/scheduler/scheduler.go +++ b/internal/scheduler/scheduler.go @@ -75,7 +75,10 @@ func (s *BasicScheduler) Schedule(process Process) error { } // Add the process to the scheduler _, err := s.cron.AddFunc(process.GetCronExpr(), func() { - s.executeProcess(process) + err := s.executeProcess(process) + if err != nil { + log.Error().Err(err).Msgf("Error executing process %s", process.GetName()) + } }) if err != nil { return fmt.Errorf("error adding process to scheduler: %w", err) @@ -91,6 +94,8 @@ func (s *BasicScheduler) Start() error { } func (s *BasicScheduler) Stop() error { + s.mu.Lock() + defer s.mu.Unlock() s.stopped = true s.cron.Stop() return nil diff --git a/internal/state/artifact.go b/internal/state/artifact.go index 81ff6c8..2a80fcc 100644 --- a/internal/state/artifact.go +++ b/internal/state/artifact.go @@ -39,6 +39,10 @@ func NewArtifact(deleted bool, repository string, tags []string, digest, artifac } } +func (a *Artifact) GetLabels() []string { + return a.Labels +} + func (a *Artifact) GetRepository() string { return a.Repository } diff --git a/internal/state/fetcher.go b/internal/state/fetcher.go index d2f9710..7eb8632 100644 --- a/internal/state/fetcher.go +++ b/internal/state/fetcher.go @@ -19,9 +19,9 @@ type StateFetcher interface { } type baseStateFetcher struct { - group_name string - state_artifact_name string - state_artifact_reader StateReader + groupName string + stateArtifactName string + stateArtifactReader StateReader } type URLStateFetcher struct { @@ -39,9 +39,9 @@ func NewURLStateFetcher() StateFetcher { url = utils.FormatRegistryURL(url) return &URLStateFetcher{ baseStateFetcher: baseStateFetcher{ - group_name: config.GetGroupName(), - state_artifact_name: config.GetStateArtifactName(), - state_artifact_reader: NewState(), + groupName: config.GetGroupName(), + stateArtifactName: config.GetStateArtifactName(), + stateArtifactReader: NewState(), }, url: url, } @@ -50,9 +50,9 @@ func NewURLStateFetcher() StateFetcher { func NewFileStateFetcher() StateFetcher { return &FileStateArtifactFetcher{ baseStateFetcher: baseStateFetcher{ - group_name: config.GetGroupName(), - state_artifact_name: config.GetStateArtifactName(), - state_artifact_reader: NewState(), + groupName: config.GetGroupName(), + stateArtifactName: config.GetStateArtifactName(), + stateArtifactReader: NewState(), }, filePath: config.GetInput(), } @@ -84,7 +84,7 @@ func (f *URLStateFetcher) FetchStateArtifact(state interface{}) error { sourceRegistry := utils.FormatRegistryURL(config.GetRemoteRegistryURL()) tag := "latest" - img, err := crane.Pull(fmt.Sprintf("%s/%s/%s:%s", sourceRegistry, f.group_name, f.state_artifact_name, tag), options...) + img, err := crane.Pull(fmt.Sprintf("%s/%s/%s:%s", sourceRegistry, f.groupName, f.stateArtifactName, tag), options...) if err != nil { return fmt.Errorf("failed to pull the state artifact: %v", err) } diff --git a/main.go b/main.go index ba941f4..3f77dec 100644 --- a/main.go +++ b/main.go @@ -11,5 +11,6 @@ func main() { err := cmd.Execute() if err != nil { fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(1) } } From a08405d461d1ee0306368630e55f01479a20b81c Mon Sep 17 00:00:00 2001 From: Mehul-Kumar-27 <202151092@iiitvadodara.ac.in> Date: Tue, 29 Oct 2024 18:16:42 +0530 Subject: [PATCH 25/36] fixes --- internal/state/replicator.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/state/replicator.go b/internal/state/replicator.go index 879bc93..3814879 100644 --- a/internal/state/replicator.go +++ b/internal/state/replicator.go @@ -54,7 +54,7 @@ func (r *BasicReplicator) Replicate(ctx context.Context, replicationEntities []A log.Info().Msgf("Pulling image %s from repository %s at registry %s with tag %s", replicationEntity.GetName(), replicationEntity.GetRepository(), r.sourceRegistry, replicationEntity.GetTags()[0]) // Pull the image from the source registry - srcImage, err := crane.Pull(fmt.Sprintf("%s/%s/%s:%s", r.sourceRegistry, replicationEntity.GetName(), replicationEntity.GetRepository(), replicationEntity.GetTags()[0]), options...) + srcImage, err := crane.Pull(fmt.Sprintf("%s/%s/%s:%s", r.sourceRegistry, replicationEntity.GetRepository(), replicationEntity.GetName(), replicationEntity.GetTags()[0]), options...) if err != nil { log.Error().Msgf("Failed to pull image: %v", err) return err @@ -64,7 +64,7 @@ func (r *BasicReplicator) Replicate(ctx context.Context, replicationEntities []A ociImage := mutate.MediaType(srcImage, types.OCIManifestSchema1) // Push the converted OCI image to the Zot registry - err = crane.Push(ociImage, fmt.Sprintf("%s/%s/%s:%s", r.remoteRegistryURL, replicationEntity.GetName(), replicationEntity.GetRepository(), replicationEntity.GetTags()[0]), options...) + err = crane.Push(ociImage, fmt.Sprintf("%s/%s/%s:%s", r.remoteRegistryURL, replicationEntity.GetRepository(), replicationEntity.GetName(), replicationEntity.GetTags()[0]), options...) if err != nil { log.Error().Msgf("Failed to push image: %v", err) return err @@ -90,7 +90,7 @@ func (r *BasicReplicator) DeleteReplicationEntity(ctx context.Context, replicati for _, entity := range replicationEntity { log.Info().Msgf("Deleting image %s from repository %s at registry %s with tag %s", entity.GetName(), entity.GetRepository(), r.remoteRegistryURL, entity.GetTags()[0]) - err := crane.Delete(fmt.Sprintf("%s/%s/%s:%s", r.remoteRegistryURL, entity.GetName(), entity.GetRepository(), entity.GetTags()[0]), options...) + err := crane.Delete(fmt.Sprintf("%s/%s/%s:%s", r.remoteRegistryURL, entity.GetRepository(), entity.GetName() ,entity.GetTags()[0]), options...) if err != nil { log.Error().Msgf("Failed to delete image: %v", err) return err @@ -99,4 +99,4 @@ func (r *BasicReplicator) DeleteReplicationEntity(ctx context.Context, replicati } return nil -} +} \ No newline at end of file From 0dc11605bcbcb7e9531a6a0cfd93c154d12cbcdd Mon Sep 17 00:00:00 2001 From: Mehul-Kumar-27 <202151092@iiitvadodara.ac.in> Date: Mon, 4 Nov 2024 22:32:10 +0530 Subject: [PATCH 26/36] adding config command for crio --- cmd/container_runtime/containerd.go | 45 ++----- cmd/container_runtime/crio.go | 169 +++++++++++++++++++++++++++ cmd/container_runtime/crio_config.go | 51 ++++++++ cmd/container_runtime/host.go | 2 +- cmd/root.go | 1 + 5 files changed, 234 insertions(+), 34 deletions(-) create mode 100644 cmd/container_runtime/crio.go create mode 100644 cmd/container_runtime/crio_config.go diff --git a/cmd/container_runtime/containerd.go b/cmd/container_runtime/containerd.go index 6ac9159..f044110 100644 --- a/cmd/container_runtime/containerd.go +++ b/cmd/container_runtime/containerd.go @@ -29,20 +29,21 @@ type ContainerdController interface { Generate(ctx context.Context, configPath string, log *zerolog.Logger) error } -var DefaultGenPath string +var DefaultContainerDGenPath string func init() { cwd, err := os.Getwd() if err != nil { fmt.Printf("Error getting current working directory: %v\n", err) - if _, err := os.Stat(DefaultGenPath); os.IsNotExist(err) { - err := os.MkdirAll(DefaultGenPath, os.ModePerm) + DefaultContainerDGenPath = "/runtime/containerd" + if _, err := os.Stat(DefaultContainerDGenPath); os.IsNotExist(err) { + err := os.MkdirAll(DefaultContainerDGenPath, os.ModePerm) if err != nil { fmt.Printf("Error creating default directory: %v\n", err) } } } else { - DefaultGenPath = filepath.Join(cwd, "runtime/containerd") + DefaultContainerDGenPath = filepath.Join(cwd, "runtime/containerd") } } @@ -56,29 +57,7 @@ func NewContainerdCommand() *cobra.Command { Use: "containerd", Short: "Creates the config file for the containerd runtime to fetch the images from the local repository", PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - var err error - utils.SetupContextForCommand(cmd) - config.InitConfig() - log := logger.FromContext(cmd.Context()) - if config.GetOwnRegistry() { - log.Info().Msg("Using own registry for config generation") - address, err := utils.ValidateRegistryAddress(config.GetOwnRegistryAdr(), config.GetOwnRegistryPort()) - if err != nil { - log.Err(err).Msg("Error validating registry address") - return err - } - log.Info().Msgf("Registry address validated: %s", address) - defaultZotConfig.HTTP.Address = config.GetOwnRegistryAdr() - defaultZotConfig.HTTP.Port = config.GetOwnRegistryPort() - } else { - log.Info().Msg("Using default registry for config generation") - defaultZotConfig, err = registry.ReadConfig(config.GetZotConfigPath()) - if err != nil || defaultZotConfig == nil { - return fmt.Errorf("could not read config: %w", err) - } - log.Info().Msgf("Default config read successfully: %v", defaultZotConfig.HTTP.Address+":"+defaultZotConfig.HTTP.Port) - } - return utils.CreateRuntimeDirectory(DefaultGenPath) + return SetupContainerRuntimeCommand(cmd, &defaultZotConfig, DefaultContainerDGenPath) }, RunE: func(cmd *cobra.Command, args []string) error { log := logger.FromContext(cmd.Context()) @@ -86,13 +65,13 @@ func NewContainerdCommand() *cobra.Command { satelliteHostConfig := NewSatelliteHostConfig(defaultZotConfig.GetLocalRegistryURL(), sourceRegistry) if generateConfig { log.Info().Msg("Generating containerd config file for containerd ...") - log.Info().Msgf("Fetching containerd config from path path: %s", containerdConfigPath) - err := GenerateContainerdHostConfig(containerDCertPath, DefaultGenPath, log, *satelliteHostConfig) + log.Info().Msgf("Fetching containerd config from path: %s", containerdConfigPath) + err := GenerateContainerdHostConfig(containerDCertPath, DefaultContainerDGenPath, log, *satelliteHostConfig) if err != nil { log.Err(err).Msg("Error generating containerd config") return fmt.Errorf("could not generate containerd config: %w", err) } - return GenerateConfig(defaultZotConfig, log, containerdConfigPath, containerDCertPath) + return GenerateContainerdConfig(defaultZotConfig, log, containerdConfigPath, containerDCertPath) } return nil }, @@ -105,10 +84,10 @@ func NewContainerdCommand() *cobra.Command { return containerdCmd } -// GenerateConfig generates the containerd config file for the containerd runtime +// GenerateContainerdConfig generates the containerd config file for the containerd runtime // It takes the zot config a logger and the containerd config path // It reads the containerd config file and adds the local registry to the config file -func GenerateConfig(defaultZotConfig *registry.DefaultZotConfig, log *zerolog.Logger, containerdConfigPath, containerdCertPath string) error { +func GenerateContainerdConfig(defaultZotConfig *registry.DefaultZotConfig, log *zerolog.Logger, containerdConfigPath, containerdCertPath string) error { // First Read the present config file at the configPath data, err := utils.ReadFile(containerdConfigPath, false) if err != nil { @@ -141,7 +120,7 @@ func GenerateConfig(defaultZotConfig *registry.DefaultZotConfig, log *zerolog.Lo containerdConfig.DisabledPlugins = filteredPlugins } // ToDo: Find a way to remove the unwanted configuration added to the config file while marshalling - pathToWrite := filepath.Join(DefaultGenPath, DefaultGeneratedTomlName) + pathToWrite := filepath.Join(DefaultContainerDGenPath, DefaultGeneratedTomlName) log.Info().Msgf("Writing the containerd config to path: %s", pathToWrite) // Now we write the config to the file data, err = toml.Marshal(containerdConfig) diff --git a/cmd/container_runtime/crio.go b/cmd/container_runtime/crio.go new file mode 100644 index 0000000..aa32da7 --- /dev/null +++ b/cmd/container_runtime/crio.go @@ -0,0 +1,169 @@ +package runtime + +import ( + "fmt" + "os" + "path/filepath" + + "container-registry.com/harbor-satellite/internal/config" + "container-registry.com/harbor-satellite/internal/utils" + "container-registry.com/harbor-satellite/logger" + "container-registry.com/harbor-satellite/registry" + "github.com/pelletier/go-toml/v2" + "github.com/rs/zerolog" + "github.com/spf13/cobra" +) + +const ( + DefaultCrioRegistryConfigPath = "/etc/containers/registries.conf.d/crio.conf" +) + +var DefaultCrioGenPath string + +func init() { + cwd, err := os.Getwd() + if err != nil { + fmt.Printf("Error getting current working directory: %v\n", err) + if _, err := os.Stat(DefaultCrioGenPath); os.IsNotExist(err) { + DefaultCrioGenPath = "runtime/crio" + err := os.MkdirAll(DefaultCrioGenPath, os.ModePerm) + if err != nil { + fmt.Printf("Error creating default directory: %v\n", err) + } + } + } else { + DefaultCrioGenPath = filepath.Join(cwd, "runtime/crio") + } +} + +func NewCrioCommand() *cobra.Command { + var defaultZotConfig *registry.DefaultZotConfig + var generateConfig bool + var crioConfigPath string + + crioCmd := &cobra.Command{ + Use: "crio", + Short: "Creates the config file for the crio runtime to fetch the images from the local repository", + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + return SetupContainerRuntimeCommand(cmd, &defaultZotConfig, DefaultCrioGenPath) + }, + RunE: func(cmd *cobra.Command, args []string) error { + log := logger.FromContext(cmd.Context()) + if generateConfig { + log.Info().Msg("Generating the config file for crio ...") + log.Info().Msgf("Fetching crio registry config file form path: %s", crioConfigPath) + // Generate the config file + err := GenerateCrioRegistryConfig(defaultZotConfig, crioConfigPath, log) + if err != nil { + log.Err(err).Msg("Error generating crio registry config") + return err + } + } + return nil + }, + } + crioCmd.Flags().BoolVarP(&generateConfig, "gen", "g", false, "Generate the config file") + crioCmd.PersistentFlags().StringVarP(&crioConfigPath, "config", "c", DefaultCrioRegistryConfigPath, "Path to the crio registry config file") + return crioCmd +} + +func GenerateCrioRegistryConfig(defaultZotConfig *registry.DefaultZotConfig, crioConfigPath string, log *zerolog.Logger) error { + // Read the current crio registry config file + data, err := utils.ReadFile(crioConfigPath, false) + if err != nil { + return fmt.Errorf("could not read crio registry config file: %w", err) + } + var crioRegistryConfig CriORegistryConfig + err = toml.Unmarshal(data, &crioRegistryConfig) + if err != nil { + log.Err(err).Msg("Error unmarshalling crio registry config") + return fmt.Errorf("could not unmarshal crio registry config: %w", err) + } + // Update the crio registry config file + // - Add the local registry to the unqualified search registries if not already present + var found bool = false + var localRegistry string = utils.FormatRegistryURL(defaultZotConfig.GetLocalRegistryURL()) + for _, registry := range crioRegistryConfig.UnqualifiedSearchRegistries { + if registry == localRegistry { + found = true + break + } + } + if !found { + crioRegistryConfig.UnqualifiedSearchRegistries = append(crioRegistryConfig.UnqualifiedSearchRegistries, localRegistry) + } + // Now range over the registries and find if there is a registry with the prefix satellite + // If there is a registry with the prefix satellite, update the location to the local registry + found = false + for _, registries := range crioRegistryConfig.Registries { + if registries.Prefix == "satellite" { + found = true + if registries.Location == "" { + registries.Location = DockerURL + } + // Add the local registry to the first position in the mirrors + mirror := Mirror{ + Location: localRegistry, + Insecure: config.UseUnsecure(), + } + registries.Mirrors = append([]Mirror{mirror}, registries.Mirrors...) + } + } + if !found { + // Add the satellite registry to the registries + registry := Registry{ + Prefix: "satellite", + Location: DockerURL, + Mirrors: []Mirror{ + { + Location: localRegistry, + Insecure: config.UseUnsecure(), + }, + }, + } + crioRegistryConfig.Registries = append(crioRegistryConfig.Registries, registry) + } + // Now marshal the updated crio registry config + updatedData, err := toml.Marshal(crioRegistryConfig) + if err != nil { + log.Err(err).Msg("Error marshalling crio registry config") + return fmt.Errorf("could not marshal crio registry config: %w", err) + } + // Write the updated crio registry config to the file + pathToWrite := filepath.Join(DefaultCrioGenPath, "crio.conf") + log.Info().Msgf("Writing the crio registry config to path: %s", pathToWrite) + err = utils.WriteFile(pathToWrite, updatedData) + if err != nil { + log.Err(err).Msg("Error writing crio registry config") + return fmt.Errorf("could not write crio registry config: %w", err) + } + log.Info().Msg("Successfully wrote the crio registry config") + return nil +} + +func SetupContainerRuntimeCommand(cmd *cobra.Command, defaultZotConfig **registry.DefaultZotConfig, defaultGenPath string) error { + var err error + utils.SetupContextForCommand(cmd) + config.InitConfig() + log := logger.FromContext(cmd.Context()) + + if config.GetOwnRegistry() { + log.Info().Msg("Using own registry for config generation") + address, err := utils.ValidateRegistryAddress(config.GetOwnRegistryAdr(), config.GetOwnRegistryPort()) + if err != nil { + log.Err(err).Msg("Error validating registry address") + return err + } + log.Info().Msgf("Registry address validated: %s", address) + (*defaultZotConfig).HTTP.Address = config.GetOwnRegistryAdr() + (*defaultZotConfig).HTTP.Port = config.GetOwnRegistryPort() + } else { + log.Info().Msg("Using default registry for config generation") + *defaultZotConfig, err = registry.ReadConfig(config.GetZotConfigPath()) + if err != nil || *defaultZotConfig == nil { + return fmt.Errorf("could not read config: %w", err) + } + log.Info().Msgf("Default config read successfully: %v", (*defaultZotConfig).HTTP.Address+":"+(*defaultZotConfig).HTTP.Port) + } + return utils.CreateRuntimeDirectory(defaultGenPath) +} diff --git a/cmd/container_runtime/crio_config.go b/cmd/container_runtime/crio_config.go new file mode 100644 index 0000000..8575a5a --- /dev/null +++ b/cmd/container_runtime/crio_config.go @@ -0,0 +1,51 @@ +package runtime + +// CriORegistryConfig represents the overall configuration for container image registries. +type CriORegistryConfig struct { + // UnqualifiedSearchRegistries is an array of host[:port] registries to try + // when pulling an unqualified image, in the specified order. + UnqualifiedSearchRegistries []string `toml:"unqualified-search-registries,omitempty"` + + // Registries is a list of registry configurations, each defining the behavior for a specific prefix or namespace. + Registries []Registry `toml:"registry,omitempty"` +} + +// Registry represents a specific registry configuration. +type Registry struct { + // Prefix is used to choose the relevant [[registry]] TOML table. + // Only the table with the longest match for the input image name + // (considering namespace/repo/tag/digest separators) is used. + // If this field is missing, it defaults to the value of Location. + // Example: "example.com/foo" + Prefix string `toml:"prefix,omitempty"` + + // Insecure allows unencrypted HTTP as well as TLS connections with untrusted certificates + // if set to true. This should only be enabled for trusted registries to avoid security risks. + Insecure bool `toml:"insecure,omitempty"` + + // Blocked, if set to true, prevents pulling images with matching names from this registry. + // This can be used to blacklist certain registries. + Blocked bool `toml:"blocked,omitempty"` + + // Location specifies the physical location of the "prefix"-rooted namespace. + // By default, this is equal to "prefix". It can be empty for wildcarded prefixes (e.g., "*.example.com"), + // in which case the input reference is used as-is without modification. + // Example: "internal-registry-for-example.com/bar" + Location string `toml:"location,omitempty"` + + // Mirrors is an array of potential mirror locations for the "prefix"-rooted namespace. + // Mirrors are attempted in the specified order; the first reachable mirror containing the image + // is used. If no mirror has the image, the primary location or the unmodified user-specified reference is tried last. + Mirrors []Mirror `toml:"mirror,omitempty"` +} + +// Mirror represents a mirror registry configuration. +type Mirror struct { + // Location specifies the address of the mirror. The mirror will be used if it contains the image. + // Example: "example-mirror-0.local/mirror-for-foo" + Location string `toml:"location,omitempty"` + + // Insecure allows access to the mirror over unencrypted HTTP or with untrusted TLS certificates + // if set to true. This should be used cautiously. + Insecure bool `toml:"insecure,omitempty"` +} diff --git a/cmd/container_runtime/host.go b/cmd/container_runtime/host.go index 410001d..f6789e8 100644 --- a/cmd/container_runtime/host.go +++ b/cmd/container_runtime/host.go @@ -16,7 +16,7 @@ const ( SatelliteConfigPath = "satellite" HostToml = "host_gen.toml" DefaultTomlConfigPath = "_default" - DockerURL = "https://registry-1.docker.io" + DockerURL = "docker.io" ) type ContainerdHostConfig struct { diff --git a/cmd/root.go b/cmd/root.go index 06926b4..480f4aa 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -32,6 +32,7 @@ func NewRootCommand() *cobra.Command { }, } rootCmd.AddCommand(runtime.NewContainerdCommand()) + rootCmd.AddCommand(runtime.NewCrioCommand()) return rootCmd } From f7540854e8aa1ae21aae23af41b84ab384fa31b4 Mon Sep 17 00:00:00 2001 From: Mehul-Kumar-27 <202151092@iiitvadodara.ac.in> Date: Tue, 5 Nov 2024 00:55:38 +0530 Subject: [PATCH 27/36] moving from toml config to json config --- .gitignore | 1 + config.json | 15 ++ internal/config/config.go | 374 +++++++++++++++++----------------- internal/config/new_config.go | 136 +++++++++++++ internal/server/server.go | 5 +- internal/state/fetcher.go | 29 +-- main.go | 15 +- 7 files changed, 357 insertions(+), 218 deletions(-) create mode 100644 config.json create mode 100644 internal/config/new_config.go diff --git a/.gitignore b/.gitignore index 32234d8..420e0d1 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,4 @@ secrets.txt __debug_bin1949266242 /zot +/runtime \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..1e90a3e --- /dev/null +++ b/config.json @@ -0,0 +1,15 @@ +{ + "auth": { + "name": "admin", + "registry": "https://registry.bupd.xyz", + "secret": "Harbor12345" + }, + "bring_own_registry": false, + "ground_control_url": "http://localhost:8080", + "log_level": "info", + "own_registry_adr": "127.0.0.1", + "own_registry_port": "8585", + "states": ["https://registry.bupd.xyz/satellite/group10/state:latest"], + "url_or_file": "https://registry.bupd.xyz", + "zotconfigpath": "./registry/config.json" +} diff --git a/internal/config/config.go b/internal/config/config.go index d4a9946..11d0d59 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,193 +1,185 @@ package config -import ( - "fmt" - "os" - - "github.com/joho/godotenv" - "github.com/spf13/viper" -) - -var AppConfig *Config - -type Config struct { - log_level string - own_registry bool - own_registry_adr string - own_registry_port string - zot_config_path string - input string - zot_url string - registry string - repository string - user_input string - scheme string - api_version string - image string - harbor_password string - harbor_username string - env string - use_unsecure bool - remote_registry_url string - group_name string - state_artifact_name string - state_fetch_period string -} - -func GetLogLevel() string { - return AppConfig.log_level -} - -func GetOwnRegistry() bool { - return AppConfig.own_registry -} - -func GetOwnRegistryAdr() string { - return AppConfig.own_registry_adr -} - -func GetOwnRegistryPort() string { - return AppConfig.own_registry_port -} - -func GetZotConfigPath() string { - return AppConfig.zot_config_path -} - -func GetInput() string { - return AppConfig.input -} - -func SetZotURL(url string) { - AppConfig.zot_url = url -} - -func GetZotURL() string { - return AppConfig.zot_url -} - -func SetRegistry(registry string) { - AppConfig.registry = registry -} - -func GetRegistry() string { - return AppConfig.registry -} - -func SetRepository(repository string) { - AppConfig.repository = repository -} - -func GetRepository() string { - return AppConfig.repository -} - -func SetUserInput(user_input string) { - AppConfig.user_input = user_input -} - -func GetUserInput() string { - return AppConfig.user_input -} - -func SetScheme(scheme string) { - AppConfig.scheme = scheme -} - -func GetScheme() string { - return AppConfig.scheme -} - -func SetAPIVersion(api_version string) { - AppConfig.api_version = api_version -} - -func GetAPIVersion() string { - return AppConfig.api_version -} - -func SetImage(image string) { - AppConfig.image = image -} - -func GetImage() string { - return AppConfig.image -} - -func UseUnsecure() bool { - return AppConfig.use_unsecure -} - -func GetHarborPassword() string { - return AppConfig.harbor_password -} - -func GetHarborUsername() string { - return AppConfig.harbor_username -} - -func SetRemoteRegistryURL(url string) { - AppConfig.remote_registry_url = url -} - -func GetRemoteRegistryURL() string { - return AppConfig.remote_registry_url -} - -func GetGroupName() string { - return AppConfig.group_name -} - -func GetStateArtifactName() string { - return AppConfig.state_artifact_name -} - -func GetStateFetchPeriod() string { - return AppConfig.state_fetch_period -} - -func LoadConfig() (*Config, error) { - viper.SetConfigName("config") - viper.SetConfigType("toml") - viper.AddConfigPath(".") - if err := viper.ReadInConfig(); err != nil { - return nil, fmt.Errorf("error reading config file at path '%s': %w", viper.ConfigFileUsed(), err) - } - - // Load environment and start satellite - if err := godotenv.Load(); err != nil { - return &Config{}, fmt.Errorf("error loading .env file: %w", err) - } - var use_unsecure bool - if os.Getenv("USE_UNSECURE") == "true" { - use_unsecure = true - } else { - use_unsecure = false - } - - return &Config{ - log_level: viper.GetString("log_level"), - own_registry: viper.GetBool("bring_own_registry"), - own_registry_adr: viper.GetString("own_registry_adr"), - own_registry_port: viper.GetString("own_registry_port"), - zot_config_path: viper.GetString("zotConfigPath"), - input: viper.GetString("url_or_file"), - harbor_password: os.Getenv("HARBOR_PASSWORD"), - harbor_username: os.Getenv("HARBOR_USERNAME"), - env: os.Getenv("ENV"), - zot_url: os.Getenv("ZOT_URL"), - use_unsecure: use_unsecure, - group_name: os.Getenv("GROUP_NAME"), - state_artifact_name: os.Getenv("STATE_ARTIFACT_NAME"), - state_fetch_period: os.Getenv("STATE_FETCH_PERIOD"), - }, nil -} - -func InitConfig() error { - var err error - AppConfig, err = LoadConfig() - if err != nil { - return err - } - return nil -} +// import ( +// "fmt" +// "os" + +// "github.com/joho/godotenv" +// "github.com/spf13/viper" +// ) + +// var AppConfig *Config + +// type Config struct { +// log_level string +// own_registry bool +// own_registry_adr string +// own_registry_port string +// zot_config_path string +// input string +// zot_url string +// registry string +// repository string +// user_input string +// scheme string +// api_version string +// image string +// harbor_password string +// harbor_username string +// env string +// use_unsecure bool +// remote_registry_url string +// group_name string +// state_artifact_name string +// state_fetch_period string +// } + +// func GetLogLevel() string { +// return AppConfig.log_level +// } + +// func GetOwnRegistry() bool { +// return AppConfig.own_registry +// } + +// func GetOwnRegistryAdr() string { +// return AppConfig.own_registry_adr +// } + +// func GetOwnRegistryPort() string { +// return AppConfig.own_registry_port +// } + +// func GetZotConfigPath() string { +// return AppConfig.zot_config_path +// } + +// func GetInput() string { +// return AppConfig.input +// } + +// func SetZotURL(url string) { +// AppConfig.zot_url = url +// } + +// func GetZotURL() string { +// return AppConfig.zot_url +// } + +// func SetRegistry(registry string) { +// AppConfig.registry = registry +// } + +// func GetRegistry() string { +// return AppConfig.registry +// } + +// func SetRepository(repository string) { +// AppConfig.repository = repository +// } + +// func GetRepository() string { +// return AppConfig.repository +// } + +// func SetUserInput(user_input string) { +// AppConfig.user_input = user_input +// } + +// func GetUserInput() string { +// return AppConfig.user_input +// } + +// func SetScheme(scheme string) { +// AppConfig.scheme = scheme +// } + +// func GetScheme() string { +// return AppConfig.scheme +// } + +// func SetAPIVersion(api_version string) { +// AppConfig.api_version = api_version +// } + +// func GetAPIVersion() string { +// return AppConfig.api_version +// } + +// func SetImage(image string) { +// AppConfig.image = image +// } + +// func GetImage() string { +// return AppConfig.image +// } + +// func UseUnsecure() bool { +// return AppConfig.use_unsecure +// } + +// func GetHarborPassword() string { +// return AppConfig.harbor_password +// } + +// func GetHarborUsername() string { +// return AppConfig.harbor_username +// } + +// func SetRemoteRegistryURL(url string) { +// AppConfig.remote_registry_url = url +// } + +// func GetRemoteRegistryURL() string { +// return AppConfig.remote_registry_url +// } + +// func GetGroupName() string { +// return AppConfig.group_name +// } + +// func GetStateArtifactName() string { +// return AppConfig.state_artifact_name +// } + +// func GetStateFetchPeriod() string { +// return AppConfig.state_fetch_period +// } + +// func LoadConfig() (*Config, error) { +// viper.SetConfigName("config") +// viper.SetConfigType("toml") +// viper.AddConfigPath(".") +// if err := viper.ReadInConfig(); err != nil { +// return nil, fmt.Errorf("error reading config file at path '%s': %w", viper.ConfigFileUsed(), err) +// } + +// // Load environment and start satellite +// if err := godotenv.Load(); err != nil { +// return &Config{}, fmt.Errorf("error loading .env file: %w", err) +// } +// var use_unsecure bool +// if os.Getenv("USE_UNSECURE") == "true" { +// use_unsecure = true +// } else { +// use_unsecure = false +// } + +// return &Config{ +// log_level: viper.GetString("log_level"), +// own_registry: viper.GetBool("bring_own_registry"), +// own_registry_adr: viper.GetString("own_registry_adr"), +// own_registry_port: viper.GetString("own_registry_port"), +// zot_config_path: viper.GetString("zotConfigPath"), +// input: viper.GetString("url_or_file"), +// harbor_password: os.Getenv("HARBOR_PASSWORD"), +// harbor_username: os.Getenv("HARBOR_USERNAME"), +// env: os.Getenv("ENV"), +// zot_url: os.Getenv("ZOT_URL"), +// use_unsecure: use_unsecure, +// group_name: os.Getenv("GROUP_NAME"), +// state_artifact_name: os.Getenv("STATE_ARTIFACT_NAME"), +// state_fetch_period: os.Getenv("STATE_FETCH_PERIOD"), +// }, nil +// } + diff --git a/internal/config/new_config.go b/internal/config/new_config.go new file mode 100644 index 0000000..8dc3463 --- /dev/null +++ b/internal/config/new_config.go @@ -0,0 +1,136 @@ +package config + +import ( + "encoding/json" + "fmt" + "os" +) + +var appConfig *Config + +const DefaultConfigPath string = "config.json" + +type Auth struct { + Name string `json:"name"` + Registry string `json:"registry"` + Secret string `json:"secret"` +} + +type Config struct { + Auths Auth `json:"auth"` + BringOwnRegistry bool `json:"bring_own_registry"` + GroundControlURL string `json:"ground_control_url"` + LogLevel string `json:"log_level"` + OwnRegistryAddress string `json:"own_registry_adr"` + OwnRegistryPort string `json:"own_registry_port"` + States []string `json:"states"` + URLOrFile string `json:"url_or_file"` + ZotConfigPath string `json:"zotconfigpath"` + UseUnsecure bool `json:"use_unsecure"` + ZotUrl string `json:"zot_url"` + StateFetchPeriod string `json:"state_fetch_period"` +} + +func GetLogLevel() string { + return appConfig.LogLevel +} + +func GetOwnRegistry() bool { + return appConfig.BringOwnRegistry +} + +func GetOwnRegistryAdr() string { + return appConfig.OwnRegistryAddress +} + +func GetOwnRegistryPort() string { + return appConfig.OwnRegistryPort +} + +func GetZotConfigPath() string { + return appConfig.ZotConfigPath +} + +func GetInput() string { + return appConfig.URLOrFile +} + +func SetZotURL(url string) { + appConfig.ZotUrl = url +} + +func GetZotURL() string { + return appConfig.ZotUrl +} + +func UseUnsecure() bool { + return appConfig.UseUnsecure +} + +func GetHarborPassword() string { + return appConfig.Auths.Secret +} + +func GetHarborUsername() string { + return appConfig.Auths.Name +} + +func SetRemoteRegistryURL(url string) { + appConfig.Auths.Registry = url +} + +func GetRemoteRegistryURL() string { + return appConfig.Auths.Registry +} + +func GetStateFetchPeriod() string { + return appConfig.StateFetchPeriod +} + +func ParseConfigFromJson(jsonData string) (*Config, error) { + var config Config + err := json.Unmarshal([]byte(jsonData), &config) + if err != nil { + return nil, err + } + return &config, nil +} + +func ReadConfigData(configPath string) ([]byte, error) { + + fileInfo, err := os.Stat(configPath) + if err != nil { + return nil, err + } + if fileInfo.IsDir() { + return nil, os.ErrNotExist + } + data, err := os.ReadFile(configPath) + if err != nil { + return nil, err + } + return data, nil +} + +func LoadConfig() (*Config, error) { + configData, err := ReadConfigData(DefaultConfigPath) + if err != nil { + fmt.Printf("Error reading config file: %v\n", err) + os.Exit(1) + } + config, err := ParseConfigFromJson(string(configData)) + if err != nil { + fmt.Printf("Error parsing config file: %v\n", err) + os.Exit(1) + } + return config, nil +} + +func InitConfig() error { + var err error + appConfig, err = LoadConfig() + if err != nil { + return err + } + return nil +} diff --git a/internal/server/server.go b/internal/server/server.go index 019ed01..661e98c 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -6,7 +6,6 @@ import ( "fmt" "net/http" - "container-registry.com/harbor-satellite/internal/config" "github.com/rs/zerolog" "golang.org/x/sync/errgroup" ) @@ -22,16 +21,14 @@ type App struct { server *http.Server ctx context.Context Logger *zerolog.Logger - config *config.Config } -func NewApp(router Router, ctx context.Context, logger *zerolog.Logger, config *config.Config, registrars ...RouteRegistrar) *App { +func NewApp(router Router, ctx context.Context, logger *zerolog.Logger, registrars ...RouteRegistrar) *App { return &App{ router: router, registrars: registrars, ctx: ctx, Logger: logger, - config: config, server: &http.Server{Addr: ":9090", Handler: router}, } } diff --git a/internal/state/fetcher.go b/internal/state/fetcher.go index d2f9710..b72cc59 100644 --- a/internal/state/fetcher.go +++ b/internal/state/fetcher.go @@ -19,9 +19,8 @@ type StateFetcher interface { } type baseStateFetcher struct { - group_name string - state_artifact_name string - state_artifact_reader StateReader + username string + password string } type URLStateFetcher struct { @@ -34,27 +33,24 @@ type FileStateArtifactFetcher struct { filePath string } -func NewURLStateFetcher() StateFetcher { - url := config.GetRemoteRegistryURL() - url = utils.FormatRegistryURL(url) +func NewURLStateFetcher(stateURL, userName, password string) StateFetcher { + url := utils.FormatRegistryURL(stateURL) return &URLStateFetcher{ baseStateFetcher: baseStateFetcher{ - group_name: config.GetGroupName(), - state_artifact_name: config.GetStateArtifactName(), - state_artifact_reader: NewState(), + username: userName, + password: password, }, url: url, } } -func NewFileStateFetcher() StateFetcher { +func NewFileStateFetcher(filePath, userName, password string) StateFetcher { return &FileStateArtifactFetcher{ baseStateFetcher: baseStateFetcher{ - group_name: config.GetGroupName(), - state_artifact_name: config.GetStateArtifactName(), - state_artifact_reader: NewState(), + username: userName, + password: password, }, - filePath: config.GetInput(), + filePath: filePath, } } @@ -81,10 +77,7 @@ func (f *URLStateFetcher) FetchStateArtifact(state interface{}) error { options = append(options, crane.Insecure) } - sourceRegistry := utils.FormatRegistryURL(config.GetRemoteRegistryURL()) - tag := "latest" - - img, err := crane.Pull(fmt.Sprintf("%s/%s/%s:%s", sourceRegistry, f.group_name, f.state_artifact_name, tag), options...) + img, err := crane.Pull(f.url, options...) if err != nil { return fmt.Errorf("failed to pull the state artifact: %v", err) } diff --git a/main.go b/main.go index 37e46a9..11467f2 100644 --- a/main.go +++ b/main.go @@ -91,7 +91,6 @@ func setupServerApp(ctx context.Context, log *zerolog.Logger) *server.App { router, ctx, log, - config.AppConfig, &server.MetricsRegistrar{}, &server.DebugRegistrar{}, &satellite.SatelliteRegistrar{}, @@ -131,20 +130,26 @@ func processInput(ctx context.Context, log *zerolog.Logger) (state.StateFetcher, return nil, err } - return processFileInput(log) + return processFileInput(input, log) } func processURLInput(input string, log *zerolog.Logger) (state.StateFetcher, error) { log.Info().Msg("Input is a valid URL") config.SetRemoteRegistryURL(input) - stateArtifactFetcher := state.NewURLStateFetcher() + username := config.GetHarborUsername() + password := config.GetHarborPassword() + + stateArtifactFetcher := state.NewURLStateFetcher(input, username, password) return stateArtifactFetcher, nil } -func processFileInput(log *zerolog.Logger) (state.StateFetcher, error) { - stateArtifactFetcher := state.NewFileStateFetcher() +func processFileInput(input string, log *zerolog.Logger) (state.StateFetcher, error) { + log.Info().Msg("Input is a valid file path") + username := config.GetHarborUsername() + password := config.GetHarborPassword() + stateArtifactFetcher := state.NewFileStateFetcher(input, username, password) return stateArtifactFetcher, nil } From abb1a3ece04bd76a3766ae25648590abfb66ac88 Mon Sep 17 00:00:00 2001 From: Mehul-Kumar-27 <202151092@iiitvadodara.ac.in> Date: Tue, 5 Nov 2024 02:08:20 +0530 Subject: [PATCH 28/36] making config.json work with the replicator --- config.json | 5 +- internal/config/config.go | 321 ++++++++++++++------------------ internal/config/new_config.go | 136 -------------- internal/satellite/satellite.go | 13 +- internal/state/state_process.go | 163 +++++++++++----- internal/utils/utils.go | 2 + main.go | 55 +----- 7 files changed, 267 insertions(+), 428 deletions(-) delete mode 100644 internal/config/new_config.go diff --git a/config.json b/config.json index 1e90a3e..807d2f9 100644 --- a/config.json +++ b/config.json @@ -9,7 +9,8 @@ "log_level": "info", "own_registry_adr": "127.0.0.1", "own_registry_port": "8585", - "states": ["https://registry.bupd.xyz/satellite/group10/state:latest"], + "states": ["https://registry.bupd.xyz/satellite-test-group-state/state:latest"], "url_or_file": "https://registry.bupd.xyz", - "zotconfigpath": "./registry/config.json" + "zotconfigpath": "./registry/config.json", + "use_unsecure": true } diff --git a/internal/config/config.go b/internal/config/config.go index 11d0d59..2e2a9bb 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,185 +1,140 @@ package config -// import ( -// "fmt" -// "os" - -// "github.com/joho/godotenv" -// "github.com/spf13/viper" -// ) - -// var AppConfig *Config - -// type Config struct { -// log_level string -// own_registry bool -// own_registry_adr string -// own_registry_port string -// zot_config_path string -// input string -// zot_url string -// registry string -// repository string -// user_input string -// scheme string -// api_version string -// image string -// harbor_password string -// harbor_username string -// env string -// use_unsecure bool -// remote_registry_url string -// group_name string -// state_artifact_name string -// state_fetch_period string -// } - -// func GetLogLevel() string { -// return AppConfig.log_level -// } - -// func GetOwnRegistry() bool { -// return AppConfig.own_registry -// } - -// func GetOwnRegistryAdr() string { -// return AppConfig.own_registry_adr -// } - -// func GetOwnRegistryPort() string { -// return AppConfig.own_registry_port -// } - -// func GetZotConfigPath() string { -// return AppConfig.zot_config_path -// } - -// func GetInput() string { -// return AppConfig.input -// } - -// func SetZotURL(url string) { -// AppConfig.zot_url = url -// } - -// func GetZotURL() string { -// return AppConfig.zot_url -// } - -// func SetRegistry(registry string) { -// AppConfig.registry = registry -// } - -// func GetRegistry() string { -// return AppConfig.registry -// } - -// func SetRepository(repository string) { -// AppConfig.repository = repository -// } - -// func GetRepository() string { -// return AppConfig.repository -// } - -// func SetUserInput(user_input string) { -// AppConfig.user_input = user_input -// } - -// func GetUserInput() string { -// return AppConfig.user_input -// } - -// func SetScheme(scheme string) { -// AppConfig.scheme = scheme -// } - -// func GetScheme() string { -// return AppConfig.scheme -// } - -// func SetAPIVersion(api_version string) { -// AppConfig.api_version = api_version -// } - -// func GetAPIVersion() string { -// return AppConfig.api_version -// } - -// func SetImage(image string) { -// AppConfig.image = image -// } - -// func GetImage() string { -// return AppConfig.image -// } - -// func UseUnsecure() bool { -// return AppConfig.use_unsecure -// } - -// func GetHarborPassword() string { -// return AppConfig.harbor_password -// } - -// func GetHarborUsername() string { -// return AppConfig.harbor_username -// } - -// func SetRemoteRegistryURL(url string) { -// AppConfig.remote_registry_url = url -// } - -// func GetRemoteRegistryURL() string { -// return AppConfig.remote_registry_url -// } - -// func GetGroupName() string { -// return AppConfig.group_name -// } - -// func GetStateArtifactName() string { -// return AppConfig.state_artifact_name -// } - -// func GetStateFetchPeriod() string { -// return AppConfig.state_fetch_period -// } - -// func LoadConfig() (*Config, error) { -// viper.SetConfigName("config") -// viper.SetConfigType("toml") -// viper.AddConfigPath(".") -// if err := viper.ReadInConfig(); err != nil { -// return nil, fmt.Errorf("error reading config file at path '%s': %w", viper.ConfigFileUsed(), err) -// } - -// // Load environment and start satellite -// if err := godotenv.Load(); err != nil { -// return &Config{}, fmt.Errorf("error loading .env file: %w", err) -// } -// var use_unsecure bool -// if os.Getenv("USE_UNSECURE") == "true" { -// use_unsecure = true -// } else { -// use_unsecure = false -// } - -// return &Config{ -// log_level: viper.GetString("log_level"), -// own_registry: viper.GetBool("bring_own_registry"), -// own_registry_adr: viper.GetString("own_registry_adr"), -// own_registry_port: viper.GetString("own_registry_port"), -// zot_config_path: viper.GetString("zotConfigPath"), -// input: viper.GetString("url_or_file"), -// harbor_password: os.Getenv("HARBOR_PASSWORD"), -// harbor_username: os.Getenv("HARBOR_USERNAME"), -// env: os.Getenv("ENV"), -// zot_url: os.Getenv("ZOT_URL"), -// use_unsecure: use_unsecure, -// group_name: os.Getenv("GROUP_NAME"), -// state_artifact_name: os.Getenv("STATE_ARTIFACT_NAME"), -// state_fetch_period: os.Getenv("STATE_FETCH_PERIOD"), -// }, nil -// } - +import ( + "encoding/json" + "fmt" + "os" +) + +var appConfig *Config + +const DefaultConfigPath string = "config.json" + +type Auth struct { + Name string `json:"name"` + Registry string `json:"registry"` + Secret string `json:"secret"` +} + +type Config struct { + Auth Auth `json:"auth"` + BringOwnRegistry bool `json:"bring_own_registry"` + GroundControlURL string `json:"ground_control_url"` + LogLevel string `json:"log_level"` + OwnRegistryAddress string `json:"own_registry_adr"` + OwnRegistryPort string `json:"own_registry_port"` + States []string `json:"states"` + URLOrFile string `json:"url_or_file"` + ZotConfigPath string `json:"zotconfigpath"` + UseUnsecure bool `json:"use_unsecure"` + ZotUrl string `json:"zot_url"` + StateFetchPeriod string `json:"state_fetch_period"` +} + +func GetLogLevel() string { + return appConfig.LogLevel +} + +func GetOwnRegistry() bool { + return appConfig.BringOwnRegistry +} + +func GetOwnRegistryAdr() string { + return appConfig.OwnRegistryAddress +} + +func GetOwnRegistryPort() string { + return appConfig.OwnRegistryPort +} + +func GetZotConfigPath() string { + return appConfig.ZotConfigPath +} + +func GetInput() string { + return appConfig.URLOrFile +} + +func SetZotURL(url string) { + appConfig.ZotUrl = url +} + +func GetZotURL() string { + return appConfig.ZotUrl +} + +func UseUnsecure() bool { + return appConfig.UseUnsecure +} + +func GetHarborPassword() string { + return appConfig.Auth.Secret +} + +func GetHarborUsername() string { + return appConfig.Auth.Name +} + +func SetRemoteRegistryURL(url string) { + appConfig.Auth.Registry = url +} + +func GetRemoteRegistryURL() string { + return appConfig.Auth.Registry +} + +func GetStateFetchPeriod() string { + return appConfig.StateFetchPeriod +} + +func GetStates() []string { + return appConfig.States +} + +func ParseConfigFromJson(jsonData string) (*Config, error) { + var config Config + err := json.Unmarshal([]byte(jsonData), &config) + if err != nil { + return nil, err + } + return &config, nil +} + +func ReadConfigData(configPath string) ([]byte, error) { + + fileInfo, err := os.Stat(configPath) + if err != nil { + return nil, err + } + if fileInfo.IsDir() { + return nil, os.ErrNotExist + } + data, err := os.ReadFile(configPath) + if err != nil { + return nil, err + } + return data, nil +} + +func LoadConfig() (*Config, error) { + configData, err := ReadConfigData(DefaultConfigPath) + if err != nil { + fmt.Printf("Error reading config file: %v\n", err) + os.Exit(1) + } + config, err := ParseConfigFromJson(string(configData)) + if err != nil { + fmt.Printf("Error parsing config file: %v\n", err) + os.Exit(1) + } + return config, nil +} + +func InitConfig() error { + var err error + appConfig, err = LoadConfig() + if err != nil { + return err + } + return nil +} diff --git a/internal/config/new_config.go b/internal/config/new_config.go deleted file mode 100644 index 8dc3463..0000000 --- a/internal/config/new_config.go +++ /dev/null @@ -1,136 +0,0 @@ -package config - -import ( - "encoding/json" - "fmt" - "os" -) - -var appConfig *Config - -const DefaultConfigPath string = "config.json" - -type Auth struct { - Name string `json:"name"` - Registry string `json:"registry"` - Secret string `json:"secret"` -} - -type Config struct { - Auths Auth `json:"auth"` - BringOwnRegistry bool `json:"bring_own_registry"` - GroundControlURL string `json:"ground_control_url"` - LogLevel string `json:"log_level"` - OwnRegistryAddress string `json:"own_registry_adr"` - OwnRegistryPort string `json:"own_registry_port"` - States []string `json:"states"` - URLOrFile string `json:"url_or_file"` - ZotConfigPath string `json:"zotconfigpath"` - UseUnsecure bool `json:"use_unsecure"` - ZotUrl string `json:"zot_url"` - StateFetchPeriod string `json:"state_fetch_period"` -} - -func GetLogLevel() string { - return appConfig.LogLevel -} - -func GetOwnRegistry() bool { - return appConfig.BringOwnRegistry -} - -func GetOwnRegistryAdr() string { - return appConfig.OwnRegistryAddress -} - -func GetOwnRegistryPort() string { - return appConfig.OwnRegistryPort -} - -func GetZotConfigPath() string { - return appConfig.ZotConfigPath -} - -func GetInput() string { - return appConfig.URLOrFile -} - -func SetZotURL(url string) { - appConfig.ZotUrl = url -} - -func GetZotURL() string { - return appConfig.ZotUrl -} - -func UseUnsecure() bool { - return appConfig.UseUnsecure -} - -func GetHarborPassword() string { - return appConfig.Auths.Secret -} - -func GetHarborUsername() string { - return appConfig.Auths.Name -} - -func SetRemoteRegistryURL(url string) { - appConfig.Auths.Registry = url -} - -func GetRemoteRegistryURL() string { - return appConfig.Auths.Registry -} - -func GetStateFetchPeriod() string { - return appConfig.StateFetchPeriod -} - -func ParseConfigFromJson(jsonData string) (*Config, error) { - var config Config - err := json.Unmarshal([]byte(jsonData), &config) - if err != nil { - return nil, err - } - return &config, nil -} - -func ReadConfigData(configPath string) ([]byte, error) { - - fileInfo, err := os.Stat(configPath) - if err != nil { - return nil, err - } - if fileInfo.IsDir() { - return nil, os.ErrNotExist - } - data, err := os.ReadFile(configPath) - if err != nil { - return nil, err - } - return data, nil -} - -func LoadConfig() (*Config, error) { - configData, err := ReadConfigData(DefaultConfigPath) - if err != nil { - fmt.Printf("Error reading config file: %v\n", err) - os.Exit(1) - } - config, err := ParseConfigFromJson(string(configData)) - if err != nil { - fmt.Printf("Error parsing config file: %v\n", err) - os.Exit(1) - } - return config, nil -} - -func InitConfig() error { - var err error - appConfig, err = LoadConfig() - if err != nil { - return err - } - return nil -} diff --git a/internal/satellite/satellite.go b/internal/satellite/satellite.go index cc1f415..1d0d038 100644 --- a/internal/satellite/satellite.go +++ b/internal/satellite/satellite.go @@ -12,15 +12,13 @@ import ( ) type Satellite struct { - stateReader state.StateReader - stateArtifactFetcher state.StateFetcher - schedulerKey scheduler.SchedulerKey + stateReader state.StateReader + schedulerKey scheduler.SchedulerKey } -func NewSatellite(ctx context.Context, stateArtifactFetcher state.StateFetcher, schedulerKey scheduler.SchedulerKey) *Satellite { +func NewSatellite(ctx context.Context, schedulerKey scheduler.SchedulerKey) *Satellite { return &Satellite{ - stateArtifactFetcher: stateArtifactFetcher, - schedulerKey: schedulerKey, + schedulerKey: schedulerKey, } } @@ -45,7 +43,8 @@ func (s *Satellite) Run(ctx context.Context) error { // Create a simple notifier and add it to the process notifier := notifier.NewSimpleNotifier(ctx) // Creating a process to fetch and replicate the state - fetchAndReplicateStateProcess := state.NewFetchAndReplicateStateProcess(scheduler.NextID(), cronExpr, s.stateArtifactFetcher, notifier, userName, password, zotURL, sourceRegistry, useUnsecure) + states := config.GetStates() + fetchAndReplicateStateProcess := state.NewFetchAndReplicateStateProcess(scheduler.NextID(), cronExpr, notifier, userName, password, zotURL, sourceRegistry, useUnsecure, states) // Add the process to the scheduler scheduler.Schedule(fetchAndReplicateStateProcess) diff --git a/internal/state/state_process.go b/internal/state/state_process.go index 14bcc42..a130d09 100644 --- a/internal/state/state_process.go +++ b/internal/state/state_process.go @@ -6,6 +6,7 @@ import ( "fmt" "sync" + "container-registry.com/harbor-satellite/internal/config" "container-registry.com/harbor-satellite/internal/notifier" "container-registry.com/harbor-satellite/internal/utils" "container-registry.com/harbor-satellite/logger" @@ -25,26 +26,38 @@ type FetchAndReplicateAuthConfig struct { } type FetchAndReplicateStateProcess struct { - id uint64 - name string - stateArtifactFetcher StateFetcher - cronExpr string - isRunning bool - stateReader StateReader - notifier notifier.Notifier - mu *sync.Mutex - authConfig FetchAndReplicateAuthConfig -} - -func NewFetchAndReplicateStateProcess(id uint64, cronExpr string, stateFetcher StateFetcher, notifier notifier.Notifier, username, password, remoteRegistryURL, sourceRegistryURL string, useUnsecure bool) *FetchAndReplicateStateProcess { + id uint64 + name string + cronExpr string + isRunning bool + stateMap []StateMap + notifier notifier.Notifier + mu *sync.Mutex + authConfig FetchAndReplicateAuthConfig +} + +type StateMap struct { + url string + State StateReader +} + +func NewStateMap(url []string) []StateMap { + var stateMap []StateMap + for _, u := range url { + stateMap = append(stateMap, StateMap{url: u, State: nil}) + } + return stateMap +} + +func NewFetchAndReplicateStateProcess(id uint64, cronExpr string, notifier notifier.Notifier, username, password, remoteRegistryURL, sourceRegistryURL string, useUnsecure bool, states []string) *FetchAndReplicateStateProcess { return &FetchAndReplicateStateProcess{ - id: id, - name: FetchAndReplicateStateProcessName, - cronExpr: cronExpr, - isRunning: false, - stateArtifactFetcher: stateFetcher, - notifier: notifier, - mu: &sync.Mutex{}, + id: id, + name: FetchAndReplicateStateProcessName, + cronExpr: cronExpr, + isRunning: false, + notifier: notifier, + mu: &sync.Mutex{}, + stateMap: NewStateMap(states), authConfig: FetchAndReplicateAuthConfig{ Username: username, Password: password, @@ -62,32 +75,49 @@ func (f *FetchAndReplicateStateProcess) Execute(ctx context.Context) error { return fmt.Errorf("process %s already running", f.GetName()) } defer f.stop() - newStateFetched, err := f.FetchAndProcessState(log) - if err != nil { - return err - } - log.Info().Msg("State fetched successfully") - deleteEntity, replicateEntity, newState := f.GetChanges(newStateFetched, log) - f.LogChanges(deleteEntity, replicateEntity, log) - if err := f.notifier.Notify(); err != nil { - log.Error().Err(err).Msg("Error sending notification") - } - replicator := NewBasicReplicator(f.authConfig.Username, f.authConfig.Password, f.authConfig.remoteRegistryURL, f.authConfig.sourceRegistry, f.authConfig.useUnsecure) - // Delete the entities from the remote registry - if err := replicator.DeleteReplicationEntity(ctx, deleteEntity); err != nil { - log.Error().Err(err).Msg("Error deleting entities") - return err - } - // Replicate the entities to the remote registry - if err := replicator.Replicate(ctx, replicateEntity); err != nil { - log.Error().Err(err).Msg("Error replicating state") - return err + + for _, state := range f.stateMap { + log.Info().Msgf("Processing state for %s", state.url) + stateFetcher, err := processInput(state.url, f.authConfig.Username, f.authConfig.Password, log) + if err != nil { + log.Error().Err(err).Msg("Error processing input") + return err + } + newStateFetched, err := f.FetchAndProcessState(stateFetcher, log) + if err != nil { + log.Error().Err(err).Msg("Error fetching state") + return err + } + log.Info().Msgf("State fetched successfully for %s", state.url) + deleteEntity, replicateEntity, newState := f.GetChanges(newStateFetched, log, state.State) + f.LogChanges(deleteEntity, replicateEntity, log) + if err := f.notifier.Notify(); err != nil { + log.Error().Err(err).Msg("Error sending notification") + } + log.Info().Msg("Replicating state") + log.Info().Msg("Replicator config") + log.Info().Msgf("Username: %s", f.authConfig.Username) + log.Info().Msgf("Password: %s", f.authConfig.Password) + log.Info().Msgf("Remote registry URL: %s", f.authConfig.remoteRegistryURL) + log.Info().Msgf("Source registry: %s", f.authConfig.sourceRegistry) + + replicator := NewBasicReplicator(f.authConfig.Username, f.authConfig.Password, f.authConfig.remoteRegistryURL, f.authConfig.sourceRegistry, f.authConfig.useUnsecure) + // Delete the entities from the remote registry + if err := replicator.DeleteReplicationEntity(ctx, deleteEntity); err != nil { + log.Error().Err(err).Msg("Error deleting entities") + return err + } + // Replicate the entities to the remote registry + if err := replicator.Replicate(ctx, replicateEntity); err != nil { + log.Error().Err(err).Msg("Error replicating state") + return err + } + state.State = newState } - f.stateReader = newState return nil } -func (f *FetchAndReplicateStateProcess) GetChanges(newState StateReader, log *zerolog.Logger) ([]ArtifactReader, []ArtifactReader, StateReader) { +func (f *FetchAndReplicateStateProcess) GetChanges(newState StateReader, log *zerolog.Logger, oldState StateReader) ([]ArtifactReader, []ArtifactReader, StateReader) { log.Info().Msg("Getting changes") // Remove artifacts with null tags from the new state newState = f.RemoveNullTagArtifacts(newState) @@ -95,14 +125,14 @@ func (f *FetchAndReplicateStateProcess) GetChanges(newState StateReader, log *ze var entityToDelete []ArtifactReader var entityToReplicate []ArtifactReader - if f.stateReader == nil { + if oldState == nil { log.Warn().Msg("Old state is nil") return entityToDelete, newState.GetArtifacts(), newState } - + // Create maps for quick lookups oldArtifactsMap := make(map[string]ArtifactReader) - for _, oldArtifact := range f.stateReader.GetArtifacts() { + for _, oldArtifact := range oldState.GetArtifacts() { tag := oldArtifact.GetTags()[0] oldArtifactsMap[oldArtifact.GetName()+"|"+tag] = oldArtifact } @@ -199,9 +229,9 @@ func ProcessState(state *StateReader) (*StateReader, error) { return state, nil } -func (f *FetchAndReplicateStateProcess) FetchAndProcessState(log *zerolog.Logger) (StateReader, error) { +func (f *FetchAndReplicateStateProcess) FetchAndProcessState(fetcher StateFetcher, log *zerolog.Logger) (StateReader, error) { state := NewState() - err := f.stateArtifactFetcher.FetchStateArtifact(&state) + err := fetcher.FetchStateArtifact(&state) if err != nil { log.Error().Err(err).Msg("Error fetching state artifact") return nil, err @@ -214,3 +244,44 @@ func (f *FetchAndReplicateStateProcess) LogChanges(deleteEntity, replicateEntity log.Warn().Msgf("Total artifacts to delete: %d", len(deleteEntity)) log.Warn().Msgf("Total artifacts to replicate: %d", len(replicateEntity)) } + +func processInput(input, username, password string, log *zerolog.Logger) (StateFetcher, error) { + + if utils.IsValidURL(input) { + return processURLInput(utils.FormatRegistryURL(input), username, password, log) + } + + log.Info().Msg("Input is not a valid URL, checking if it is a file path") + if err := validateFilePath(input, log); err != nil { + return nil, err + } + + return processFileInput(input, username, password, log) +} + +func validateFilePath(path string, log *zerolog.Logger) error { + if utils.HasInvalidPathChars(path) { + log.Error().Msg("Path contains invalid characters") + return fmt.Errorf("invalid file path: %s", path) + } + if err := utils.GetAbsFilePath(path); err != nil { + log.Error().Err(err).Msg("No file found") + return fmt.Errorf("no file found: %s", path) + } + return nil +} + +func processURLInput(input, username, password string, log *zerolog.Logger) (StateFetcher, error) { + log.Info().Msg("Input is a valid URL") + config.SetRemoteRegistryURL(input) + + stateArtifactFetcher := NewURLStateFetcher(input, username, password) + + return stateArtifactFetcher, nil +} + +func processFileInput(input, username, password string, log *zerolog.Logger) (StateFetcher, error) { + log.Info().Msg("Input is a valid file path") + stateArtifactFetcher := NewFileStateFetcher(input, username, password) + return stateArtifactFetcher, nil +} diff --git a/internal/utils/utils.go b/internal/utils/utils.go index 92dcefa..0b2ce7d 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -44,6 +44,8 @@ func HandleOwnRegistry() error { // LaunchDefaultZotRegistry launches the default Zot registry using the Zot config path func LaunchDefaultZotRegistry() error { + defaultZotURL := fmt.Sprintf("%s:%s", "127.0.0.1", "8585") + config.SetZotURL(defaultZotURL) launch, err := registry.LaunchRegistry(config.GetZotConfigPath()) if !launch { return fmt.Errorf("error launching registry: %w", err) diff --git a/main.go b/main.go index 11467f2..2bff96c 100644 --- a/main.go +++ b/main.go @@ -11,7 +11,6 @@ import ( "container-registry.com/harbor-satellite/internal/satellite" "container-registry.com/harbor-satellite/internal/scheduler" "container-registry.com/harbor-satellite/internal/server" - "container-registry.com/harbor-satellite/internal/state" "container-registry.com/harbor-satellite/internal/utils" "container-registry.com/harbor-satellite/logger" "golang.org/x/sync/errgroup" @@ -55,13 +54,8 @@ func run() error { log.Error().Err(err).Msg("Error starting scheduler") return err } - // Process Input (file or URL) - stateArtifactFetcher, err := processInput(ctx, log) - if err != nil || stateArtifactFetcher == nil { - return fmt.Errorf("error processing input: %w", err) - } - satelliteService := satellite.NewSatellite(ctx, stateArtifactFetcher, scheduler.GetSchedulerKey()) + satelliteService := satellite.NewSatellite(ctx, scheduler.GetSchedulerKey()) g.Go(func() error { return satelliteService.Run(ctx) @@ -117,50 +111,3 @@ func handleRegistrySetup(g *errgroup.Group, log *zerolog.Logger, cancel context. } return nil } - -func processInput(ctx context.Context, log *zerolog.Logger) (state.StateFetcher, error) { - input := config.GetInput() - - if utils.IsValidURL(input) { - return processURLInput(input, log) - } - - log.Info().Msg("Input is not a valid URL, checking if it is a file path") - if err := validateFilePath(input, log); err != nil { - return nil, err - } - - return processFileInput(input, log) -} - -func processURLInput(input string, log *zerolog.Logger) (state.StateFetcher, error) { - log.Info().Msg("Input is a valid URL") - config.SetRemoteRegistryURL(input) - - username := config.GetHarborUsername() - password := config.GetHarborPassword() - - stateArtifactFetcher := state.NewURLStateFetcher(input, username, password) - - return stateArtifactFetcher, nil -} - -func processFileInput(input string, log *zerolog.Logger) (state.StateFetcher, error) { - log.Info().Msg("Input is a valid file path") - username := config.GetHarborUsername() - password := config.GetHarborPassword() - stateArtifactFetcher := state.NewFileStateFetcher(input, username, password) - return stateArtifactFetcher, nil -} - -func validateFilePath(path string, log *zerolog.Logger) error { - if utils.HasInvalidPathChars(path) { - log.Error().Msg("Path contains invalid characters") - return fmt.Errorf("invalid file path: %s", path) - } - if err := utils.GetAbsFilePath(path); err != nil { - log.Error().Err(err).Msg("No file found") - return fmt.Errorf("no file found: %s", path) - } - return nil -} From a641df3d5bcd4ebe6b0b2851a044b64a00892580 Mon Sep 17 00:00:00 2001 From: Mehul-Kumar-27 <202151092@iiitvadodara.ac.in> Date: Tue, 5 Nov 2024 02:19:11 +0530 Subject: [PATCH 29/36] avoid printing confedential information in log --- internal/state/state_process.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/internal/state/state_process.go b/internal/state/state_process.go index a130d09..738c2f6 100644 --- a/internal/state/state_process.go +++ b/internal/state/state_process.go @@ -94,13 +94,7 @@ func (f *FetchAndReplicateStateProcess) Execute(ctx context.Context) error { if err := f.notifier.Notify(); err != nil { log.Error().Err(err).Msg("Error sending notification") } - log.Info().Msg("Replicating state") - log.Info().Msg("Replicator config") - log.Info().Msgf("Username: %s", f.authConfig.Username) - log.Info().Msgf("Password: %s", f.authConfig.Password) - log.Info().Msgf("Remote registry URL: %s", f.authConfig.remoteRegistryURL) - log.Info().Msgf("Source registry: %s", f.authConfig.sourceRegistry) - + replicator := NewBasicReplicator(f.authConfig.Username, f.authConfig.Password, f.authConfig.remoteRegistryURL, f.authConfig.sourceRegistry, f.authConfig.useUnsecure) // Delete the entities from the remote registry if err := replicator.DeleteReplicationEntity(ctx, deleteEntity); err != nil { From 2f11137ae024c271a2504df547df9ebb4a34846b Mon Sep 17 00:00:00 2001 From: Mehul-Kumar-27 <202151092@iiitvadodara.ac.in> Date: Tue, 5 Nov 2024 03:05:13 +0530 Subject: [PATCH 30/36] coderabbit fixes --- internal/config/config.go | 4 ++-- internal/state/fetcher.go | 4 ++-- internal/state/state_process.go | 27 ++++++++++++++------------- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index 2e2a9bb..4f6aa02 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -120,12 +120,12 @@ func LoadConfig() (*Config, error) { configData, err := ReadConfigData(DefaultConfigPath) if err != nil { fmt.Printf("Error reading config file: %v\n", err) - os.Exit(1) + return nil, err } config, err := ParseConfigFromJson(string(configData)) if err != nil { fmt.Printf("Error parsing config file: %v\n", err) - os.Exit(1) + return nil, err } return config, nil } diff --git a/internal/state/fetcher.go b/internal/state/fetcher.go index b72cc59..476b2f4 100644 --- a/internal/state/fetcher.go +++ b/internal/state/fetcher.go @@ -68,8 +68,8 @@ func (f *FileStateArtifactFetcher) FetchStateArtifact(state interface{}) error { func (f *URLStateFetcher) FetchStateArtifact(state interface{}) error { auth := authn.FromConfig(authn.AuthConfig{ - Username: config.GetHarborUsername(), - Password: config.GetHarborPassword(), + Username: f.username, + Password: f.password, }) options := []crane.Option{crane.WithAuth(auth)} diff --git a/internal/state/state_process.go b/internal/state/state_process.go index 738c2f6..3b9c2cb 100644 --- a/internal/state/state_process.go +++ b/internal/state/state_process.go @@ -20,9 +20,9 @@ const DefaultFetchAndReplicateStateTimePeriod string = "00h00m010s" type FetchAndReplicateAuthConfig struct { Username string Password string - useUnsecure bool - remoteRegistryURL string - sourceRegistry string + UseUnsecure bool + RemoteRegistryURL string + SourceRegistry string } type FetchAndReplicateStateProcess struct { @@ -61,9 +61,9 @@ func NewFetchAndReplicateStateProcess(id uint64, cronExpr string, notifier notif authConfig: FetchAndReplicateAuthConfig{ Username: username, Password: password, - useUnsecure: useUnsecure, - remoteRegistryURL: remoteRegistryURL, - sourceRegistry: sourceRegistryURL, + UseUnsecure: useUnsecure, + RemoteRegistryURL: remoteRegistryURL, + SourceRegistry: sourceRegistryURL, }, } } @@ -76,9 +76,9 @@ func (f *FetchAndReplicateStateProcess) Execute(ctx context.Context) error { } defer f.stop() - for _, state := range f.stateMap { - log.Info().Msgf("Processing state for %s", state.url) - stateFetcher, err := processInput(state.url, f.authConfig.Username, f.authConfig.Password, log) + for i := range f.stateMap { + log.Info().Msgf("Processing state for %s", f.stateMap[i].url) + stateFetcher, err := processInput(f.stateMap[i].url, f.authConfig.Username, f.authConfig.Password, log) if err != nil { log.Error().Err(err).Msg("Error processing input") return err @@ -88,14 +88,14 @@ func (f *FetchAndReplicateStateProcess) Execute(ctx context.Context) error { log.Error().Err(err).Msg("Error fetching state") return err } - log.Info().Msgf("State fetched successfully for %s", state.url) - deleteEntity, replicateEntity, newState := f.GetChanges(newStateFetched, log, state.State) + log.Info().Msgf("State fetched successfully for %s", f.stateMap[i].url) + deleteEntity, replicateEntity, newState := f.GetChanges(newStateFetched, log, f.stateMap[i].State) f.LogChanges(deleteEntity, replicateEntity, log) if err := f.notifier.Notify(); err != nil { log.Error().Err(err).Msg("Error sending notification") } - replicator := NewBasicReplicator(f.authConfig.Username, f.authConfig.Password, f.authConfig.remoteRegistryURL, f.authConfig.sourceRegistry, f.authConfig.useUnsecure) + replicator := NewBasicReplicator(f.authConfig.Username, f.authConfig.Password, f.authConfig.RemoteRegistryURL, f.authConfig.SourceRegistry, f.authConfig.UseUnsecure) // Delete the entities from the remote registry if err := replicator.DeleteReplicationEntity(ctx, deleteEntity); err != nil { log.Error().Err(err).Msg("Error deleting entities") @@ -106,7 +106,8 @@ func (f *FetchAndReplicateStateProcess) Execute(ctx context.Context) error { log.Error().Err(err).Msg("Error replicating state") return err } - state.State = newState + // Update the state directly in the slice + f.stateMap[i].State = newState } return nil } From c194bb0b9d43cfa69a225170c7415da8bac39dec Mon Sep 17 00:00:00 2001 From: Mehul-Kumar-27 <202151092@iiitvadodara.ac.in> Date: Tue, 12 Nov 2024 19:27:33 +0530 Subject: [PATCH 31/36] fixing startup --- cmd/root.go | 61 +---------------------------------------------------- 1 file changed, 1 insertion(+), 60 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 480f4aa..66a2c7b 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -2,14 +2,12 @@ package cmd import ( "context" - "fmt" runtime "container-registry.com/harbor-satellite/cmd/container_runtime" "container-registry.com/harbor-satellite/internal/config" "container-registry.com/harbor-satellite/internal/satellite" "container-registry.com/harbor-satellite/internal/scheduler" "container-registry.com/harbor-satellite/internal/server" - "container-registry.com/harbor-satellite/internal/state" "container-registry.com/harbor-satellite/internal/utils" "container-registry.com/harbor-satellite/logger" "github.com/rs/zerolog" @@ -60,16 +58,8 @@ func run(ctx context.Context, cancel context.CancelFunc) error { log.Error().Err(err).Msg("Error starting scheduler") return err } - // Process Input (file or URL) - stateArtifactFetcher, err := processInput(ctx, log) - if err != nil || stateArtifactFetcher == nil { - return fmt.Errorf("error processing input: %w", err) - } - if stateArtifactFetcher == nil { - return fmt.Errorf("state artifact fetcher is nil") - } - satelliteService := satellite.NewSatellite(ctx, stateArtifactFetcher, scheduler.GetSchedulerKey()) + satelliteService := satellite.NewSatellite(ctx, scheduler.GetSchedulerKey()) g.Go(func() error { return satelliteService.Run(ctx) @@ -79,13 +69,6 @@ func run(ctx context.Context, cancel context.CancelFunc) error { return g.Wait() } -func initConfig() error { - if err := config.InitConfig(); err != nil { - return fmt.Errorf("error initializing config: %w", err) - } - return nil -} - func setupServerApp(ctx context.Context, log *zerolog.Logger) *server.App { router := server.NewDefaultRouter("/api/v1") router.Use(server.LoggingMiddleware) @@ -94,7 +77,6 @@ func setupServerApp(ctx context.Context, log *zerolog.Logger) *server.App { router, ctx, log, - config.AppConfig, &server.MetricsRegistrar{}, &server.DebugRegistrar{}, &satellite.SatelliteRegistrar{}, @@ -121,44 +103,3 @@ func handleRegistrySetup(g *errgroup.Group, log *zerolog.Logger, cancel context. } return nil } - -func processInput(ctx context.Context, log *zerolog.Logger) (state.StateFetcher, error) { - input := config.GetInput() - - if utils.IsValidURL(input) { - return processURLInput(input, log) - } - - log.Info().Msg("Input is not a valid URL, checking if it is a file path") - if err := validateFilePath(input, log); err != nil { - return nil, err - } - - return processFileInput(log) -} - -func processURLInput(input string, log *zerolog.Logger) (state.StateFetcher, error) { - log.Info().Msg("Input is a valid URL") - config.SetRemoteRegistryURL(input) - - stateArtifactFetcher := state.NewURLStateFetcher() - - return stateArtifactFetcher, nil -} - -func processFileInput(log *zerolog.Logger) (state.StateFetcher, error) { - stateArtifactFetcher := state.NewFileStateFetcher() - return stateArtifactFetcher, nil -} - -func validateFilePath(path string, log *zerolog.Logger) error { - if utils.HasInvalidPathChars(path) { - log.Error().Msg("Path contains invalid characters") - return fmt.Errorf("invalid file path: %s", path) - } - if err := utils.GetAbsFilePath(path); err != nil { - log.Error().Err(err).Msg("No file found") - return fmt.Errorf("no file found: %s", path) - } - return nil -} From 86430aab596c134b7cc5c6f87eb1a7fba7adafc9 Mon Sep 17 00:00:00 2001 From: Mehul-Kumar-27 <202151092@iiitvadodara.ac.in> Date: Wed, 13 Nov 2024 00:34:00 +0530 Subject: [PATCH 32/36] fixing panic error in generating container runtime config --- cmd/container_runtime/crio.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/container_runtime/crio.go b/cmd/container_runtime/crio.go index aa32da7..c84e70a 100644 --- a/cmd/container_runtime/crio.go +++ b/cmd/container_runtime/crio.go @@ -143,8 +143,8 @@ func GenerateCrioRegistryConfig(defaultZotConfig *registry.DefaultZotConfig, cri func SetupContainerRuntimeCommand(cmd *cobra.Command, defaultZotConfig **registry.DefaultZotConfig, defaultGenPath string) error { var err error - utils.SetupContextForCommand(cmd) config.InitConfig() + utils.SetupContextForCommand(cmd) log := logger.FromContext(cmd.Context()) if config.GetOwnRegistry() { From a65621994fc0036b4122c1316b9fd4eb0ab15723 Mon Sep 17 00:00:00 2001 From: Mehul-Kumar-27 <202151092@iiitvadodara.ac.in> Date: Tue, 19 Nov 2024 15:22:29 +0530 Subject: [PATCH 33/36] minor fixes --- cmd/container_runtime/crio.go | 4 ++-- cmd/container_runtime/host.go | 2 +- runtime/containerd/config.toml | 7 ------- 3 files changed, 3 insertions(+), 10 deletions(-) delete mode 100644 runtime/containerd/config.toml diff --git a/cmd/container_runtime/crio.go b/cmd/container_runtime/crio.go index c84e70a..a4ceab6 100644 --- a/cmd/container_runtime/crio.go +++ b/cmd/container_runtime/crio.go @@ -96,7 +96,7 @@ func GenerateCrioRegistryConfig(defaultZotConfig *registry.DefaultZotConfig, cri // If there is a registry with the prefix satellite, update the location to the local registry found = false for _, registries := range crioRegistryConfig.Registries { - if registries.Prefix == "satellite" { + if registries.Prefix == "satellite.io" { found = true if registries.Location == "" { registries.Location = DockerURL @@ -112,7 +112,7 @@ func GenerateCrioRegistryConfig(defaultZotConfig *registry.DefaultZotConfig, cri if !found { // Add the satellite registry to the registries registry := Registry{ - Prefix: "satellite", + Prefix: "satellite.io", Location: DockerURL, Mirrors: []Mirror{ { diff --git a/cmd/container_runtime/host.go b/cmd/container_runtime/host.go index f6789e8..27b1e49 100644 --- a/cmd/container_runtime/host.go +++ b/cmd/container_runtime/host.go @@ -13,7 +13,7 @@ import ( ) const ( - SatelliteConfigPath = "satellite" + SatelliteConfigPath = "satellite.io" HostToml = "host_gen.toml" DefaultTomlConfigPath = "_default" DockerURL = "docker.io" diff --git a/runtime/containerd/config.toml b/runtime/containerd/config.toml deleted file mode 100644 index bf77b87..0000000 --- a/runtime/containerd/config.toml +++ /dev/null @@ -1,7 +0,0 @@ -version = 2 - - - [plugins."io.containerd.grpc.v1.cri"] - - [plugins."io.containerd.grpc.v1.cri".registry] - config_path = "/etc/containerd/certs.d" From 1e8dc43cb2aa02f4660cb4cacc3ee2745fbeae17 Mon Sep 17 00:00:00 2001 From: Mehul-Kumar-27 <202151092@iiitvadodara.ac.in> Date: Tue, 19 Nov 2024 16:10:51 +0530 Subject: [PATCH 34/36] handelling config error --- cmd/container_runtime/crio.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/container_runtime/crio.go b/cmd/container_runtime/crio.go index a4ceab6..f150c44 100644 --- a/cmd/container_runtime/crio.go +++ b/cmd/container_runtime/crio.go @@ -143,7 +143,10 @@ func GenerateCrioRegistryConfig(defaultZotConfig *registry.DefaultZotConfig, cri func SetupContainerRuntimeCommand(cmd *cobra.Command, defaultZotConfig **registry.DefaultZotConfig, defaultGenPath string) error { var err error - config.InitConfig() + err = config.InitConfig() + if err != nil { + return fmt.Errorf("could not initialize config: %w", err) + } utils.SetupContextForCommand(cmd) log := logger.FromContext(cmd.Context()) From 5ca08c7ae60644589bf3baebed918904bea240b5 Mon Sep 17 00:00:00 2001 From: Mehul-Kumar-27 <202151092@iiitvadodara.ac.in> Date: Tue, 19 Nov 2024 19:02:33 +0530 Subject: [PATCH 35/36] dagger version --- dagger.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dagger.json b/dagger.json index 3328762..b79d3a0 100644 --- a/dagger.json +++ b/dagger.json @@ -2,5 +2,5 @@ "name": "harbor-satellite", "sdk": "go", "source": "ci", - "engineVersion": "v0.13.0" + "engineVersion": "v0.13.3" } From 9782e7e1b9014a60812e9a0047918224cee53c5b Mon Sep 17 00:00:00 2001 From: Mehul-Kumar-27 <202151092@iiitvadodara.ac.in> Date: Tue, 26 Nov 2024 02:01:09 +0530 Subject: [PATCH 36/36] replication fix --- internal/state/helpers.go | 50 ++++++++++++++ internal/state/replicator.go | 40 ++++++++--- internal/state/state_process.go | 115 +++++++++++--------------------- 3 files changed, 119 insertions(+), 86 deletions(-) create mode 100644 internal/state/helpers.go diff --git a/internal/state/helpers.go b/internal/state/helpers.go new file mode 100644 index 0000000..331216d --- /dev/null +++ b/internal/state/helpers.go @@ -0,0 +1,50 @@ +package state + +import ( + "fmt" + + "container-registry.com/harbor-satellite/internal/config" + "container-registry.com/harbor-satellite/internal/utils" + "github.com/rs/zerolog" +) + +func processInput(input, username, password string, log *zerolog.Logger) (StateFetcher, error) { + + if utils.IsValidURL(input) { + return processURLInput(utils.FormatRegistryURL(input), username, password, log) + } + + log.Info().Msg("Input is not a valid URL, checking if it is a file path") + if err := validateFilePath(input, log); err != nil { + return nil, err + } + + return processFileInput(input, username, password, log) +} + +func validateFilePath(path string, log *zerolog.Logger) error { + if utils.HasInvalidPathChars(path) { + log.Error().Msg("Path contains invalid characters") + return fmt.Errorf("invalid file path: %s", path) + } + if err := utils.GetAbsFilePath(path); err != nil { + log.Error().Err(err).Msg("No file found") + return fmt.Errorf("no file found: %s", path) + } + return nil +} + +func processURLInput(input, username, password string, log *zerolog.Logger) (StateFetcher, error) { + log.Info().Msg("Input is a valid URL") + config.SetRemoteRegistryURL(input) + + stateArtifactFetcher := NewURLStateFetcher(input, username, password) + + return stateArtifactFetcher, nil +} + +func processFileInput(input, username, password string, log *zerolog.Logger) (StateFetcher, error) { + log.Info().Msg("Input is a valid file path") + stateArtifactFetcher := NewFileStateFetcher(input, username, password) + return stateArtifactFetcher, nil +} diff --git a/internal/state/replicator.go b/internal/state/replicator.go index 3814879..cb72e40 100644 --- a/internal/state/replicator.go +++ b/internal/state/replicator.go @@ -13,9 +13,9 @@ import ( type Replicator interface { // Replicate copies images from the source registry to the local registry. - Replicate(ctx context.Context, replicationEntities []ArtifactReader) error + Replicate(ctx context.Context, replicationEntities []Entity) error // DeleteReplicationEntity deletes the image from the local registry. - DeleteReplicationEntity(ctx context.Context, replicationEntity []ArtifactReader) error + DeleteReplicationEntity(ctx context.Context, replicationEntity []Entity) error } type BasicReplicator struct { @@ -36,8 +36,28 @@ func NewBasicReplicator(username, password, zotURL, sourceRegistry string, useUn } } +// Entity represents an image or artifact which needs to be handled by the replicator +type Entity struct { + Name string + Repository string + Tag string + Digest string +} + +func (e Entity) GetName() string { + return e.Name +} + +func (e Entity) GetRepository() string { + return e.Repository +} + +func (e Entity) GetTag() string { + return e.Tag +} + // Replicate replicates images from the source registry to the Zot registry. -func (r *BasicReplicator) Replicate(ctx context.Context, replicationEntities []ArtifactReader) error { +func (r *BasicReplicator) Replicate(ctx context.Context, replicationEntities []Entity) error { log := logger.FromContext(ctx) auth := authn.FromConfig(authn.AuthConfig{ Username: r.username, @@ -51,10 +71,10 @@ func (r *BasicReplicator) Replicate(ctx context.Context, replicationEntities []A for _, replicationEntity := range replicationEntities { - log.Info().Msgf("Pulling image %s from repository %s at registry %s with tag %s", replicationEntity.GetName(), replicationEntity.GetRepository(), r.sourceRegistry, replicationEntity.GetTags()[0]) + log.Info().Msgf("Pulling image %s from repository %s at registry %s with tag %s", replicationEntity.GetName(), replicationEntity.GetRepository(), r.sourceRegistry, replicationEntity.GetTag()) // Pull the image from the source registry - srcImage, err := crane.Pull(fmt.Sprintf("%s/%s/%s:%s", r.sourceRegistry, replicationEntity.GetRepository(), replicationEntity.GetName(), replicationEntity.GetTags()[0]), options...) + srcImage, err := crane.Pull(fmt.Sprintf("%s/%s/%s:%s", r.sourceRegistry, replicationEntity.GetRepository(), replicationEntity.GetName(), replicationEntity.GetTag()), options...) if err != nil { log.Error().Msgf("Failed to pull image: %v", err) return err @@ -64,7 +84,7 @@ func (r *BasicReplicator) Replicate(ctx context.Context, replicationEntities []A ociImage := mutate.MediaType(srcImage, types.OCIManifestSchema1) // Push the converted OCI image to the Zot registry - err = crane.Push(ociImage, fmt.Sprintf("%s/%s/%s:%s", r.remoteRegistryURL, replicationEntity.GetRepository(), replicationEntity.GetName(), replicationEntity.GetTags()[0]), options...) + err = crane.Push(ociImage, fmt.Sprintf("%s/%s/%s:%s", r.remoteRegistryURL, replicationEntity.GetRepository(), replicationEntity.GetName(), replicationEntity.GetTag()), options...) if err != nil { log.Error().Msgf("Failed to push image: %v", err) return err @@ -75,7 +95,7 @@ func (r *BasicReplicator) Replicate(ctx context.Context, replicationEntities []A return nil } -func (r *BasicReplicator) DeleteReplicationEntity(ctx context.Context, replicationEntity []ArtifactReader) error { +func (r *BasicReplicator) DeleteReplicationEntity(ctx context.Context, replicationEntity []Entity) error { log := logger.FromContext(ctx) auth := authn.FromConfig(authn.AuthConfig{ Username: r.username, @@ -88,9 +108,9 @@ func (r *BasicReplicator) DeleteReplicationEntity(ctx context.Context, replicati } for _, entity := range replicationEntity { - log.Info().Msgf("Deleting image %s from repository %s at registry %s with tag %s", entity.GetName(), entity.GetRepository(), r.remoteRegistryURL, entity.GetTags()[0]) + log.Info().Msgf("Deleting image %s from repository %s at registry %s with tag %s", entity.GetName(), entity.GetRepository(), r.remoteRegistryURL, entity.GetTag()) - err := crane.Delete(fmt.Sprintf("%s/%s/%s:%s", r.remoteRegistryURL, entity.GetRepository(), entity.GetName() ,entity.GetTags()[0]), options...) + err := crane.Delete(fmt.Sprintf("%s/%s/%s:%s", r.remoteRegistryURL, entity.GetRepository(), entity.GetName(), entity.GetTag()), options...) if err != nil { log.Error().Msgf("Failed to delete image: %v", err) return err @@ -99,4 +119,4 @@ func (r *BasicReplicator) DeleteReplicationEntity(ctx context.Context, replicati } return nil -} \ No newline at end of file +} diff --git a/internal/state/state_process.go b/internal/state/state_process.go index 3b9c2cb..4c10d76 100644 --- a/internal/state/state_process.go +++ b/internal/state/state_process.go @@ -2,11 +2,9 @@ package state import ( "context" - "encoding/json" "fmt" "sync" - "container-registry.com/harbor-satellite/internal/config" "container-registry.com/harbor-satellite/internal/notifier" "container-registry.com/harbor-satellite/internal/utils" "container-registry.com/harbor-satellite/logger" @@ -37,14 +35,15 @@ type FetchAndReplicateStateProcess struct { } type StateMap struct { - url string - State StateReader + url string + State StateReader + Entities []Entity } func NewStateMap(url []string) []StateMap { var stateMap []StateMap for _, u := range url { - stateMap = append(stateMap, StateMap{url: u, State: nil}) + stateMap = append(stateMap, StateMap{url: u, State: nil, Entities: nil}) } return stateMap } @@ -89,7 +88,7 @@ func (f *FetchAndReplicateStateProcess) Execute(ctx context.Context) error { return err } log.Info().Msgf("State fetched successfully for %s", f.stateMap[i].url) - deleteEntity, replicateEntity, newState := f.GetChanges(newStateFetched, log, f.stateMap[i].State) + deleteEntity, replicateEntity, newState := f.GetChanges(newStateFetched, log, f.stateMap[i].Entities) f.LogChanges(deleteEntity, replicateEntity, log) if err := f.notifier.Notify(); err != nil { log.Error().Err(err).Msg("Error sending notification") @@ -108,51 +107,52 @@ func (f *FetchAndReplicateStateProcess) Execute(ctx context.Context) error { } // Update the state directly in the slice f.stateMap[i].State = newState + f.stateMap[i].Entities = FetchEntitiesFromState(newState) } return nil } -func (f *FetchAndReplicateStateProcess) GetChanges(newState StateReader, log *zerolog.Logger, oldState StateReader) ([]ArtifactReader, []ArtifactReader, StateReader) { +func (f *FetchAndReplicateStateProcess) GetChanges(newState StateReader, log *zerolog.Logger, oldEntites []Entity) ([]Entity, []Entity, StateReader) { log.Info().Msg("Getting changes") // Remove artifacts with null tags from the new state newState = f.RemoveNullTagArtifacts(newState) + newEntites := FetchEntitiesFromState(newState) - var entityToDelete []ArtifactReader - var entityToReplicate []ArtifactReader + var entityToDelete []Entity + var entityToReplicate []Entity - if oldState == nil { - log.Warn().Msg("Old state is nil") - return entityToDelete, newState.GetArtifacts(), newState + if oldEntites == nil { + log.Warn().Msg("Old state has zero entites, replicating the complete state") + return entityToDelete, newEntites, newState } // Create maps for quick lookups - oldArtifactsMap := make(map[string]ArtifactReader) - for _, oldArtifact := range oldState.GetArtifacts() { - tag := oldArtifact.GetTags()[0] - oldArtifactsMap[oldArtifact.GetName()+"|"+tag] = oldArtifact + oldEntityMap := make(map[string]Entity) + for _, oldEntity := range oldEntites { + oldEntityMap[oldEntity.Name+"|"+oldEntity.Tag] = oldEntity } // Check new artifacts and update lists - for _, newArtifact := range newState.GetArtifacts() { - nameTagKey := newArtifact.GetName() + "|" + newArtifact.GetTags()[0] - oldArtifact, exists := oldArtifactsMap[nameTagKey] + for _, newEntity := range newEntites { + nameTagKey := newEntity.Name + "|" + newEntity.Tag + oldEntity, exists := oldEntityMap[nameTagKey] if !exists { // New artifact doesn't exist in old state, add to replication list - entityToReplicate = append(entityToReplicate, newArtifact) - } else if newArtifact.GetDigest() != oldArtifact.GetDigest() { + entityToReplicate = append(entityToReplicate, newEntity) + } else if newEntity.Digest != oldEntity.Digest { // Artifact exists but has changed, add to both lists - entityToReplicate = append(entityToReplicate, newArtifact) - entityToDelete = append(entityToDelete, oldArtifact) + entityToReplicate = append(entityToReplicate, newEntity) + entityToDelete = append(entityToDelete, oldEntity) } // Remove processed old artifact from map - delete(oldArtifactsMap, nameTagKey) + delete(oldEntityMap, nameTagKey) } // Remaining artifacts in oldArtifactsMap should be deleted - for _, oldArtifact := range oldArtifactsMap { - entityToDelete = append(entityToDelete, oldArtifact) + for _, oldEntity := range oldEntityMap { + entityToDelete = append(entityToDelete, oldEntity) } return entityToDelete, entityToReplicate, newState @@ -200,17 +200,6 @@ func (f *FetchAndReplicateStateProcess) RemoveNullTagArtifacts(state StateReader return state } -func PrintPrettyJson(info interface{}, log *zerolog.Logger, message string) error { - log.Warn().Msg("Printing pretty JSON") - stateJSON, err := json.MarshalIndent(info, "", " ") - if err != nil { - log.Error().Err(err).Msg("Error marshalling state to JSON") - return err - } - log.Info().Msgf("%s: %s", message, stateJSON) - return nil -} - func ProcessState(state *StateReader) (*StateReader, error) { for _, artifact := range (*state).GetArtifacts() { repo, image, err := utils.GetRepositoryAndImageNameFromArtifact(artifact.GetRepository()) @@ -235,48 +224,22 @@ func (f *FetchAndReplicateStateProcess) FetchAndProcessState(fetcher StateFetche return state, nil } -func (f *FetchAndReplicateStateProcess) LogChanges(deleteEntity, replicateEntity []ArtifactReader, log *zerolog.Logger) { +func (f *FetchAndReplicateStateProcess) LogChanges(deleteEntity, replicateEntity []Entity, log *zerolog.Logger) { log.Warn().Msgf("Total artifacts to delete: %d", len(deleteEntity)) log.Warn().Msgf("Total artifacts to replicate: %d", len(replicateEntity)) } -func processInput(input, username, password string, log *zerolog.Logger) (StateFetcher, error) { - - if utils.IsValidURL(input) { - return processURLInput(utils.FormatRegistryURL(input), username, password, log) - } - - log.Info().Msg("Input is not a valid URL, checking if it is a file path") - if err := validateFilePath(input, log); err != nil { - return nil, err - } - - return processFileInput(input, username, password, log) -} - -func validateFilePath(path string, log *zerolog.Logger) error { - if utils.HasInvalidPathChars(path) { - log.Error().Msg("Path contains invalid characters") - return fmt.Errorf("invalid file path: %s", path) - } - if err := utils.GetAbsFilePath(path); err != nil { - log.Error().Err(err).Msg("No file found") - return fmt.Errorf("no file found: %s", path) +func FetchEntitiesFromState(state StateReader) []Entity { + var entities []Entity + for _, artifact := range state.GetArtifacts() { + for _, tag := range artifact.GetTags() { + entities = append(entities, Entity{ + Name: artifact.GetName(), + Repository: artifact.GetRepository(), + Tag: tag, + Digest: artifact.GetDigest(), + }) + } } - return nil -} - -func processURLInput(input, username, password string, log *zerolog.Logger) (StateFetcher, error) { - log.Info().Msg("Input is a valid URL") - config.SetRemoteRegistryURL(input) - - stateArtifactFetcher := NewURLStateFetcher(input, username, password) - - return stateArtifactFetcher, nil -} - -func processFileInput(input, username, password string, log *zerolog.Logger) (StateFetcher, error) { - log.Info().Msg("Input is a valid file path") - stateArtifactFetcher := NewFileStateFetcher(input, username, password) - return stateArtifactFetcher, nil + return entities }