Adds state artifact fetcher (#55)

* adding state artifact fetcher to fetch the state from harbor

* completing the url fetcher

* adding schedulers and process to satellite

* adding simple notifier to fetch state process

* added description to the scheduler

* coderabbit fixes and changes to fetcher and schedulers

* adding new format of the state file

* adding config to process new state artifact file

* coderabbit review

* added ./zot to gitignore

* fixing the replication process

* fixing the replication and deletion process

* fixing paning while removing the null tags

* using repository name instead of the image name while uploading the image to the zot

* fixes

* moving from toml config to json config

* making config.json work with the replicator

* avoid printing confedential information in log

* coderabbit fixes

* dagger version

* replication fix

.env

@@ -4,3 +4,5 @@ ZOT_URL="127.0.0.1:8585"
 TOKEN=""
 ENV=dev
 USE_UNSECURE=true
+GROUP_NAME=satellite-test-group-state
+STATE_ARTIFACT_NAME=state

.gitignore

@@ -29,3 +29,6 @@ dist/
 zot/cache.db
 secrets.txt
 __debug_bin1949266242
+
+/zot
+/runtime

ci/utils.go

@@ -60,7 +60,6 @@ func (m *HarborSatellite) Service(
 
 
 // builds given component from source
-
 func (m *HarborSatellite) build(source *dagger.Directory, component string) *dagger.Directory {
 	fmt.Printf("Building %s\n", component)
 

config.json

@@ -0,0 +1,16 @@
+{
+  "auth": {
+    "name": "admin",
+    "registry": "https://registry.bupd.xyz",
+    "secret": "Harbor12345"
+  },
+  "bring_own_registry": false,
+  "ground_control_url": "http://localhost:8080",
+  "log_level": "info",
+  "own_registry_adr": "127.0.0.1",
+  "own_registry_port": "8585",
+  "states": ["https://registry.bupd.xyz/satellite-test-group-state/state:latest"],
+  "url_or_file": "https://registry.bupd.xyz",
+  "zotconfigpath": "./registry/config.json",
+  "use_unsecure": true
+}

config.toml

@@ -6,7 +6,10 @@ own_registry_adr = "127.0.0.1"
 own_registry_port = "8585"
 
 # URL of remote registry OR local file path
-url_or_file = "https://demo.goharbor.io/v2/myproject/album-server"
+# url_or_file = "https://demo.goharbor.io/v2/myproject/album-server"
+url_or_file = "https://registry.bupd.xyz"
+## for testing for local file
+# url_or_file = "./image-list/images.json"
 
 # Default path for Zot registry config.json
 zotConfigPath = "./registry/config.json"

go.mod

@@ -328,6 +328,7 @@ require (
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
+	github.com/robfig/cron/v3 v3.0.1
 	github.com/rubenv/sql-migrate v1.5.2 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect
@@ -398,8 +399,8 @@ require (
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0
-	go.opentelemetry.io/otel/log v0.3.0
-	go.opentelemetry.io/otel/metric v1.27.0
+	go.opentelemetry.io/otel/log v0.7.0
+	go.opentelemetry.io/otel/metric v1.27.0 // indirect
 	go.opentelemetry.io/otel/sdk v1.27.0
 	go.opentelemetry.io/otel/sdk/log v0.3.0
 	go.opentelemetry.io/otel/sdk/metric v1.27.0

go.sum

@@ -1289,6 +1289,8 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qq
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
+github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=

image-list/images.json

@@ -1,16 +1,25 @@
 {
-  "registryUrl": "https://demo.goharbor.io/v2/",
-  "repositories": [
+  "registry": "Satellite",
+  "artifacts": [
     {
-      "repository": "myproject",
-      "images": [
-        {
-          "name": "album-server@sha256:39879890008f12c25ea14125aa8e9ec8ef3e167f0b0ed88057e955a8fa32c430"
-        },
-        {
-          "name": "album-server:busybox"
-        }
-      ]
+      "repository": "satellite-test-group-state/alpine",
+      "tag": [
+        "latest"
+      ],
+      "labels": null,
+      "type": "IMAGE",
+      "digest": "sha256:9cee2b382fe2412cd77d5d437d15a93da8de373813621f2e4d406e3df0cf0e7c",
+      "deleted": false
+    },
+    {
+      "repository": "satellite-test-group-state/postgres",
+      "tag": [
+        "latest"
+      ],
+      "labels": null,
+      "type": "IMAGE",
+      "digest": "sha256:dde924f70bc972261013327c480adf402ea71487b5750e40569a0b74fa90c74a",
+      "deleted": false
     }
   ]
-}
+}

internal/config/config.go

@@ -1,164 +1,138 @@
 package config
 
 import (
+	"encoding/json"
 	"fmt"
 	"os"
-
-	"github.com/joho/godotenv"
-	"github.com/spf13/viper"
 )
 
-var AppConfig *Config
+var appConfig *Config
+
+const DefaultConfigPath string = "config.json"
+
+type Auth struct {
+	Name     string `json:"name"`
+	Registry string `json:"registry"`
+	Secret   string `json:"secret"`
+}
 
 type Config struct {
-	log_level         string
-	own_registry      bool
-	own_registry_adr  string
-	own_registry_port string
-	zot_config_path   string
-	input             string
-	zot_url           string
-	registry          string
-	repository        string
-	user_input        string
-	scheme            string
-	api_version       string
-	image             string
-	harbor_password   string
-	harbor_username   string
-	env               string
-	use_unsecure      bool
+	Auth               Auth     `json:"auth"`
+	BringOwnRegistry   bool     `json:"bring_own_registry"`
+	GroundControlURL   string   `json:"ground_control_url"`
+	LogLevel           string   `json:"log_level"`
+	OwnRegistryAddress string   `json:"own_registry_adr"`
+	OwnRegistryPort    string   `json:"own_registry_port"`
+	States             []string `json:"states"`
+	URLOrFile          string   `json:"url_or_file"`
+	ZotConfigPath      string   `json:"zotconfigpath"`
+	UseUnsecure        bool     `json:"use_unsecure"`
+	ZotUrl             string   `json:"zot_url"`
+	StateFetchPeriod   string   `json:"state_fetch_period"`
 }
 
 func GetLogLevel() string {
-	return AppConfig.log_level
+	return appConfig.LogLevel
 }
 
 func GetOwnRegistry() bool {
-	return AppConfig.own_registry
+	return appConfig.BringOwnRegistry
 }
 
 func GetOwnRegistryAdr() string {
-	return AppConfig.own_registry_adr
+	return appConfig.OwnRegistryAddress
 }
 
 func GetOwnRegistryPort() string {
-	return AppConfig.own_registry_port
+	return appConfig.OwnRegistryPort
 }
 
 func GetZotConfigPath() string {
-	return AppConfig.zot_config_path
+	return appConfig.ZotConfigPath
 }
 
 func GetInput() string {
-	return AppConfig.input
+	return appConfig.URLOrFile
 }
 
 func SetZotURL(url string) {
-	AppConfig.zot_url = url
+	appConfig.ZotUrl = url
 }
 
 func GetZotURL() string {
-	return AppConfig.zot_url
+	return appConfig.ZotUrl
 }
 
-func SetRegistry(registry string) {
-	AppConfig.registry = registry
-}
-
-func GetRegistry() string {
-	return AppConfig.registry
-}
-
-func SetRepository(repository string) {
-	AppConfig.repository = repository
-}
-
-func GetRepository() string {
-	return AppConfig.repository
-}
-
-func SetUserInput(user_input string) {
-	AppConfig.user_input = user_input
-}
-
-func GetUserInput() string {
-	return AppConfig.user_input
+func UseUnsecure() bool {
+	return appConfig.UseUnsecure
 }
 
-func SetScheme(scheme string) {
-	AppConfig.scheme = scheme
+func GetHarborPassword() string {
+	return appConfig.Auth.Secret
 }
 
-func GetScheme() string {
-	return AppConfig.scheme
+func GetHarborUsername() string {
+	return appConfig.Auth.Name
 }
 
-func SetAPIVersion(api_version string) {
-	AppConfig.api_version = api_version
+func SetRemoteRegistryURL(url string) {
+	appConfig.Auth.Registry = url
 }
 
-func GetAPIVersion() string {
-	return AppConfig.api_version
+func GetRemoteRegistryURL() string {
+	return appConfig.Auth.Registry
 }
 
-func SetImage(image string) {
-	AppConfig.image = image
+func GetStateFetchPeriod() string {
+	return appConfig.StateFetchPeriod
 }
 
-func GetImage() string {
-	return AppConfig.image
+func GetStates() []string {
+	return appConfig.States
 }
 
-func UseUnsecure() bool {
-	return AppConfig.use_unsecure
+func ParseConfigFromJson(jsonData string) (*Config, error) {
+	var config Config
+	err := json.Unmarshal([]byte(jsonData), &config)
+	if err != nil {
+		return nil, err
+	}
+	return &config, nil
 }
 
-func GetHarborPassword() string {
-	return AppConfig.harbor_password
-}
+func ReadConfigData(configPath string) ([]byte, error) {
 
-func GetHarborUsername() string {
-	return AppConfig.harbor_username
+	fileInfo, err := os.Stat(configPath)
+	if err != nil {
+		return nil, err
+	}
+	if fileInfo.IsDir() {
+		return nil, os.ErrNotExist
+	}
+	data, err := os.ReadFile(configPath)
+	if err != nil {
+		return nil, err
+	}
+	return data, nil
 }
 
 func LoadConfig() (*Config, error) {
-	viper.SetConfigName("config")
-	viper.SetConfigType("toml")
-	viper.AddConfigPath(".")
-	if err := viper.ReadInConfig(); err != nil {
-		return nil, fmt.Errorf("error reading config file at path '%s': %w", viper.ConfigFileUsed(), err)
-	}
-
-	// Load environment and start satellite
-	if err := godotenv.Load(); err != nil {
-		return &Config{}, fmt.Errorf("error loading .env file: %w", err)
+	configData, err := ReadConfigData(DefaultConfigPath)
+	if err != nil {
+		fmt.Printf("Error reading config file: %v\n", err)
+		return nil, err
 	}
-	var use_unsecure bool
-	if os.Getenv("USE_UNSECURE") == "true" {
-		use_unsecure = true
-	} else {
-		use_unsecure = false
+	config, err := ParseConfigFromJson(string(configData))
+	if err != nil {
+		fmt.Printf("Error parsing config file: %v\n", err)
+		return nil, err
 	}
-
-	return &Config{
-		log_level:         viper.GetString("log_level"),
-		own_registry:      viper.GetBool("bring_own_registry"),
-		own_registry_adr:  viper.GetString("own_registry_adr"),
-		own_registry_port: viper.GetString("own_registry_port"),
-		zot_config_path:   viper.GetString("zotConfigPath"),
-		input:             viper.GetString("url_or_file"),
-		harbor_password:   os.Getenv("HARBOR_PASSWORD"),
-		harbor_username:   os.Getenv("HARBOR_USERNAME"),
-		env:               os.Getenv("ENV"),
-		zot_url:           os.Getenv("ZOT_URL"),
-		use_unsecure:      use_unsecure,
-	}, nil
+	return config, nil
 }
 
 func InitConfig() error {
 	var err error
-	AppConfig, err = LoadConfig()
+	appConfig, err = LoadConfig()
 	if err != nil {
 		return err
 	}

internal/notifier/email_notifier.go

@@ -0,0 +1 @@
+package notifier

internal/notifier/notifier.go

@@ -0,0 +1,28 @@
+package notifier
+
+import (
+	"context"
+
+	"container-registry.com/harbor-satellite/logger"
+)
+
+type Notifier interface {
+	// Notify sends a notification
+	Notify() error
+}
+
+type SimpleNotifier struct{
+	ctx context.Context
+}
+
+func NewSimpleNotifier(ctx context.Context) Notifier {
+	return &SimpleNotifier{
+		ctx: ctx,
+	}
+}
+
+func (n *SimpleNotifier) Notify() error {
+	log := logger.FromContext(n.ctx)
+	log.Info().Msg("This is a simple notifier")
+	return nil
+}