Merge branch 'main' into container

.air.toml

@@ -0,0 +1,13 @@
+[build]
+  bin = "main"
+  dir = "."
+  cmd = "go build -o main ."
+  exclude_dir = ["assets", "tmp", "vendor", "testdata"]
+  exclude_regex = ["_test.go"]
+  exclude_unchanged = false
+  follow_symlink = false
+  run_after_build = "chmod +x main"
+  include_ext = ["go"]
+
+[log]
+  level = "debug"

QUICKSTART.md

@@ -0,0 +1,117 @@
+## Quickstart: Local Setup of Harbor Satellite from Source
+
+To set up Harbor Satellite locally, follow these steps:
+
+### 1. Prerequisites
+Ensure you have:
+- A Harbor registry instance (or similar OCI-compliant registry).
+- Credentials with permission to create robot accounts in the registry.
+- The latest version of Dagger installed.
+
+### 2. Set Up the Registry
+In this guide, we'll use a Harbor registry instance.
+
+- **Registry Login**: Obtain the username and password for your registry, ensuring it has appropriate permissions.
+
+### 3. Configure Ground Control
+Navigate to the `ground-control` directory and set up the following environment variables:
+
+```bash
+HARBOR_USERNAME=admin
+HARBOR_PASSWORD=Harbor12345
+HARBOR_URL=https://demo.goharbor.io
+
+PORT=8080
+APP_ENV=local
+
+DB_HOST=pgservice
+DB_PORT=5432
+DB_DATABASE=groundcontrol
+DB_USERNAME=postgres       # Customize based on your DB config
+DB_PASSWORD=password       # Customize based on your DB config
+```
+
+### 4. Run Ground Control
+To start the Ground Control service, execute the following Dagger command:
+
+```bash
+dagger call run-ground-control up
+```
+
+You can also build ground-control binary using the below command
+
+```bash
+dagger call build-dev --platform "linux/amd64" --component "ground-control" export --path=./gc-dev
+```
+
+To Run ground-control binary use
+
+```bash
+./gc-dev
+```
+
+> **Note:** Ensure you have set up Dagger with the latest version before running this command. Ground Control will run on port 8080.
+
+### 5. Configure Satellite
+Return to the root project directory:
+
+```bash
+cd ..
+```
+
+Then navigate to the `satellite` directory and verify that `config.toml` is set up correctly:
+
+```toml
+# Whether to use the built-in Zot registry or not
+bring_own_registry = false
+
+# IP address and port of the registry
+own_registry_adr = "127.0.0.1"
+own_registry_port = "8585"
+
+# URL of remote registry or local file path
+url_or_file = "https://demo.goharbor.io/v2/myproject/album-server"
+
+
+
+# Default path for Zot registry config.json
+zotConfigPath = "./registry/config.json"
+
+# Set logging level
+log_level = "info"
+```
+
+### 6. Register the Satellite with Ground Control
+Using `curl` or Postman, make a `POST` request to register the Satellite with Ground Control:
+
+```bash
+curl -X POST http://localhost:8080/satellites/register -H "Content-Type: application/json" -d '{ "name": "<satellite_name_here>" }'
+```
+
+The response will include a token string. Set this token in the Satellite `.env` file:
+
+```console
+TOKEN=<string_from_ground_control>
+```
+
+### 7. Build the Satellite
+Run the following Dagger command to build the Satellite:
+
+```bash
+dagger call build-dev --platform "linux/amd64" --component satellite export --path=./satellite-dev
+```
+
+To Run Satellite:
+```bash
+./satellite-dev
+```
+
+The Satellite service will start on port 9090. Ensure that the `ground_control_url` is correctly set in the Satellite configuration before launching.
+
+
+### 8. Finalize and Run
+After setting the token, you can now run the Satellite. This setup will launch the Satellite in a container with the following exposed ports:
+- **9090** for the Satellite service.
+- **8585** for the Zot registry (if configured).
+
+With this setup, your Harbor Satellite should be up and running!

README.md

@@ -14,6 +14,9 @@ Harbor Satellite will synchronize with the central Harbor registry, when Interne
 
 Harbor Satellite will also include a toolset enabling the monitoring and management of local decentralized registries.
 
+## QuickStart
+Please refer to the latest setup instructions in QUICKSTART.md file for detailed steps on setting up Harbor Satellite locally.
+
 ## Non-Goals
 
 T.B.D.
@@ -51,29 +54,29 @@ Harbor Satellite, at its most basic, will run in a single container and will be
 
 Harbor Satellite may be implemented following 1 or several of 3 different architectures depending on its use cases :
 
-1. **Replicating from a remote registry to a local registry.**  
+1. **Replicating from a remote registry to a local registry.**
 In this basic use case, the stateless satellite component will handle pulling images from a remote registry and then pushing them to the local OCI compliant registry. This local registry will then be accessible to other local edge devices who can pull required images directly from it.
-_(A direct access from edge device to the remote registry is still possible when network conditions permit it)._  
-The satellite component may also handle updating container runtime configurations and fetching image lists from Ground Control, a part of Harbor.  
-The stateful local regsitry will also need to handle storing and managing data on local volumes.  
-A typical use case would work as follows :  
+_(A direct access from edge device to the remote registry is still possible when network conditions permit it)._
+The satellite component may also handle updating container runtime configurations and fetching image lists from Ground Control, a part of Harbor.
+The stateful local regsitry will also need to handle storing and managing data on local volumes.
+A typical use case would work as follows :
 _In an edge computing environment where IoT devices are deployed to a location with limited or no internet connnectivity, these devices need to run containerised images but cannot pull from a central Harbor registry. A local Harbor Satellite instance can be deployed and take up this role while Internet connectivity is unreliable and distribute all required images. Once a reliable connection is re-established, the Harbor Satellite instance will be able to pull required images from its central Harbor registry and thus store up to date images locally._
 
 ![Use Case #1](docs/images/satellite_use_case_1.svg)
 <p align="center"><em>Use case #1</em></p>
 
-2. **Replicating from a remote regsitry to a local Spegel Registry**  
-The stateless satellite component send pull instructions to Spegel instances running with each node of a Kubernetes cluster. The node will then directly pull images from a remote registry and share it with other local nodes, removing the need for each of them to individually pull an image from a remote registry.  
-The network interfaces (boundaries) represented in this use case should and will be the same as those represented in use case #1  
-A typical use case would work as follows :  
+2. **Replicating from a remote regsitry to a local Spegel Registry**
+The stateless satellite component send pull instructions to Spegel instances running with each node of a Kubernetes cluster. The node will then directly pull images from a remote registry and share it with other local nodes, removing the need for each of them to individually pull an image from a remote registry.
+The network interfaces (boundaries) represented in this use case should and will be the same as those represented in use case #1
+A typical use case would work as follows :
 _In a larger scale edge computing environment with a significant amount of IoT devices needing to run containerised applications, a single local registry in might not be able to handle the increased amount of demands from edge devices. The solution is to deploy several registries to several nodes who are able to automatically replicate images across each other thanks to Spegel instances running together with each node. The Satellite component will use the same interface to instruct each node when, where and how to pull new images that need to be replicated across the cluster._
 
 ![Use Case #2](docs/images/satellite_use_case_2.svg)
 <p align="center"><em>Use case #2</em></p>
 
-3. **Proxying from a remote regsitry over the local registry**  
-The stateless satellite component will be in charge of configuring the local OCI compliant registry, which will be running in proxy mode only. This local registry will then handle pulling necessary images from the remote registry and serving them up for use by local edge devices.  
-A typical use case would work as follows :  
+3. **Proxying from a remote regsitry over the local registry**
+The stateless satellite component will be in charge of configuring the local OCI compliant registry, which will be running in proxy mode only. This local registry will then handle pulling necessary images from the remote registry and serving them up for use by local edge devices.
+A typical use case would work as follows :
 _When, for a number of possible different reasons, the remote registry side of the diagram would not be able to produce a list of images to push down to the Harbor Satellite, the Satellite would then act as a proxy and forward all requests from edge devices to the remote registry. This ensures the availability of necessary images without the need for a pre-compiled list of images_
 
 ![Use Case #3](docs/images/satellite_use_case_3.svg)

ci/main.go

@@ -3,8 +3,10 @@ package main
 import (
 	"context"
 	"fmt"
+	"log"
 	"log/slog"
 	"os"
+	"strings"
 
 	"container-registry.com/harbor-satellite/ci/internal/dagger"
 )
@@ -20,6 +22,107 @@ const (
 
 type HarborSatellite struct{}
 
+// start the dev server for ground-control.
+func (m *HarborSatellite) RunGroundControl(
+	ctx context.Context,
+	// +optional
+	// +defaultPath="."
+	source *dagger.Directory,
+) (*dagger.Service, error) {
+	golang := dag.Container().
+		From("golang:latest").
+		WithMountedCache("/go/pkg/mod", dag.CacheVolume("go-mod")).
+		WithEnvVariable("GOMODCACHE", "/go/pkg/mod").
+		WithMountedCache("/go/build-cache", dag.CacheVolume("go-build")).
+		WithEnvVariable("GOCACHE", "/go/build-cache").
+		WithMountedDirectory(PROJ_MOUNT, source).
+		WithWorkdir(PROJ_MOUNT).
+		WithExec([]string{"go", "install", "github.com/air-verse/air@latest"})
+
+	db, err := m.Db(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	golang = golang.
+		WithWorkdir(PROJ_MOUNT+"/ground-control/sql/schema").
+		WithExec([]string{"ls", "-la"}).
+		WithServiceBinding("pgservice", db).
+		WithExec([]string{"go", "install", "github.com/pressly/goose/v3/cmd/goose@latest"}).
+		WithExec([]string{"goose", "postgres", "postgres://postgres:password@pgservice:5432/groundcontrol", "up"}).
+		WithWorkdir(PROJ_MOUNT + "/ground-control").
+		WithExec([]string{"ls", "-la"}).
+		WithExec([]string{"go", "mod", "download"}).
+		WithExec([]string{"air", "-c", ".air.toml"}).
+		WithExposedPort(8080)
+
+	return golang.AsService(), nil
+}
+
+// quickly build binaries for components for given platform.
+func (m *HarborSatellite) BuildDev(
+	ctx context.Context,
+	// +optional
+	// +defaultPath="."
+	source *dagger.Directory,
+	platform string,
+	component string,
+) (*dagger.File, error) {
+	fmt.Println("🛠️  Building Harbor-Cli with Dagger...")
+	// Define the path for the binary output
+	os, arch, err := parsePlatform(platform)
+	if err != nil {
+		log.Fatalf("Error parsing platform: %v", err)
+	}
+
+	if component == "satellite" || component == "ground-control" {
+		var binaryFile *dagger.File
+		golang := dag.Container().
+			From("golang:latest").
+			WithMountedCache("/go/pkg/mod", dag.CacheVolume("go-mod")).
+			WithEnvVariable("GOMODCACHE", "/go/pkg/mod").
+			WithMountedCache("/go/build-cache", dag.CacheVolume("go-build")).
+			WithEnvVariable("GOCACHE", "/go/build-cache").
+			WithMountedDirectory(PROJ_MOUNT, source).
+			WithWorkdir(PROJ_MOUNT).
+			WithEnvVariable("GOOS", os).
+			WithEnvVariable("GOARCH", arch)
+
+		if component == "ground-control" {
+			golang = golang.
+				WithWorkdir(PROJ_MOUNT + "/ground-control").
+				WithExec([]string{"ls", "-la"}).
+				WithExec([]string{"go", "mod", "download"}).
+				WithExec([]string{"go", "build", "."})
+
+			binaryFile = golang.File(PROJ_MOUNT + "/ground-control/ground-control")
+		} else {
+			golang = golang.
+				WithExec([]string{"ls", "-la"}).
+				WithExec([]string{"go", "mod", "download"}).
+				WithExec([]string{"go", "build", "."})
+
+			binaryFile = golang.File(PROJ_MOUNT + "/harbor-satellite")
+		}
+
+		return binaryFile, nil
+	}
+
+	return nil, fmt.Errorf("error: please provide component as either satellite or ground-control")
+}
+
+// starts postgres DB container for ground-control.
+func (m *HarborSatellite) Db(ctx context.Context) (*dagger.Service, error) {
+	return dag.Container().
+		From("postgres:17").
+		WithEnvVariable("POSTGRES_USER", "postgres").
+		WithEnvVariable("POSTGRES_PASSWORD", "password").
+		WithEnvVariable("POSTGRES_HOST_AUTH_METHOD", "trust").
+		WithEnvVariable("POSTGRES_DB", "groundcontrol").
+		WithExposedPort(5432).
+		AsService().Start(ctx)
+}
+
 // Build function would start the build process for the name provided. Source should be the path to the main.go file.
 func (m *HarborSatellite) Build(
 	ctx context.Context,
@@ -78,3 +181,12 @@ func (m *HarborSatellite) Release(ctx context.Context, directory *dagger.Directo
 
 	return release_output, nil
 }
+
+// Parse the platform string into os and arch
+func parsePlatform(platform string) (string, string, error) {
+	parts := strings.Split(platform, "/")
+	if len(parts) != 2 {
+		return "", "", fmt.Errorf("invalid platform format: %s. Should be os/arch. E.g. darwin/amd64", platform)
+	}
+	return parts[0], parts[1], nil
+}

dagger.json

@@ -1,6 +1,6 @@
 {
   "name": "harbor-satellite",
+  "engineVersion": "v0.14.0",
   "sdk": "go",
-  "source": "ci",
-  "engineVersion": "v0.13.3"
+  "source": "ci"
 }

go.mod

@@ -1,8 +1,8 @@
 module container-registry.com/harbor-satellite
 
-go 1.22.3
+go 1.22.5
 
-toolchain go1.22.4
+toolchain go1.23.2
 
 require (
 	dagger.io/dagger v0.10.3
@@ -33,7 +33,7 @@ require (
 	cloud.google.com/go/iam v1.1.6 // indirect
 	cloud.google.com/go/storage v1.39.1 // indirect
 	dario.cat/mergo v1.0.0 // indirect
-	github.com/99designs/gqlgen v0.17.49
+	github.com/99designs/gqlgen v0.17.55
 	github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect
 	github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0 // indirect
 	github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect
@@ -69,7 +69,7 @@ require (
 	github.com/ThalesIgnite/crypto11 v1.2.5 // indirect
 	github.com/VividCortex/ewma v1.2.0 // indirect
 	github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
-	github.com/adrg/xdg v0.5.0 // indirect
+	github.com/adrg/xdg v0.5.1 // indirect
 	github.com/agext/levenshtein v1.2.3 // indirect
 	github.com/agnivade/levenshtein v1.1.1 // indirect
 	github.com/alecthomas/chroma v0.10.0 // indirect
@@ -379,7 +379,7 @@ require (
 	github.com/ulikunitz/xz v0.5.12 // indirect
 	github.com/vbatts/tar-split v0.11.5 // indirect
 	github.com/vbauerster/mpb/v8 v8.7.3 // indirect
-	github.com/vektah/gqlparser/v2 v2.5.16
+	github.com/vektah/gqlparser/v2 v2.5.17
 	github.com/xanzy/go-gitlab v0.102.0 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
@@ -399,27 +399,30 @@ require (
 	go.opentelemetry.io/otel v1.27.0
 	go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.0.0-20240518090000-14441aefdf88
 	go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.3.0
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.27.0
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.27.0
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0
 	go.opentelemetry.io/otel/log v0.7.0
 	go.opentelemetry.io/otel/metric v1.27.0 // indirect
 	go.opentelemetry.io/otel/sdk v1.27.0
 	go.opentelemetry.io/otel/sdk/log v0.3.0
+	go.opentelemetry.io/otel/sdk/metric v1.27.0
 	go.opentelemetry.io/otel/trace v1.27.0
 	go.opentelemetry.io/proto/otlp v1.3.1
 	go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
 	golang.org/x/crypto v0.27.0 // indirect
 	golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842
-	golang.org/x/mod v0.18.0 // indirect
+	golang.org/x/mod v0.20.0 // indirect
 	golang.org/x/net v0.29.0 // indirect
-	golang.org/x/sys v0.25.0 // indirect
+	golang.org/x/sys v0.26.0 // indirect
 	golang.org/x/term v0.24.0 // indirect
 	golang.org/x/text v0.18.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
-	golang.org/x/tools v0.22.0 // indirect
+	golang.org/x/tools v0.24.0 // indirect
 	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
 	google.golang.org/api v0.172.0 // indirect
 	google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7 // indirect