From fe4de0fec99f879cd6a11fe4397bc5e14163fcf0 Mon Sep 17 00:00:00 2001 From: Roald Brunell <87445607+OneFlyingBanana@users.noreply.github.com> Date: Tue, 2 Jul 2024 15:35:46 +0200 Subject: [PATCH] Changed logging to use zerolog zerolog logs any non-error information to stdout and any errors to stderr. Zot registry logs to stdout by default --- .vscode/launch.json | 15 ++++++ config.toml | 10 ++++ go.mod | 36 +++++++------ go.sum | 70 +++++++++++++------------ internal/replicate/replicate.go | 63 +++++++++++++--------- internal/satellite/satellite.go | 19 ++++--- internal/store/file-fetch.go | 13 +++-- internal/store/http-fetch.go | 33 ++++++++---- internal/store/in-memory-store.go | 57 ++++++++++++-------- logger/logger.go | 67 ++++++++++++++++++++++++ main.go | 87 +++++++++++++++++++------------ registry/config.json | 5 +- registry/launch-registry.go | 8 +-- 13 files changed, 323 insertions(+), 160 deletions(-) create mode 100644 .vscode/launch.json create mode 100644 logger/logger.go diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..780ab3d --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,15 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Launch Package", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}/main.go" + } + ] +} diff --git a/config.toml b/config.toml index 93b9ebd..3a2eebd 100644 --- a/config.toml +++ b/config.toml @@ -1,10 +1,20 @@ # Wether to us the built-in Zot registry or not bring_own_registry = true +# IP address and port of own registry +own_registry_adr = "127.0.0.1" +own_registry_port = "8585" # URL of own registry own_registry_adr = "127.0.0.1:5000" # URL of remote registry OR local file path +url_or_file = "https://demo.goharbor.io/v2/myproject/album-server" + +# Default path for Zot registry config.json +zotConfigPath = "./registry/config.json" + +# Set logging level +log_level = "warn" # url_or_file = "https://demo.goharbor.io/v2/myproject/album-server" url_or_file = "http://localhost:5001/v2/library/busybox" diff --git a/go.mod b/go.mod index cc348f8..6e7ff47 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,12 @@ require ( ) -require github.com/spf13/viper v1.19.0 +require ( + github.com/rs/zerolog v1.33.0 + github.com/spf13/viper v1.19.0 + go.uber.org/zap v1.27.0 + gopkg.in/natefinch/lumberjack.v2 v2.2.1 +) require ( cloud.google.com/go v0.112.1 // indirect @@ -51,7 +56,7 @@ require ( github.com/Masterminds/sprig/v3 v3.2.3 // indirect github.com/Masterminds/squirrel v1.5.4 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect - github.com/Microsoft/hcsshim v0.12.3 // indirect + github.com/Microsoft/hcsshim v0.12.4 // indirect github.com/OneOfOne/xxhash v1.2.8 // indirect github.com/ProtonMail/go-crypto v1.1.0-alpha.2 // indirect github.com/ThalesIgnite/crypto11 v1.2.5 // indirect @@ -117,7 +122,7 @@ require ( github.com/cheggaaa/pb/v3 v3.1.5 // indirect github.com/cloudflare/circl v1.3.7 // indirect github.com/containerd/cgroups/v3 v3.0.3 // indirect - github.com/containerd/containerd v1.7.17 // indirect + github.com/containerd/containerd v1.7.18 // indirect github.com/containerd/continuity v0.4.3 // indirect github.com/containerd/errdefs v0.1.0 // indirect github.com/containerd/fifo v1.1.0 // indirect @@ -143,7 +148,7 @@ require ( github.com/dlclark/regexp2 v1.4.0 // indirect github.com/docker/cli v26.1.3+incompatible // indirect github.com/docker/distribution v2.8.3+incompatible // indirect - github.com/docker/docker v26.1.3+incompatible // indirect + github.com/docker/docker v27.0.0+incompatible // indirect github.com/docker/docker-credential-helpers v0.8.1 // indirect github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect @@ -167,7 +172,7 @@ require ( github.com/go-ini/ini v1.67.0 // indirect github.com/go-jose/go-jose/v3 v3.0.3 // indirect github.com/go-ldap/ldap/v3 v3.4.8 // indirect - github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/analysis v0.23.0 // indirect github.com/go-openapi/errors v0.22.0 // indirect @@ -234,7 +239,7 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect - github.com/klauspost/compress v1.17.8 // indirect + github.com/klauspost/compress v1.17.9 // indirect github.com/klauspost/pgzip v1.2.6 // indirect github.com/knqyf263/go-apk-version v0.0.0-20200609155635-041fdbb8563f // indirect github.com/knqyf263/go-deb-version v0.0.0-20230223133812-3ed183d23422 // indirect @@ -250,6 +255,7 @@ require ( github.com/liamg/memoryfs v1.6.0 // indirect github.com/lib/pq v1.10.9 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect + github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae // indirect github.com/lunixbochs/struc v0.0.0-20200707160740-784aaebc1d40 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect @@ -318,7 +324,6 @@ require ( github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.4.7 // indirect - github.com/rs/zerolog v1.32.0 // indirect github.com/rubenv/sql-migrate v1.5.2 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect @@ -329,6 +334,7 @@ require ( github.com/secure-systems-lab/go-securesystemslib v0.8.0 // indirect github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/shibumi/go-pathspec v1.3.0 // indirect + github.com/shirou/gopsutil/v3 v3.24.5 // indirect github.com/shopspring/decimal v1.3.1 // indirect github.com/sigstore/cosign/v2 v2.2.4 // indirect github.com/sigstore/fulcio v1.4.5 // indirect @@ -358,6 +364,7 @@ require ( github.com/thales-e-security/pool v0.0.2 // indirect github.com/theupdateframework/go-tuf v0.7.0 // indirect github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399 // indirect + github.com/tklauser/go-sysconf v0.3.14 // indirect github.com/transparency-dev/merkle v0.0.2 // indirect github.com/twitchtv/twirp v8.1.3+incompatible // indirect github.com/ulikunitz/xz v0.5.12 // indirect @@ -386,24 +393,23 @@ require ( go.opentelemetry.io/otel/trace v1.27.0 // indirect go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.27.0 // indirect - golang.org/x/crypto v0.23.0 // indirect + golang.org/x/crypto v0.24.0 // indirect golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect golang.org/x/mod v0.17.0 // indirect golang.org/x/net v0.25.0 // indirect golang.org/x/oauth2 v0.20.0 // indirect - golang.org/x/sys v0.20.0 // indirect - golang.org/x/term v0.20.0 // indirect - golang.org/x/text v0.15.0 // indirect + golang.org/x/sys v0.21.0 // indirect + golang.org/x/term v0.21.0 // indirect + golang.org/x/text v0.16.0 // indirect golang.org/x/time v0.5.0 // indirect - golang.org/x/tools v0.21.0 // indirect + golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect google.golang.org/api v0.172.0 // indirect google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240617180043-68d350f18fd4 // indirect google.golang.org/grpc v1.64.0 // indirect - google.golang.org/protobuf v1.34.1 // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect gopkg.in/go-jose/go-jose.v2 v2.6.3 // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/go.sum b/go.sum index 3b0692f..848188c 100644 --- a/go.sum +++ b/go.sum @@ -277,8 +277,8 @@ github.com/Masterminds/squirrel v1.5.4/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA4 github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= -github.com/Microsoft/hcsshim v0.12.3 h1:LS9NXqXhMoqNCplK1ApmVSfB4UnVLRDWRapB6EIlxE0= -github.com/Microsoft/hcsshim v0.12.3/go.mod h1:Iyl1WVpZzr+UkzjekHZbV8o5Z9ZkxNGx6CtY2Qg/JVQ= +github.com/Microsoft/hcsshim v0.12.4 h1:Ev7YUMHAHoWNm+aDSPzc5W9s6E2jyL1szpVDJeZ/Rr4= +github.com/Microsoft/hcsshim v0.12.4/go.mod h1:Iyl1WVpZzr+UkzjekHZbV8o5Z9ZkxNGx6CtY2Qg/JVQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8= github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= @@ -517,8 +517,8 @@ github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be h1:J5BL github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be/go.mod h1:mk5IQ+Y0ZeO87b858TlA645sVcEcbiX6YqP98kt+7+w= github.com/containerd/cgroups/v3 v3.0.3 h1:S5ByHZ/h9PMe5IOQoN7E+nMc2UcLEM/V48DGDJ9kip0= github.com/containerd/cgroups/v3 v3.0.3/go.mod h1:8HBe7V3aWGLFPd/k03swSIsGjZhHI2WzJmticMgVuz0= -github.com/containerd/containerd v1.7.17 h1:KjNnn0+tAVQHAoaWRjmdak9WlvnFR/8rU1CHHy8Rm2A= -github.com/containerd/containerd v1.7.17/go.mod h1:vK+hhT4TIv2uejlcDlbVIc8+h/BqtKLIyNrtCZol8lI= +github.com/containerd/containerd v1.7.18 h1:jqjZTQNfXGoEaZdW1WwPU0RqSn1Bm2Ay/KJPUuO8nao= +github.com/containerd/containerd v1.7.18/go.mod h1:IYEk9/IO6wAPUz2bCMVUbsfXjzw5UNP5fLz4PsUygQ4= github.com/containerd/continuity v0.4.3 h1:6HVkalIp+2u1ZLH1J/pYX2oBVXlJZvh1X1A7bEZ9Su8= github.com/containerd/continuity v0.4.3/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ= github.com/containerd/errdefs v0.1.0 h1:m0wCRBiu1WJT/Fr+iOoQHMQS/eP5myQ8lCv4Dz5ZURM= @@ -596,8 +596,8 @@ github.com/docker/cli v26.1.3+incompatible h1:bUpXT/N0kDE3VUHI2r5VMsYQgi38kYuoC0 github.com/docker/cli v26.1.3+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v26.1.3+incompatible h1:lLCzRbrVZrljpVNobJu1J2FHk8V0s4BawoZippkc+xo= -github.com/docker/docker v26.1.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v27.0.0+incompatible h1:JRugTYuelmWlW0M3jakcIadDx2HUoUO6+Tf2C5jVfwA= +github.com/docker/docker v27.0.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker-credential-helpers v0.8.1 h1:j/eKUktUltBtMzKqmfLB0PAgqYyMHOp5vfsD1807oKo= github.com/docker/docker-credential-helpers v0.8.1/go.mod h1:P3ci7E3lwkZg6XiHdRKft1KckHiO9a2rNtyFbZ/ry9M= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= @@ -688,8 +688,8 @@ github.com/go-ldap/ldap/v3 v3.4.8/go.mod h1:qS3Sjlu76eHfHGpUdWkAXQTw4beih+cHsco2 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= @@ -1025,8 +1025,8 @@ github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= -github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= -github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/knqyf263/go-apk-version v0.0.0-20200609155635-041fdbb8563f h1:GvCU5GXhHq+7LeOzx/haG7HSIZokl3/0GkoUFzsRJjg= @@ -1067,8 +1067,8 @@ github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= -github.com/lufia/plan9stats v0.0.0-20240226150601-1dcf7310316a h1:3Bm7EwfUQUvhNeKIkUct/gl9eod1TcXuj8stxvi/GoI= -github.com/lufia/plan9stats v0.0.0-20240226150601-1dcf7310316a/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k= +github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae h1:dIZY4ULFcto4tAFlj1FYZl8ztUZ13bdq+PLY+NOfbyI= +github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k= github.com/lunixbochs/struc v0.0.0-20200707160740-784aaebc1d40 h1:EnfXoSqDfSNJv0VBNqY/88RNnhSGYkrHaO0mmFGbVsc= github.com/lunixbochs/struc v0.0.0-20200707160740-784aaebc1d40/go.mod h1:vy1vK6wD6j7xX6O6hXe621WabdtNkou2h7uRtTfRMyg= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= @@ -1296,8 +1296,8 @@ github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99 github.com/rs/cors v1.10.1 h1:L0uuZVXIKlI1SShY2nhFfo44TYvDPQ1w4oFkUJNfhyo= github.com/rs/cors v1.10.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/rs/zerolog v1.32.0 h1:keLypqrlIjaFsbmJOBdB/qvyF8KEtCWHwobLp5l/mQ0= -github.com/rs/zerolog v1.32.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= +github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8= +github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/rubenv/sql-migrate v1.5.2 h1:bMDqOnrJVV/6JQgQ/MxOpU+AdO8uzYYA/TxFUBzFtS0= github.com/rubenv/sql-migrate v1.5.2/go.mod h1:H38GW8Vqf8F0Su5XignRyaRcbXbJunSWxs+kmzlg0Is= github.com/russross/blackfriday v1.6.0 h1:KqfZb0pUVN2lYqZUYRddxF4OR8ZMURnJIG5Y3VRLtww= @@ -1328,8 +1328,8 @@ github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/shibumi/go-pathspec v1.3.0 h1:QUyMZhFo0Md5B8zV8x2tesohbb5kfbpTi9rBnKh5dkI= github.com/shibumi/go-pathspec v1.3.0/go.mod h1:Xutfslp817l2I1cZvgcfeMQJG5QnU2lh5tVaaMCl3jE= -github.com/shirou/gopsutil/v3 v3.24.2 h1:kcR0erMbLg5/3LcInpw0X/rrPSqq4CDPyI6A6ZRC18Y= -github.com/shirou/gopsutil/v3 v3.24.2/go.mod h1:tSg/594BcA+8UdQU2XcW803GWYgdtauFFPgJCJKZlVk= +github.com/shirou/gopsutil/v3 v3.24.5 h1:i0t8kL+kQTvpAYToeuiVk3TgDeKOFioZO3Ztz/iZ9pI= +github.com/shirou/gopsutil/v3 v3.24.5/go.mod h1:bsoOS1aStSs9ErQ1WWfxllSeS1K5D+U30r2NfcubMVk= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= @@ -1441,10 +1441,10 @@ github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399 h1:e/5i7d4oYZ+C github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399/go.mod h1:LdwHTNJT99C5fTAzDz0ud328OgXz+gierycbcIx2fRs= github.com/tjfoc/gmsm v1.4.1 h1:aMe1GlZb+0bLjn+cKTPEvvn9oUEBlJitaZiiBwsbgho= github.com/tjfoc/gmsm v1.4.1/go.mod h1:j4INPkHWMrhJb38G+J6W4Tw0AbuN8Thu3PbdVYhVcTE= -github.com/tklauser/go-sysconf v0.3.13 h1:GBUpcahXSpR2xN01jhkNAbTLRk2Yzgggk8IM08lq3r4= -github.com/tklauser/go-sysconf v0.3.13/go.mod h1:zwleP4Q4OehZHGn4CYZDipCgg9usW5IJePewFCGVEa0= -github.com/tklauser/numcpus v0.7.0 h1:yjuerZP127QG9m5Zh/mSO4wqurYil27tHrqwRoRjpr4= -github.com/tklauser/numcpus v0.7.0/go.mod h1:bb6dMVcj8A42tSE7i32fsIUCbQNllK5iDguyOZRUzAY= +github.com/tklauser/go-sysconf v0.3.14 h1:g5vzr9iPFFz24v2KZXs/pvpvh8/V9Fw6vQK5ZZb78yU= +github.com/tklauser/go-sysconf v0.3.14/go.mod h1:1ym4lWMLUOhuBOPGtRcJm7tEGX4SCYNEEEtghGG/8uY= +github.com/tklauser/numcpus v0.8.0 h1:Mx4Wwe/FjZLeQsK/6kt2EOepwwSl7SmJrK5bV/dXYgY= +github.com/tklauser/numcpus v0.8.0/go.mod h1:ZJZlAY+dmR4eut8epnzf0u/VwodKmryxR8txiloSqBE= github.com/transparency-dev/merkle v0.0.2 h1:Q9nBoQcZcgPamMkGn7ghV8XiTZ/kRxn1yCG81+twTK4= github.com/transparency-dev/merkle v0.0.2/go.mod h1:pqSy+OXefQ1EDUVmAJ8MUhHB9TXGuzVAT58PqBoHz1A= github.com/twitchtv/twirp v8.1.3+incompatible h1:+F4TdErPgSUbMZMwp13Q/KgDVuI7HJXP61mNV3/7iuU= @@ -1574,8 +1574,8 @@ golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72 golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= -golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= -golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1817,8 +1817,8 @@ golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1832,8 +1832,8 @@ golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww= golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= -golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= -golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= +golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1848,8 +1848,8 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1914,8 +1914,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= -golang.org/x/tools v0.21.0 h1:qc0xYgIbsSDt9EyWz05J5wfa7LOVW0YTLOXrqdLAWIw= -golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -2088,8 +2088,8 @@ google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7 h1:ImUcDPHjTrAqNhl google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7/go.mod h1:/3XmxOjePkvmKrHuBy4zNFw7IzxJXtAgdpXi8Ll990U= google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5 h1:P8OJ/WCl/Xo4E4zoe4/bifHpSmmKwARqyqE4nW6J2GQ= google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5/go.mod h1:RGnPtTG7r4i8sPlNyDeikXF99hMM+hN6QMm4ooG9g2g= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 h1:AgADTJarZTBqgjiUzRgfaBchgYB3/WFTC80GPwsMcRI= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240617180043-68d350f18fd4 h1:Di6ANFilr+S60a4S61ZM00vLdw0IrQOSMS2/6mrnOU0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240617180043-68d350f18fd4/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -2143,8 +2143,8 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= -google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -2162,6 +2162,8 @@ gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/resty.v1 v1.12.0 h1:CuXP0Pjfw9rOuY6EP+UvtNvt5DSqHpIxILZKT/quCZI= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI= diff --git a/internal/replicate/replicate.go b/internal/replicate/replicate.go index aa73454..783a28a 100644 --- a/internal/replicate/replicate.go +++ b/internal/replicate/replicate.go @@ -9,6 +9,7 @@ import ( "strings" "container-registry.com/harbor-satellite/internal/store" + "container-registry.com/harbor-satellite/logger" "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/crane" ) @@ -35,15 +36,16 @@ type RegistryInfo struct { Repositories []Repository `json:"repositories"` } -func NewReplicator() Replicator { +func NewReplicator(context context.Context) Replicator { return &BasicReplicator{} } func (r *BasicReplicator) Replicate(ctx context.Context, image string) error { - source := getPullSource(image) + + source := getPullSource(ctx, image) if source != "" { - CopyImage(source) + CopyImage(ctx, source) } return nil } @@ -56,18 +58,21 @@ func stripPrefix(imageName string) string { } func (r *BasicReplicator) DeleteExtraImages(ctx context.Context, imgs []store.Image) error { + log := logger.FromContext(ctx) + errLog := logger.ErrorLoggerFromContext(ctx) zotUrl := os.Getenv("ZOT_URL") host := os.Getenv("HOST") registry := os.Getenv("REGISTRY") repository := os.Getenv("REPOSITORY") localRegistry := fmt.Sprintf("%s/%s/%s/%s", zotUrl, host, registry, repository) - fmt.Println("Syncing local registry:", localRegistry) + log.Info().Msgf("Local registry: %s", localRegistry) // Get the list of images from the local registry localImages, err := crane.ListTags(localRegistry) if err != nil { - return fmt.Errorf("failed to get local registry catalog: %w", err) + errLog.Error().Msgf("failed to list tags: %v", err) + return err } // Create a map for quick lookup of the provided image list @@ -82,29 +87,31 @@ func (r *BasicReplicator) DeleteExtraImages(ctx context.Context, imgs []store.Im for _, localImage := range localImages { if _, exists := imageMap[localImage]; !exists { // Image is not in the provided list, delete it - fmt.Print("Deleting image: ", localRegistry+":"+localImage, " ... ") + log.Info().Msgf("Deleting image: %s", localImage) err := crane.Delete(fmt.Sprintf("%s:%s", localRegistry, localImage)) if err != nil { - fmt.Printf("failed to delete image %s: %v\n", localImage, err) - return nil + errLog.Error().Msgf("failed to delete image: %v", err) + return err } - fmt.Printf("Deleted image: %s\n", localImage) + log.Info().Msgf("Image deleted: %s", localImage) } } return nil } -func getPullSource(image string) string { +func getPullSource(ctx context.Context, image string) string { + errLog := logger.ErrorLoggerFromContext(ctx) input := os.Getenv("USER_INPUT") scheme := os.Getenv("SCHEME") if strings.HasPrefix(scheme, "http://") || strings.HasPrefix(scheme, "https://") { url := os.Getenv("HOST") + "/" + os.Getenv("REGISTRY") + "/" + image return url } else { - registryInfo, err := getFileInfo(input) + registryInfo, err := getFileInfo(ctx, input) if err != nil { - return "Error loading file info: " + err.Error() + errLog.Error().Msgf("Error getting file info: %v", err) + return "" } registryURL := registryInfo.RegistryUrl registryURL = strings.TrimPrefix(registryURL, "https://") @@ -117,11 +124,13 @@ func getPullSource(image string) string { } } -func getFileInfo(input string) (*RegistryInfo, error) { +func getFileInfo(ctx context.Context, input string) (*RegistryInfo, error) { + errLog := logger.ErrorLoggerFromContext(ctx) // Get the current working directory workingDir, err := os.Getwd() if err != nil { - return nil, fmt.Errorf("failed to get working directory: %w", err) + errLog.Error().Msgf("Error getting current directory: %v", err) + return nil, err } // Construct the full path by joining the working directory and the input path @@ -130,22 +139,27 @@ func getFileInfo(input string) (*RegistryInfo, error) { // Read the file jsonData, err := os.ReadFile(fullPath) if err != nil { - return nil, fmt.Errorf("failed to read file: %w", err) + errLog.Error().Msgf("Error reading file: %v", err) + return nil, err } var registryInfo RegistryInfo err = json.Unmarshal(jsonData, ®istryInfo) if err != nil { - return nil, fmt.Errorf("failed to unmarshal JSON: %w", err) + errLog.Error().Msgf("Error unmarshalling JSON data: %v", err) + return nil, err } return ®istryInfo, nil } -func CopyImage(imageName string) error { - fmt.Println("Copying image:", imageName) +func CopyImage(ctx context.Context, imageName string) error { + log := logger.FromContext(ctx) + errLog := logger.ErrorLoggerFromContext(ctx) + log.Info().Msgf("Copying image: %s", imageName) zotUrl := os.Getenv("ZOT_URL") if zotUrl == "" { + errLog.Error().Msg("ZOT_URL environment variable is not set") return fmt.Errorf("ZOT_URL environment variable is not set") } @@ -158,6 +172,7 @@ func CopyImage(imageName string) error { username := os.Getenv("HARBOR_USERNAME") password := os.Getenv("HARBOR_PASSWORD") if username == "" || password == "" { + errLog.Error().Msg("HARBOR_USERNAME or HARBOR_PASSWORD environment variable is not set") return fmt.Errorf("HARBOR_USERNAME or HARBOR_PASSWORD environment variable is not set") } @@ -169,28 +184,26 @@ func CopyImage(imageName string) error { // Pull the image with authentication srcImage, err := crane.Pull(imageName, crane.WithAuth(auth), crane.Insecure) if err != nil { - fmt.Printf("Failed to pull image: %v\n", err) + errLog.Error().Msgf("Failed to pull image: %v", err) return fmt.Errorf("failed to pull image: %w", err) } else { - fmt.Println("Image pulled successfully") - fmt.Printf("Pulled image details: %+v\n", srcImage) + log.Info().Msg("Image pulled successfully") } // Push the image to the destination registry err = crane.Push(srcImage, destRef, crane.Insecure) if err != nil { - fmt.Printf("Failed to push image: %v\n", err) + errLog.Error().Msgf("Failed to push image: %v", err) return fmt.Errorf("failed to push image: %w", err) } else { - fmt.Println("Image pushed successfully") - fmt.Printf("Pushed image to: %s\n", destRef) + log.Info().Msg("Image pushed successfully") } // Delete ./local-oci-layout directory // This is required because it is a temporary directory used by crane to pull and push images to and from // And crane does not automatically clean it if err := os.RemoveAll("./local-oci-layout"); err != nil { - fmt.Printf("Failed to remove directory: %v\n", err) + errLog.Error().Msgf("Failed to remove directory: %v", err) return fmt.Errorf("failed to remove directory: %w", err) } diff --git a/internal/satellite/satellite.go b/internal/satellite/satellite.go index e7bdf79..91eb644 100644 --- a/internal/satellite/satellite.go +++ b/internal/satellite/satellite.go @@ -2,11 +2,11 @@ package satellite import ( "context" - "fmt" "time" "container-registry.com/harbor-satellite/internal/replicate" "container-registry.com/harbor-satellite/internal/store" + "container-registry.com/harbor-satellite/logger" ) type Satellite struct { @@ -14,7 +14,7 @@ type Satellite struct { replicator replicate.Replicator } -func NewSatellite(storer store.Storer, replicator replicate.Replicator) *Satellite { +func NewSatellite(ctx context.Context, storer store.Storer, replicator replicate.Replicator) *Satellite { return &Satellite{ storer: storer, replicator: replicator, @@ -22,23 +22,28 @@ func NewSatellite(storer store.Storer, replicator replicate.Replicator) *Satelli } func (s *Satellite) Run(ctx context.Context) error { + log := logger.FromContext(ctx) + errLog := logger.ErrorLoggerFromContext(ctx) + // Execute the initial operation immediately without waiting for the ticker imgs, err := s.storer.List(ctx) if err != nil { + errLog.Error().Err(err).Msg("Error listing images") return err } if len(imgs) == 0 { - fmt.Println("No images to replicate") + log.Info().Msg("No images to replicate") } else { for _, img := range imgs { err = s.replicator.Replicate(ctx, img.Name) if err != nil { + errLog.Error().Err(err).Msg("Error replicating image") return err } } s.replicator.DeleteExtraImages(ctx, imgs) } - fmt.Print("--------------------------------\n") + log.Info().Msg("--------------------------------\n") // Temporarily set to faster tick rate for testing purposes ticker := time.NewTicker(3 * time.Second) @@ -51,20 +56,22 @@ func (s *Satellite) Run(ctx context.Context) error { case <-ticker.C: imgs, err := s.storer.List(ctx) if err != nil { + errLog.Error().Err(err).Msg("Error listing images") return err } if len(imgs) == 0 { - fmt.Println("No images to replicate") + log.Info().Msg("No images to replicate") } else { for _, img := range imgs { err = s.replicator.Replicate(ctx, img.Name) if err != nil { + errLog.Error().Err(err).Msg("Error replicating image") return err } } s.replicator.DeleteExtraImages(ctx, imgs) } } - fmt.Print("--------------------------------\n") + log.Info().Msg("--------------------------------\n") } } diff --git a/internal/store/file-fetch.go b/internal/store/file-fetch.go index 3ff56af..78e0690 100644 --- a/internal/store/file-fetch.go +++ b/internal/store/file-fetch.go @@ -3,9 +3,10 @@ package store import ( "context" "encoding/json" - "fmt" "os" "path/filepath" + + "container-registry.com/harbor-satellite/logger" ) type FileImageList struct { @@ -24,15 +25,16 @@ type ImageData struct { Repositories []Repository `json:"repositories"` } -func (f *FileImageList) Type() string { +func (f *FileImageList) Type(ctx context.Context) string { return "File" } -func FileImageListFetcher(relativePath string) *FileImageList { +func FileImageListFetcher(ctx context.Context, relativePath string) *FileImageList { + errLog := logger.ErrorLoggerFromContext(ctx) // Get the current working directory dir, err := os.Getwd() if err != nil { - fmt.Println("Error getting current directory:", err) + errLog.Error().Err(err).Msg("Error getting current directory") return nil } @@ -45,11 +47,13 @@ func FileImageListFetcher(relativePath string) *FileImageList { } func (client *FileImageList) List(ctx context.Context) ([]Image, error) { + errLog := logger.ErrorLoggerFromContext(ctx) var images []Image // Read the file data, err := os.ReadFile(client.Path) if err != nil { + errLog.Error().Err(err).Msg("Error reading file") return nil, err } @@ -57,6 +61,7 @@ func (client *FileImageList) List(ctx context.Context) ([]Image, error) { // Parse the JSON data err = json.Unmarshal(data, &imageData) if err != nil { + errLog.Error().Err(err).Msg("Error unmarshalling JSON data") return nil, err } diff --git a/internal/store/http-fetch.go b/internal/store/http-fetch.go index 309287a..352a315 100644 --- a/internal/store/http-fetch.go +++ b/internal/store/http-fetch.go @@ -9,7 +9,9 @@ import ( "net/http" "os" "strings" + "time" + "container-registry.com/harbor-satellite/logger" "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/crane" ) @@ -23,17 +25,19 @@ type TagListResponse struct { Tags []string `json:"tags"` } -func RemoteImageListFetcher(url string) *RemoteImageList { +func RemoteImageListFetcher(ctx context.Context, url string) *RemoteImageList { return &RemoteImageList{ BaseURL: url, } } -func (r *RemoteImageList) Type() string { +func (r *RemoteImageList) Type(ctx context.Context) string { return "Remote" } func (client *RemoteImageList) List(ctx context.Context) ([]Image, error) { + log := logger.FromContext(ctx) + errLog := logger.ErrorLoggerFromContext(ctx) // Construct the URL for fetching tags url := client.BaseURL + "/tags/list" @@ -45,30 +49,39 @@ func (client *RemoteImageList) List(ctx context.Context) ([]Image, error) { // Create a new HTTP request req, err := http.NewRequest("GET", url, nil) if err != nil { - return nil, fmt.Errorf("failed to create request: %w", err) + errLog.Error().Msgf("failed to create request: %v", err) + return nil, err } // Set the Authorization header req.Header.Set("Authorization", "Basic "+auth) - // Send the request - httpClient := &http.Client{} + // Configure the HTTP client with a timeout + httpClient := &http.Client{ + Timeout: 5 * time.Second, + } + + // Send the HTTP request + log.Info().Msgf("Sending request to %s", url) resp, err := httpClient.Do(req) if err != nil { - return nil, fmt.Errorf("failed to fetch tags: %w", err) + errLog.Error().Msgf("failed to send request: %v", err) + return nil, err } defer resp.Body.Close() // Read the response body body, err := io.ReadAll(resp.Body) if err != nil { - return nil, fmt.Errorf("failed to read response body: %w", err) + errLog.Error().Msgf("failed to read response body: %v", err) + return nil, err } // Unmarshal the JSON response var tagListResponse TagListResponse if err := json.Unmarshal(body, &tagListResponse); err != nil { - return nil, fmt.Errorf("failed to unmarshal JSON response: %w", err) + errLog.Error().Msgf("failed to unmarshal response: %v", err) + return nil, err } // Prepare a slice to store the images @@ -80,11 +93,11 @@ func (client *RemoteImageList) List(ctx context.Context) ([]Image, error) { Name: fmt.Sprintf("%s:%s", tagListResponse.Name, tag), }) } - fmt.Println("Fetched", len(images), "images :", images) return images, nil } func (client *RemoteImageList) GetDigest(ctx context.Context, tag string) (string, error) { + errLog := logger.ErrorLoggerFromContext(ctx) // Construct the image reference imageRef := fmt.Sprintf("%s:%s", client.BaseURL, tag) // Remove extra characters from the URL @@ -101,7 +114,7 @@ func (client *RemoteImageList) GetDigest(ctx context.Context, tag string) (strin Password: password, }), crane.Insecure) if err != nil { - fmt.Printf("failed to fetch digest for %s: %v\n", imageRef, err) + errLog.Error().Msgf("failed to get digest using crane: %v", err) return "", nil } diff --git a/internal/store/in-memory-store.go b/internal/store/in-memory-store.go index de3a702..c0913e2 100644 --- a/internal/store/in-memory-store.go +++ b/internal/store/in-memory-store.go @@ -6,6 +6,7 @@ import ( "os" "strings" + "container-registry.com/harbor-satellite/logger" "github.com/google/go-containerregistry/pkg/crane" ) @@ -28,28 +29,31 @@ type Storer interface { type ImageFetcher interface { List(ctx context.Context) ([]Image, error) GetDigest(ctx context.Context, tag string) (string, error) - Type() string + Type(ctx context.Context) string } -func NewInMemoryStore(fetcher ImageFetcher) Storer { - return &inMemoryStore{ +func NewInMemoryStore(ctx context.Context, fetcher ImageFetcher) (context.Context, Storer) { + return ctx, &inMemoryStore{ images: make(map[string]string), fetcher: fetcher, } } func (s *inMemoryStore) List(ctx context.Context) ([]Image, error) { + log := logger.FromContext(ctx) + errLog := logger.ErrorLoggerFromContext(ctx) var imageList []Image var change bool + err := error(nil) // Fetch images from the file/remote source - imageList, err := s.fetcher.List(ctx) + imageList, err = s.fetcher.List(ctx) if err != nil { return nil, err } // Handle File and Remote fetcher types differently - switch s.fetcher.Type() { + switch s.fetcher.Type(ctx) { case "File": for _, img := range imageList { // Check if the image already exists in the store @@ -58,7 +62,7 @@ func (s *inMemoryStore) List(ctx context.Context) ([]Image, error) { s.AddImage(ctx, img.Name) change = true } else { - fmt.Printf("Image %s already exists in the store\n", img.Name) + log.Info().Msgf("Image %s already exists in the store", img.Name) } } @@ -84,9 +88,9 @@ func (s *inMemoryStore) List(ctx context.Context) ([]Image, error) { } // Print out the entire store for debugging purposes - fmt.Println("Current store:") + log.Info().Msg("Current store:") for image := range s.images { - fmt.Printf("Image: %s\n", image) + log.Info().Msgf("Image: %s", image) } case "Remote": @@ -104,7 +108,7 @@ func (s *inMemoryStore) List(ctx context.Context) ([]Image, error) { tagParts := strings.Split(img.Name, ":") // Check if there is a tag part, min length is 1 char if len(tagParts) < 2 { - fmt.Println("No tag part found in the image reference") + errLog.Error().Msgf("Invalid image reference: %s", img.Name) } // Use the last part as the tag tag := tagParts[len(tagParts)-1] @@ -115,7 +119,7 @@ func (s *inMemoryStore) List(ctx context.Context) ([]Image, error) { } // Check if the image exists and matches the digest - if !(s.checkImageAndDigest(digest, img.Name)) { + if !(s.checkImageAndDigest(ctx, digest, img.Name)) { change = true } @@ -135,9 +139,9 @@ func (s *inMemoryStore) List(ctx context.Context) ([]Image, error) { } } // Print out the entire store for debugging purposes - fmt.Println("Current store:") + log.Info().Msg("Current store:") for digest, imageRef := range s.images { - fmt.Printf("Digest: %s, Image: %s\n", digest, imageRef) + log.Info().Msgf("Image: %s, Digest: %s", imageRef, digest) } // Empty and refill imageList with the contents from s.images @@ -148,58 +152,64 @@ func (s *inMemoryStore) List(ctx context.Context) ([]Image, error) { } if change { - fmt.Println("Changes detected in the store") + log.Info().Msg("Changes detected in the store") change = false return imageList, nil } else { - fmt.Println("No changes detected in the store") + log.Info().Msg("No changes detected in the store") return nil, nil } } func (s *inMemoryStore) Add(ctx context.Context, digest string, image string) error { + log := logger.FromContext(ctx) // Check if the image already exists in the store if _, exists := s.images[digest]; exists { - fmt.Printf("Image: %s, digest: %s already exists in the store.\n", image, digest) + log.Info().Msgf("Image: %s, digest: %s already exists in the store.", image, digest) return fmt.Errorf("image %s already exists in the store", image) } else { // Add the image and its digest to the store s.images[digest] = image - fmt.Printf("Successfully added image: %s, digest: %s\n", image, digest) + log.Info().Msgf("Successfully added image: %s, digest: %s", image, digest) return nil } } func (s *inMemoryStore) AddImage(ctx context.Context, image string) error { + log := logger.FromContext(ctx) // Add the image to the store s.images[image] = "" - fmt.Printf("Added image: %s\n", image) + log.Info().Msgf("Added image: %s", image) return nil } func (s *inMemoryStore) Remove(ctx context.Context, digest string, image string) error { + log := logger.FromContext(ctx) // Check if the image exists in the store if _, exists := s.images[digest]; exists { // Remove the image and its digest from the store delete(s.images, digest) - fmt.Printf("Successfully removed image: %s, digest: %s\n", image, digest) + log.Info().Msgf("Successfully removed image: %s, digest: %s", image, digest) return nil } else { - fmt.Printf("Failed to remove image: %s, digest: %s. Not found in the store.\n", image, digest) + log.Warn().Msgf("Failed to remove image: %s, digest: %s. Not found in the store.", image, digest) return fmt.Errorf("image %s not found in the store", image) } } func (s *inMemoryStore) RemoveImage(ctx context.Context, image string) error { + log := logger.FromContext(ctx) // Remove the image from the store delete(s.images, image) - fmt.Printf("Removed image: %s\n", image) + log.Info().Msgf("Removed image: %s", image) return nil } // TODO: Rework complicated logic and add support for multiple repositories // checkImageAndDigest checks if the image exists in the store and if the digest matches the image reference -func (s *inMemoryStore) checkImageAndDigest(digest string, image string) bool { +func (s *inMemoryStore) checkImageAndDigest(ctx context.Context, digest string, image string) bool { + errLog := logger.ErrorLoggerFromContext(ctx) + // Check if the received image exists in the store for storeDigest, storeImage := range s.images { if storeImage == image { @@ -210,7 +220,7 @@ func (s *inMemoryStore) checkImageAndDigest(digest string, image string) bool { tag := strings.Split(image, ":")[1] localRegistryDigest, err := GetLocalDigest(context.Background(), tag) if err != nil { - fmt.Println("Error getting digest from local registry:", err) + errLog.Error().Msgf("Error getting digest from local registry: %v", err) return false } else { // Check if the digest from the local registry matches the digest from the store @@ -237,6 +247,8 @@ func (s *inMemoryStore) checkImageAndDigest(digest string, image string) bool { } func GetLocalDigest(ctx context.Context, tag string) (string, error) { + errLog := logger.ErrorLoggerFromContext(ctx) + zotUrl := os.Getenv("ZOT_URL") userURL := os.Getenv("USER_INPUT") // Remove extra characters from the URLs @@ -250,6 +262,7 @@ func GetLocalDigest(ctx context.Context, tag string) (string, error) { // Use crane.Digest to get the digest of the image digest, err := crane.Digest(url) if err != nil { + errLog.Error().Msgf("Error getting digest using crane: %v", err) return "", fmt.Errorf("failed to get digest using crane: %w", err) } diff --git a/logger/logger.go b/logger/logger.go new file mode 100644 index 0000000..eb813ec --- /dev/null +++ b/logger/logger.go @@ -0,0 +1,67 @@ +package logger + +import ( + "context" + "os" + + "github.com/rs/zerolog" +) + +type contextKey string + +const loggerKey contextKey = "logger" +const errorLoggerKey contextKey = "errorLogger" + +// NewContextWithLogger creates a new context with a zerolog logger for stdout adn stderr and sets the global log level. +func NewContextWithLogger(ctx context.Context, logLevel string) context.Context { + // Set log level to configured value + switch logLevel { + case "debug": + zerolog.SetGlobalLevel(zerolog.DebugLevel) + case "info": + zerolog.SetGlobalLevel(zerolog.InfoLevel) + case "warn": + zerolog.SetGlobalLevel(zerolog.WarnLevel) + case "error": + zerolog.SetGlobalLevel(zerolog.ErrorLevel) + case "fatal": + zerolog.SetGlobalLevel(zerolog.FatalLevel) + case "panic": + zerolog.SetGlobalLevel(zerolog.PanicLevel) + default: + zerolog.SetGlobalLevel(zerolog.InfoLevel) + } + // Use os.Stdout for the main logger + logger := zerolog.New(os.Stdout).With().Timestamp().Logger() + ctx = context.WithValue(ctx, loggerKey, &logger) + + // Use os.Stderr for the error logger + errorLogger := zerolog.New(os.Stderr).With().Timestamp().Logger() + ctx = context.WithValue(ctx, errorLoggerKey, &errorLogger) + + return ctx +} + +// FromContext extracts the main logger from the context. +func FromContext(ctx context.Context) *zerolog.Logger { + logger, ok := ctx.Value(loggerKey).(*zerolog.Logger) + if !ok { + // Fallback to a default logger if none is found in the context. + defaultLogger := zerolog.New(os.Stderr).With().Timestamp().Logger() + defaultLogger.Error().Msg("Failed to extract logger from context") + return &defaultLogger + } + return logger +} + +// ErrorLoggerFromContext extracts the error logger from the context. +func ErrorLoggerFromContext(ctx context.Context) *zerolog.Logger { + errorLogger, ok := ctx.Value(errorLoggerKey).(*zerolog.Logger) + if !ok { + // Fallback to a default logger if none is found in the context. + defaultErrorLogger := zerolog.New(os.Stderr).With().Timestamp().Logger() + defaultErrorLogger.Error().Msg("Failed to extract error logger from context") + return &defaultErrorLogger + } + return errorLogger +} diff --git a/main.go b/main.go index 0201807..c9bc433 100644 --- a/main.go +++ b/main.go @@ -4,7 +4,7 @@ import ( "context" "errors" "fmt" - "log" + "net" "net/http" "net/http/pprof" "net/url" @@ -18,20 +18,30 @@ import ( "container-registry.com/harbor-satellite/internal/replicate" "container-registry.com/harbor-satellite/internal/satellite" "container-registry.com/harbor-satellite/internal/store" + "container-registry.com/harbor-satellite/logger" "container-registry.com/harbor-satellite/registry" "golang.org/x/sync/errgroup" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/spf13/viper" "github.com/joho/godotenv" ) func main() { + viper.SetConfigName("config") + viper.SetConfigType("toml") + viper.AddConfigPath(".") + if err := viper.ReadInConfig(); err != nil { + fmt.Println("Error reading config file, ", err) + fmt.Println("Exiting Satellite") + os.Exit(1) + } + err := run() if err != nil { - fmt.Println(err) os.Exit(1) } } @@ -43,13 +53,12 @@ func run() error { defer cancel() g, ctx := errgroup.WithContext(ctx) - viper.SetConfigName("config") - viper.SetConfigType("toml") - viper.AddConfigPath(".") + logLevel := viper.GetString("log_level") + ctx = logger.NewContextWithLogger(ctx, logLevel) - if err := viper.ReadInConfig(); err != nil { - return fmt.Errorf("fatal error config file: %w", err) - } + log := logger.FromContext(ctx) + errLog := logger.ErrorLoggerFromContext(ctx) + log.Info().Msg("Satellite starting") mux := http.NewServeMux() mux.Handle("/metrics", promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{})) @@ -84,26 +93,33 @@ func run() error { registryAdr := viper.GetString("own_registry_adr") // Validate registryAdr format - // matched, err := regexp.MatchString(`^127\.0\.0\.1:\d{1,5}$`, registryAdr) - // if err != nil { - // return fmt.Errorf("error validating registry address: %w", err) - // } - // if matched { - // return fmt.Errorf("invalid registry address format: %s", registryAdr) - // } - os.Setenv("ZOT_URL", registryAdr) - fmt.Println("Registry URL set to:", registryAdr) + ip := net.ParseIP(registryAdr) + if ip == nil { + errLog.Error().Msg("Invalid IP address") + return errors.New("invalid IP address") + } + if ip.To4() != nil { + log.Info().Msg("IP address is valid IPv4") + } else { + errLog.Error().Msg("IP address is IPv6 format and unsupported") + return errors.New("IP address is IPv6 format and unsupported") + } + registryPort := viper.GetString("own_registry_port") + os.Setenv("ZOT_URL", registryAdr+":"+registryPort) } else { + log.Info().Msg("Launching default registry") g.Go(func() error { - launch, err := registry.LaunchRegistry() + launch, err := registry.LaunchRegistry(viper.GetString("zotConfigPath")) if launch { cancel() - return nil - } else { - log.Println("Error launching registry :", err) + return err + } + if err != nil { cancel() + errLog.Error().Err(err).Msg("Failed to launch default registry") return err } + return nil }) } @@ -111,25 +127,25 @@ func run() error { parsedURL, err := url.Parse(input) if err != nil || parsedURL.Scheme == "" { if strings.ContainsAny(input, "\\:*?\"<>|") { - fmt.Println("Path contains invalid characters. Please check the configuration.") - return fmt.Errorf("invalid file path") + errLog.Error().Msg("Path contains invalid characters. Please check the configuration.") + return err } dir, err := os.Getwd() if err != nil { - fmt.Println("Error getting current directory:", err) + errLog.Error().Err(err).Msg("Error getting current directory") return err } absPath := filepath.Join(dir, input) if _, err := os.Stat(absPath); os.IsNotExist(err) { - fmt.Println("No URL or file found. Please check the configuration.") - return fmt.Errorf("file not found") + errLog.Error().Err(err).Msg("No URL or file found. Please check the configuration.") + return err } - fmt.Println("Input is a valid file path.") - fetcher = store.FileImageListFetcher(input) + log.Info().Msg("Input is a valid file path.") + fetcher = store.FileImageListFetcher(ctx, input) os.Setenv("USER_INPUT", input) } else { - fmt.Println("Input is a valid URL.") - fetcher = store.RemoteImageListFetcher(input) + log.Info().Msg("Input is a valid URL.") + fetcher = store.RemoteImageListFetcher(ctx, input) os.Setenv("USER_INPUT", input) parts := strings.SplitN(input, "://", 2) scheme := parts[0] + "://" @@ -148,19 +164,22 @@ func run() error { err = godotenv.Load() if err != nil { - log.Fatalf("Error loading.env file: %v", err) + errLog.Error().Err(err).Msg("Error loading.env file") + return err } - storer := store.NewInMemoryStore(fetcher) - replicator := replicate.NewReplicator() - s := satellite.NewSatellite(storer, replicator) + ctx, storer := store.NewInMemoryStore(ctx, fetcher) + replicator := replicate.NewReplicator(ctx) + s := satellite.NewSatellite(ctx, storer, replicator) g.Go(func() error { return s.Run(ctx) }) + log.Info().Msg("Satellite running") err = g.Wait() if err != nil { + errLog.Error().Err(err).Msg("Error running satellite") return err } return nil diff --git a/registry/config.json b/registry/config.json index 34e2b82..7b8f91a 100644 --- a/registry/config.json +++ b/registry/config.json @@ -1,13 +1,10 @@ { "distSpecVersion": "1.1.0", "storage": { - "rootDirectory": "/tmp/zot" + "rootDirectory": "./zot" }, "http": { "address": "127.0.0.1", "port": "8585" - }, - "log": { - "level": "" } } diff --git a/registry/launch-registry.go b/registry/launch-registry.go index b4cac8d..1d7c536 100644 --- a/registry/launch-registry.go +++ b/registry/launch-registry.go @@ -1,24 +1,20 @@ package registry import ( - "log" - "zotregistry.dev/zot/pkg/cli/server" ) -func LaunchRegistry() (bool, error) { - log.Println("Launching Registry") +func LaunchRegistry(zotConfigPath string) (bool, error) { // Create the root command for the server rootCmd := server.NewServerRootCmd() // Set the arguments - rootCmd.SetArgs([]string{"serve", "./registry/config.json"}) + rootCmd.SetArgs([]string{"serve", zotConfigPath}) // Execute the root command err := rootCmd.Execute() if err != nil { - log.Fatalf("Error executing server root command: %v", err) return false, err }