From 7e52b359ea53eca4e2b771ae4c8481b02968f0fe Mon Sep 17 00:00:00 2001 From: Marco Deicas Date: Wed, 4 Oct 2023 10:56:02 -0400 Subject: [PATCH 01/21] update go version (#1333) * update go in workflows Signed-off-by: Marco Deicas * fix formatting in yaml file Signed-off-by: Marco Deicas * set up go before running golangci-lint-action Signed-off-by: Marco Deicas * remove optimization Signed-off-by: Marco Deicas * update go.mod and go.sum Signed-off-by: Marco Deicas --------- Signed-off-by: Marco Deicas --- .github/workflows/ci.yaml | 14 +++-- .github/workflows/postmerge.yaml | 2 +- .github/workflows/release.yaml | 2 + go.mod | 2 +- go.sum | 61 +++++++++++++++++++ .../datasource/csubsource/csubsource.go | 1 - 6 files changed, 74 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5c265365dd..8db2f6c8f3 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -65,7 +65,7 @@ jobs: - name: setup-go uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # tag=v3.2.1 with: - go-version: '1.19' + go-version: '1.21' - uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2 with: path: ~/go/pkg/mod @@ -86,7 +86,7 @@ jobs: - name: setup-go uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # tag=v3.2.1 with: - go-version: '1.19' + go-version: '1.21' - uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2 with: path: ~/go/pkg/mod @@ -105,7 +105,7 @@ jobs: - name: setup-go uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # tag=v3.2.1 with: - go-version: '1.19' + go-version: '1.21' - name: Install formatter run: go install golang.org/x/tools/cmd/goimports@latest - name: Check format @@ -118,6 +118,10 @@ jobs: steps: - name: Checkout code uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # tag=v3 + - name: setup-go + uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # tag=v4.1.0 + with: + go-version: '1.21' - name: golangci-lint uses: golangci/golangci-lint-action@3a919529898de77ec3da873e3063ca4b10e7f5cc # tag=v3.2.0 with: @@ -134,7 +138,7 @@ jobs: - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 - uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe with: - go-version: '~1.19' + go-version: '~1.21' - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 with: python-version: '3.10' @@ -190,7 +194,7 @@ jobs: - name: setup-go uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # tag=v3.2.1 with: - go-version: '1.19' + go-version: '1.21' - uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2 with: path: ~/go/pkg/mod diff --git a/.github/workflows/postmerge.yaml b/.github/workflows/postmerge.yaml index 26219a44b8..3abb7cf36b 100644 --- a/.github/workflows/postmerge.yaml +++ b/.github/workflows/postmerge.yaml @@ -30,7 +30,7 @@ jobs: - name: setup-go uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # tag=v3.2.1 with: - go-version: '1.19' + go-version: '1.21' - uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2 with: path: ~/go/pkg/mod diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 28f8679d70..e6dbf54e8a 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -48,6 +48,8 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Set up Go uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.1.0 + with: + go-version: '1.21' - name: Install cosign uses: sigstore/cosign-installer@11086d25041f77fe8fe7b9ea4e48e3b9192b8f19 # main - name: Install syft diff --git a/go.mod b/go.mod index 8b3289612a..ebbd94fad2 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/guacsec/guac -go 1.19 +go 1.21 require ( cloud.google.com/go/storage v1.33.0 diff --git a/go.sum b/go.sum index ce1191a5a7..d505ce9a43 100644 --- a/go.sum +++ b/go.sum @@ -37,6 +37,7 @@ cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1 cloud.google.com/go/iam v1.1.1 h1:lW7fzj15aVIXYHREOqjRBV9PsH0Z6u8Y46a1YGvQP4Y= cloud.google.com/go/iam v1.1.1/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= cloud.google.com/go/kms v1.15.0 h1:xYl5WEaSekKYN5gGRyhjvZKM22GVBBCzegGNVPy+aIs= +cloud.google.com/go/kms v1.15.0/go.mod h1:c9J991h5DTl+kg7gi3MYomh12YEENGrf48ee/N/2CDM= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -68,6 +69,7 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/CycloneDX/cyclonedx-go v0.7.2 h1:kKQ0t1dPOlugSIYVOMiMtFqeXI2wp/f5DBIdfux8gnQ= github.com/CycloneDX/cyclonedx-go v0.7.2/go.mod h1:K2bA+324+Og0X84fA8HhN2X066K7Bxz4rpMQ4ZhjtSk= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= +github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/DATA-DOG/go-txdb v0.1.7 h1:ibr3YvD3SKI4oBPbXbmzsn7eCPlg9oFdDdFtsWCvy7Q= github.com/DATA-DOG/go-txdb v0.1.7/go.mod h1:l06JaBQdV+y4aWAmDmWj4NwfnJknEXBxg8d4B8sJzXA= github.com/Khan/genqlient v0.6.0 h1:Bwb1170ekuNIVIwTJEqvO8y7RxBxXu639VJOkKSrwAk= @@ -96,7 +98,9 @@ github.com/anchore/go-struct-converter v0.0.0-20221118182256-c68fdcfa2092/go.mod github.com/anchore/go-struct-converter v0.0.0-20221221214134-65614c61201e h1:rlGNuYoCgsG/rB90tj4qF/5V09JHguPIyoBL8wn/rXg= github.com/anchore/go-struct-converter v0.0.0-20221221214134-65614c61201e/go.mod h1:rYqSE9HbjzpHTI74vwPvae4ZVYZd1lue2ta6xHPdblA= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ= +github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6iT90AvPUL1NNfNw= github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo= github.com/arangodb/go-driver v1.6.0 h1:NFWj/idqXZxhFVueihMSI2R9NotNIsgvNfM/xmpekb4= @@ -106,27 +110,47 @@ github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e/go.mod h1:m github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q= github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/aws/aws-sdk-go v1.45.20 h1:U/wLZEwqVB6o2XlcJ7um8kczx+A1X2MgO2y4wdKDQTs= github.com/aws/aws-sdk-go v1.45.20/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go-v2 v1.20.0 h1:INUDpYLt4oiPOJl0XwZDK2OVAVf0Rzo+MGVTv9f+gy8= +github.com/aws/aws-sdk-go-v2 v1.20.0/go.mod h1:uWOr0m0jDsiWw8nnXiqZ+YG6LdvAlGYDLLf2NmHZoy4= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.11 h1:/MS8AzqYNAhhRNalOmxUvYs8VEbNGifTnzhPFdcRQkQ= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.11/go.mod h1:va22++AdXht4ccO3kH2SHkHHYvZ2G9Utz+CXKmm2CaU= github.com/aws/aws-sdk-go-v2/config v1.18.32 h1:tqEOvkbTxwEV7hToRcJ1xZRjcATqwDVsWbAscgRKyNI= +github.com/aws/aws-sdk-go-v2/config v1.18.32/go.mod h1:U3ZF0fQRRA4gnbn9GGvOWLoT2EzzZfAWeKwnVrm1rDc= github.com/aws/aws-sdk-go-v2/credentials v1.13.31 h1:vJyON3lG7R8VOErpJJBclBADiWTwzcwdkQpTKx8D2sk= +github.com/aws/aws-sdk-go-v2/credentials v1.13.31/go.mod h1:T4sESjBtY2lNxLgkIASmeP57b5j7hTQqCbqG0tWnxC4= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.7 h1:X3H6+SU21x+76LRglk21dFRgMTJMa5QcpW+SqUf5BBg= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.7/go.mod h1:3we0V09SwcJBzNlnyovrR2wWJhWmVdqAsmVs4uronv8= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.76 h1:DJ1kHj0GI9BbX+XhF0kHxlzOVjcncmDUXmCvXdbfdAE= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.76/go.mod h1:/AZCdswMSgwpB2yMSFfY5H4pVeBLnCuPehdmO/r3xSM= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.37 h1:zr/gxAZkMcvP71ZhQOcvdm8ReLjFgIXnIn0fw5AM7mo= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.37/go.mod h1:Pdn4j43v49Kk6+82spO3Tu5gSeQXRsxo56ePPQAvFiA= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.31 h1:0HCMIkAkVY9KMgueD8tf4bRTUanzEYvhw7KkPXIMpO0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.31/go.mod h1:fTJDMe8LOFYtqiFFFeHA+SVMAwqLhoq0kcInYoLa9Js= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.38 h1:+i1DOFrW3YZ3apE45tCal9+aDKK6kNEbW6Ib7e1nFxE= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.38/go.mod h1:1/jLp0OgOaWIetycOmycW+vYTYgTZFPttJQRgsI1PoU= github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.0 h1:U5yySdwt2HPo/pnQec04DImLzWORbeWML1fJiLkKruI= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.0/go.mod h1:EhC/83j8/hL/UB1WmExo3gkElaja/KlmZM/gl1rTfjM= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.12 h1:uAiiHnWihGP2rVp64fHwzLDrswGjEjsPszwRYMiYQPU= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.12/go.mod h1:fUTHpOXqRQpXvEpDPSa3zxCc2fnpW6YnBoba+eQr+Bg= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.32 h1:kvN1jPHr9UffqqG3bSgZ8tx4+1zKVHz/Ktw/BwW6hX8= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.32/go.mod h1:QmMEM7es84EUkbYWcpnkx8i5EW2uERPfrTFeOch128Y= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.31 h1:auGDJ0aLZahF5SPvkJ6WcUuX7iQ7kyl2MamV7Tm8QBk= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.31/go.mod h1:3+lloe3sZuBQw1aBc5MyndvodzQlyqCZ7x1QPDHaWP4= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.0 h1:Wgjft9X4W5pMeuqgPCHIQtbZ87wsgom7S5F8obreg+c= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.0/go.mod h1:FWNzS4+zcWAP05IF7TDYTY1ysZAzIvogxWaDT9p8fsA= github.com/aws/aws-sdk-go-v2/service/s3 v1.38.1 h1:mTgFVlfQT8gikc5+/HwD8UL9jnUro5MGv8n/VEYF12I= +github.com/aws/aws-sdk-go-v2/service/s3 v1.38.1/go.mod h1:6SOWLiobcZZshbmECRTADIRYliPL0etqFSigauQEeT0= github.com/aws/aws-sdk-go-v2/service/sso v1.13.1 h1:DSNpSbfEgFXRV+IfEcKE5kTbqxm+MeF5WgyeRlsLnHY= +github.com/aws/aws-sdk-go-v2/service/sso v1.13.1/go.mod h1:TC9BubuFMVScIU+TLKamO6VZiYTkYoEHqlSQwAe2omw= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.1 h1:hd0SKLMdOL/Sl6Z0np1PX9LeH2gqNtBe0MhTedA8MGI= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.1/go.mod h1:XO/VcyoQ8nKyKfFW/3DMsRQXsfh/052tHTWmg3xBXRg= github.com/aws/aws-sdk-go-v2/service/sts v1.21.1 h1:pAOJj+80tC8sPVgSDHzMYD6KLWsaLQ1kZw31PTeORbs= +github.com/aws/aws-sdk-go-v2/service/sts v1.21.1/go.mod h1:G8SbvL0rFk4WOJroU8tKBczhsbhj2p/YY7qeJezJ3CI= github.com/aws/smithy-go v1.14.0 h1:+X90sB94fizKjDmwb4vyl2cTTPXTE5E2G/1mjByb0io= +github.com/aws/smithy-go v1.14.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= @@ -135,6 +159,7 @@ github.com/bombsimon/logrusr/v2 v2.0.1/go.mod h1:ByVAX+vHdLGAfdroiMg6q0zgq2FODY2 github.com/bradleyfalzon/ghinstallation/v2 v2.6.0 h1:IRY7Xy588KylkoycsUhFpW7cdGpy5Y5BPsz4IfuJtGk= github.com/bradleyfalzon/ghinstallation/v2 v2.6.0/go.mod h1:oQ3etOwN3TRH4EwgW5/7MxSVMGlMlzG/O8TU7eYdoSk= github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= +github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/caarlos0/env/v6 v6.10.1 h1:t1mPSxNpei6M5yAeu1qtRdPAK29Nbcf/n3G7x+b3/II= @@ -159,6 +184,7 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb h1:EDmT6Q9Zs+SbUoc7Ik9EfrFqcylYqgPZ9ANSbTAntnE= +github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb/go.mod h1:ZjrT6AXHbDs86ZSdt/osfBi5qfexBrKUdONk989Wnk4= github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be h1:J5BL2kskAlV9ckgEsNQXscjIaLiOYiZ75d4e94E6dcQ= github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be/go.mod h1:mk5IQ+Y0ZeO87b858TlA645sVcEcbiX6YqP98kt+7+w= github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSkDwbkEETK84kQgEeFwDC+62k= @@ -187,7 +213,9 @@ github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNk github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 h1:UhxFibDNY/bfvqU5CAUmr9zpesgbU6SWc8/B4mflAE4= github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= +github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -197,14 +225,18 @@ github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5y github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a h1:yDWHCSQ40h88yih2JAcL6Ls/kVkSE8GFACTGVnMPruw= +github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a/go.mod h1:7Ga40egUymuWXxAe151lTNnCv97MddSOVsjpPPkityA= github.com/facebookgo/limitgroup v0.0.0-20150612190941-6abd8d71ec01 h1:IeaD1VDVBPlx3viJT9Md8if8IxxJnO+x0JCGb054heg= +github.com/facebookgo/limitgroup v0.0.0-20150612190941-6abd8d71ec01/go.mod h1:ypD5nozFk9vcGw1ATYefw6jHe/jZP++Z15/+VTMcWhc= github.com/facebookgo/muster v0.0.0-20150708232844-fd3d7953fd52 h1:a4DFiKFJiDRGFD1qIcqGLX/WlUMD9dyLSLDt+9QZgt8= +github.com/facebookgo/muster v0.0.0-20150708232844-fd3d7953fd52/go.mod h1:yIquW87NGRw1FU5p5lEkpnt/QxoH5uPAOUlOVkAUuMg= github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA= +github.com/frankban/quicktest v1.14.5/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= @@ -213,11 +245,13 @@ github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbS github.com/fsouza/fake-gcs-server v1.47.4 h1:gfBhBxEra20/Om02cvcyL8EnekV8KDb01Yffjat6AKQ= github.com/fsouza/fake-gcs-server v1.47.4/go.mod h1:vqUZbI12uy9IkRQ54Q4p5AniQsSiUq8alO9Nv2egMmA= github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY= +github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU= github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow= github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f h1:Pz0DHeFij3XFhoBRGUDPzSJ+w2UcK5/0JvF8DRI58r8= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo= github.com/go-git/go-git/v5 v5.9.0 h1:cD9SFA7sHVRdJ7AYck1ZaAa/yeuBvGPxwXDL8cxrObY= github.com/go-git/go-git/v5 v5.9.0/go.mod h1:RKIqga24sWdMGZF+1Ekv9kylsDz6LzdTSI2s/OsZWE0= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= @@ -235,7 +269,9 @@ github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg= +github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= github.com/goark/errs v1.3.2 h1:ifccNe1aK7Xezt4XVYwHUqalmnfhuphnEvh3FshCReQ= github.com/goark/errs v1.3.2/go.mod h1:ZsQucxaDFVfSB8I99j4bxkDRfNOrlKINwg72QMuRWKw= github.com/goark/go-cvss v1.6.6 h1:WJFuIWqmAw1Ilb9USv0vuX+nYzOWJp8lIujseJ/y3sU= @@ -306,13 +342,16 @@ github.com/google/go-github/v53 v53.2.0/go.mod h1:XhFRObz+m/l+UCm9b7KSIC3lT3NWSX github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/go-replayers/grpcreplay v1.1.0 h1:S5+I3zYyZ+GQz68OfbURDdt/+cSMqCK1wrvNx7WBzTE= +github.com/google/go-replayers/grpcreplay v1.1.0/go.mod h1:qzAvJ8/wi57zq7gWqaE6AwLM6miiXUQwP1S+I9icmhk= github.com/google/go-replayers/httpreplay v1.2.0 h1:VM1wEyyjaoU53BwrOnaf9VhAyQQEEioJvFYxYcLRKzk= +github.com/google/go-replayers/httpreplay v1.2.0/go.mod h1:WahEFFZZ7a1P4VM1qEeHy+tME4bwyqPcwWbNlUI1Mcg= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= +github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/osv-scanner v1.3.6 h1:8oZxk2HfdMjsLZ/NIidEEb2srcIP8301hKbR+FPXaPo= github.com/google/osv-scanner v1.3.6/go.mod h1:Yl29UCfOjWRA/wQLmvK0Ehs7HztmRqtgvHa3SomAJFo= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -326,6 +365,7 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8 h1:n6vlPhxsA+BW/XsS5+uqi7GyzaLa5MH7qlSLBZtRdiA= +github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/renameio/v2 v2.0.0 h1:UifI23ZTGY8Tt29JbYFiuyIU3eX+RNFtUwefq9qAhxg= github.com/google/renameio/v2 v2.0.0/go.mod h1:BtmJXm5YlszgC+TD4HOEEUFgkJP3nLxehU6hfe7jRt4= @@ -361,6 +401,7 @@ github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9n github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-hclog v1.2.0 h1:La19f8d7WIlm4ogzNHB0JGqs5AUDAZ2UfCY4sJXcJdM= +github.com/hashicorp/go-hclog v1.2.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA= @@ -374,7 +415,9 @@ github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T github.com/hashicorp/hcl/v2 v2.13.0 h1:0Apadu1w6M11dyGFxWnmhhcMjkbAiKCv7G1r/2QgCNc= github.com/hashicorp/hcl/v2 v2.13.0/go.mod h1:e4z5nxYlWNPdDSNYX+ph14EvWYMFm3eP0zIUqPc2jr0= github.com/honeycombio/beeline-go v1.10.0 h1:cUDe555oqvw8oD76BQJ8alk7FP0JZ/M/zXpNvOEDLDc= +github.com/honeycombio/beeline-go v1.10.0/go.mod h1:Zz5WMeQCJzFt2Mvf8t6HC1X8RLskLVR/e8rvcmXB1G8= github.com/honeycombio/libhoney-go v1.16.0 h1:kPpqoz6vbOzgp7jC6SR7SkNj7rua7rgxvznI6M3KdHc= +github.com/honeycombio/libhoney-go v1.16.0/go.mod h1:izP4fbREuZ3vqC4HlCAmPrcPT9gxyxejRjGtCYpmBn0= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -393,6 +436,7 @@ github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHW github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmhodges/clock v0.0.0-20160418191101-880ee4c33548 h1:dYTbLf4m0a5u0KLmPfB6mgxbcV7588bOCx79hxa5Sr4= +github.com/jmhodges/clock v0.0.0-20160418191101-880ee4c33548/go.mod h1:hGT6jSUVzF6no3QaDSMLGLEHtHSBSefs+MgcDWnmhmo= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= @@ -404,17 +448,20 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM= github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= +github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/letsencrypt/boulder v0.0.0-20221109233200-85aa52084eaf h1:ndns1qx/5dL43g16EQkPV/i8+b3l5bYQwLeoSBe7tS8= github.com/letsencrypt/boulder v0.0.0-20221109233200-85aa52084eaf/go.mod h1:aGkAgvWY/IUcVFfuly53REpfv5edu25oij+qHRFaraA= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= @@ -437,13 +484,17 @@ github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y= +github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= github.com/minio/minio-go/v7 v7.0.61 h1:87c+x8J3jxQ5VUGimV9oHdpjsAvy3fhneEBKuoKEVUI= +github.com/minio/minio-go/v7 v7.0.61/go.mod h1:BTu8FcrEw+HidY0zd/0eny43QnVNkXRPXrLXFuQBHXg= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-wordwrap v1.0.0 h1:6GlHJ/LTGMrIJbwgdqdl2eEH8o+Exx/0m8ir9Gns0u4= @@ -479,10 +530,12 @@ github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vv github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= +github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= +github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0-rc3 h1:fzg1mXZFj8YdPeNkRXMg+zb88BFV0Ys52cJydRwBkb8= @@ -527,7 +580,9 @@ github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ= github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/secure-systems-lab/go-securesystemslib v0.7.0 h1:OwvJ5jQf9LnIAS83waAjPbcMsODrTQUpJ02eNLUoxBg= @@ -591,6 +646,7 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/terminalstatic/go-xsd-validate v0.1.5 h1:RqpJnf6HGE2CB/lZB1A8BYguk8uRtcvYAPLCF15qguo= +github.com/terminalstatic/go-xsd-validate v0.1.5/go.mod h1:18lsvYFofBflqCrvo1umpABZ99+GneNTw2kEEc8UPJw= github.com/theupdateframework/go-tuf v0.5.2 h1:habfDzTmpbzBLIFGWa2ZpVhYvFBoK0C1onC3a4zuPRA= github.com/theupdateframework/go-tuf v0.5.2/go.mod h1:SyMV5kg5n4uEclsyxXJZI2UxPFJNDc4Y+r7wv+MlvTA= github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399 h1:e/5i7d4oYZ+C1wj2THlRK+oAhjeS/TRQwMfkIuet3w0= @@ -615,7 +671,9 @@ github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -637,6 +695,7 @@ go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= +go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= @@ -1040,6 +1099,7 @@ google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alexcesaro/statsd.v2 v2.0.0 h1:FXkZSCZIH17vLCO5sO2UucTHsH9pc+17F6pl3JVCwMc= +gopkg.in/alexcesaro/statsd.v2 v2.0.0/go.mod h1:i0ubccKGzBVNBpdGV5MocxyA/XlLUJzA7SLonnE4drU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1065,6 +1125,7 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= +gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.1.0 h1:rVV8Tcg/8jHUkPUorwjaMTtemIMVXfIPKiOqnhEhakk= gotest.tools/v3 v3.1.0/go.mod h1:fHy7eyTmJFO5bQbUsEGQ1v4m2J3Jz9eWL54TP2/ZuYQ= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/pkg/collectsub/datasource/csubsource/csubsource.go b/pkg/collectsub/datasource/csubsource/csubsource.go index 599a477311..21316af4de 100644 --- a/pkg/collectsub/datasource/csubsource/csubsource.go +++ b/pkg/collectsub/datasource/csubsource/csubsource.go @@ -57,7 +57,6 @@ func (d *csubDataSources) GetDataSources(ctx context.Context) (*datasource.DataS return nil, err } ds := entriesToSources(ctx, entries) - d.lastEntries = ds return ds, nil } From 24dc68f69cc0ceb9783fbbae8f45893577f6a3de Mon Sep 17 00:00:00 2001 From: Marco Deicas Date: Wed, 4 Oct 2023 15:58:27 -0400 Subject: [PATCH 02/21] Fix lint errors and increase golangci-lint timeout (#1351) * increase ci timout Signed-off-by: Marco Deicas * update to not use deprecated functions Signed-off-by: Marco Deicas --------- Signed-off-by: Marco Deicas --- .golangci.yaml | 2 +- .../verifier/sigstore_verifier/sigstore_verifier_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.golangci.yaml b/.golangci.yaml index 548fc3ecb9..42b073a74f 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -1,5 +1,5 @@ run: - timeout: 3m + timeout: 6m skip-files: - ".*\\.pb\\.go$" - "pkg/assembler/generated/.*" diff --git a/pkg/ingestor/verifier/sigstore_verifier/sigstore_verifier_test.go b/pkg/ingestor/verifier/sigstore_verifier/sigstore_verifier_test.go index 79277b51c8..0c88692c2e 100644 --- a/pkg/ingestor/verifier/sigstore_verifier/sigstore_verifier_test.go +++ b/pkg/ingestor/verifier/sigstore_verifier/sigstore_verifier_test.go @@ -91,9 +91,9 @@ func setupOneProvider(t *testing.T) { func randomData(t *testing.T, n int) []byte { t.Helper() - rand.Seed(time.Now().UnixNano()) + gen := rand.New(rand.NewSource(time.Now().UnixNano())) data := make([]byte, n) - if _, err := rand.Read(data[:]); err != nil { + if _, err := gen.Read(data[:]); err != nil { t.Fatal(err) } return data From 686fcad099a0a60014d4d79852f4d95f0abc7089 Mon Sep 17 00:00:00 2001 From: Jeff Mendoza Date: Wed, 4 Oct 2023 14:16:04 -0700 Subject: [PATCH 03/21] Bump github.com/nats-io/nats-server (#1352) This also bumped golang.org/x/exp which included some breaking changes. We will move off golang.org/x/exp/slices onto the stdlib one soon. Signed-off-by: Jeff Mendoza --- go.mod | 6 +++--- go.sum | 12 ++++++------ pkg/assembler/backends/arangodb/artifact_test.go | 4 ++-- pkg/assembler/backends/arangodb/builder_test.go | 4 ++-- pkg/assembler/backends/arangodb/hashEqual_test.go | 4 ++-- pkg/assembler/backends/arangodb/pkg_test.go | 5 +++-- pkg/assembler/backends/arangodb/src_test.go | 5 +++-- .../backends/arangodb/vulnerability_test.go | 5 +++-- pkg/assembler/backends/ent/backend/backend.go | 6 +----- pkg/assembler/backends/ent/backend/hashequal_test.go | 5 +++-- pkg/assembler/backends/ent/backend/license_test.go | 5 +++-- pkg/assembler/backends/ent/backend/package.go | 7 ++++--- .../backends/ent/backend/vulnerability_test.go | 5 +++-- pkg/assembler/backends/inmem/hashEqual_test.go | 4 ++-- pkg/assembler/backends/inmem/license_test.go | 4 ++-- pkg/assembler/backends/inmem/vulnerability_test.go | 5 +++-- 16 files changed, 45 insertions(+), 41 deletions(-) diff --git a/go.mod b/go.mod index ebbd94fad2..7c9f67a016 100644 --- a/go.mod +++ b/go.mod @@ -120,7 +120,7 @@ require ( github.com/moby/buildkit v0.12.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/nats-io/jwt/v2 v2.4.1 // indirect + github.com/nats-io/jwt/v2 v2.5.2 // indirect github.com/nats-io/nkeys v0.4.5 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect @@ -200,7 +200,7 @@ require ( github.com/lib/pq v1.10.9 github.com/manifoldco/promptui v0.9.0 github.com/mitchellh/go-homedir v1.1.0 - github.com/nats-io/nats-server/v2 v2.9.21 + github.com/nats-io/nats-server/v2 v2.10.1 github.com/nats-io/nats.go v1.30.1 github.com/openvex/go-vex v0.2.5 github.com/ossf/scorecard/v4 v4.12.0 @@ -214,6 +214,6 @@ require ( github.com/spf13/viper v1.16.0 github.com/stretchr/testify v1.8.4 github.com/vektah/gqlparser/v2 v2.5.10 - golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 gopkg.in/yaml.v3 v3.0.1 ) diff --git a/go.sum b/go.sum index d505ce9a43..1377d4d4c9 100644 --- a/go.sum +++ b/go.sum @@ -508,10 +508,10 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/nats-io/jwt/v2 v2.4.1 h1:Y35W1dgbbz2SQUYDPCaclXcuqleVmpbRa7646Jf2EX4= -github.com/nats-io/jwt/v2 v2.4.1/go.mod h1:24BeQtRwxRV8ruvC4CojXlx/WQ/VjuwlYiH+vu/+ibI= -github.com/nats-io/nats-server/v2 v2.9.21 h1:2TBTh0UDE74eNXQmV4HofsmRSCiVN0TH2Wgrp6BD6fk= -github.com/nats-io/nats-server/v2 v2.9.21/go.mod h1:ozqMZc2vTHcNcblOiXMWIXkf8+0lDGAi5wQcG+O1mHU= +github.com/nats-io/jwt/v2 v2.5.2 h1:DhGH+nKt+wIkDxM6qnVSKjokq5t59AZV5HRcFW0zJwU= +github.com/nats-io/jwt/v2 v2.5.2/go.mod h1:24BeQtRwxRV8ruvC4CojXlx/WQ/VjuwlYiH+vu/+ibI= +github.com/nats-io/nats-server/v2 v2.10.1 h1:MIJ614dhOIdo71iSzY8ln78miXwrYvlvXHUyS+XdKZQ= +github.com/nats-io/nats-server/v2 v2.10.1/go.mod h1:3PMvMSu2cuK0J9YInRLWdFpFsswKKGUS77zVSAudRto= github.com/nats-io/nats.go v1.30.1 h1:o5RND+GaKgzNm2IOSLmHunWs6vH0GooAAaZZitiGJWk= github.com/nats-io/nats.go v1.30.1/go.mod h1:dcfhUgmQNN4GJEfIb2f9R7Fow+gzBF4emzDHrVBd5qM= github.com/nats-io/nkeys v0.4.5 h1:Zdz2BUlFm4fJlierwvGK+yl20IAKUm7eV6AAZXEhkPk= @@ -727,8 +727,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 h1:MGwJjxBy0HJshjDNfLsYO8xppfqWlA5ZT9OhtUUhTNw= -golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= diff --git a/pkg/assembler/backends/arangodb/artifact_test.go b/pkg/assembler/backends/arangodb/artifact_test.go index 7be1aecb65..d4a7d986d8 100644 --- a/pkg/assembler/backends/arangodb/artifact_test.go +++ b/pkg/assembler/backends/arangodb/artifact_test.go @@ -119,8 +119,8 @@ func getArangoConfig() *ArangoConfig { } } -func lessArtifact(a, b *model.Artifact) bool { - return a.Digest < b.Digest +func lessArtifact(a, b *model.Artifact) int { + return strings.Compare(a.Digest, b.Digest) } func Test_IngestArtifacts(t *testing.T) { diff --git a/pkg/assembler/backends/arangodb/builder_test.go b/pkg/assembler/backends/arangodb/builder_test.go index 82f13717e6..010f7dff71 100644 --- a/pkg/assembler/backends/arangodb/builder_test.go +++ b/pkg/assembler/backends/arangodb/builder_test.go @@ -116,8 +116,8 @@ func Test_IngestBuilder(t *testing.T) { } } -func lessBuilder(a, b *model.Builder) bool { - return a.URI < b.URI +func lessBuilder(a, b *model.Builder) int { + return strings.Compare(a.URI, b.URI) } func Test_IngestBuilders(t *testing.T) { diff --git a/pkg/assembler/backends/arangodb/hashEqual_test.go b/pkg/assembler/backends/arangodb/hashEqual_test.go index 7dc11fc182..ca7b1a1177 100644 --- a/pkg/assembler/backends/arangodb/hashEqual_test.go +++ b/pkg/assembler/backends/arangodb/hashEqual_test.go @@ -510,7 +510,7 @@ func TestHashEqual(t *testing.T) { if err != nil { return } - less := func(a, b *model.Artifact) bool { return a.Digest < b.Digest } + less := func(a, b *model.Artifact) int { return strings.Compare(a.Digest, b.Digest) } for _, he := range got { slices.SortFunc(he.Artifacts, less) } @@ -820,7 +820,7 @@ func TestIngestHashEquals(t *testing.T) { if err != nil { return } - less := func(a, b *model.Artifact) bool { return a.Digest < b.Digest } + less := func(a, b *model.Artifact) int { return strings.Compare(a.Digest, b.Digest) } for _, he := range got { slices.SortFunc(he.Artifacts, less) } diff --git a/pkg/assembler/backends/arangodb/pkg_test.go b/pkg/assembler/backends/arangodb/pkg_test.go index e01b1818ff..06c7d9b788 100644 --- a/pkg/assembler/backends/arangodb/pkg_test.go +++ b/pkg/assembler/backends/arangodb/pkg_test.go @@ -771,8 +771,9 @@ func Test_PackagesName(t *testing.T) { } } -func lessPkg(a, b *model.Package) bool { - return a.Namespaces[0].Names[0].Name < b.Namespaces[0].Names[0].Name +func lessPkg(a, b *model.Package) int { + return strings.Compare(a.Namespaces[0].Names[0].Name, + b.Namespaces[0].Names[0].Name) } func Test_IngestPackages(t *testing.T) { diff --git a/pkg/assembler/backends/arangodb/src_test.go b/pkg/assembler/backends/arangodb/src_test.go index 55f65feb9f..6ea72f87b8 100644 --- a/pkg/assembler/backends/arangodb/src_test.go +++ b/pkg/assembler/backends/arangodb/src_test.go @@ -29,8 +29,9 @@ import ( "golang.org/x/exp/slices" ) -func lessSource(a, b *model.Source) bool { - return a.Namespaces[0].Namespace < b.Namespaces[0].Namespace +func lessSource(a, b *model.Source) int { + return strings.Compare(a.Namespaces[0].Namespace, + b.Namespaces[0].Namespace) } func Test_IngestSources(t *testing.T) { diff --git a/pkg/assembler/backends/arangodb/vulnerability_test.go b/pkg/assembler/backends/arangodb/vulnerability_test.go index 4b5b89462f..9202903123 100644 --- a/pkg/assembler/backends/arangodb/vulnerability_test.go +++ b/pkg/assembler/backends/arangodb/vulnerability_test.go @@ -29,8 +29,9 @@ import ( "golang.org/x/exp/slices" ) -func lessCve(a, b *model.Vulnerability) bool { - return a.VulnerabilityIDs[0].VulnerabilityID < b.VulnerabilityIDs[0].VulnerabilityID +func lessCve(a, b *model.Vulnerability) int { + return strings.Compare(a.VulnerabilityIDs[0].VulnerabilityID, + b.VulnerabilityIDs[0].VulnerabilityID) } func TestVulnerability(t *testing.T) { diff --git a/pkg/assembler/backends/ent/backend/backend.go b/pkg/assembler/backends/ent/backend/backend.go index 55e383e83c..40c4772ef1 100644 --- a/pkg/assembler/backends/ent/backend/backend.go +++ b/pkg/assembler/backends/ent/backend/backend.go @@ -22,16 +22,12 @@ import ( "github.com/guacsec/guac/pkg/assembler/backends" "github.com/guacsec/guac/pkg/assembler/backends/ent" "github.com/vektah/gqlparser/v2/gqlerror" - "golang.org/x/exp/slices" // Import regular postgres driver _ "github.com/lib/pq" ) -var ( - PathContains = slices.Contains[string] - Errorf = gqlerror.Errorf -) +var Errorf = gqlerror.Errorf // MaxPageSize is the maximum number of results that will be returned in a single query. const MaxPageSize = 1000 diff --git a/pkg/assembler/backends/ent/backend/hashequal_test.go b/pkg/assembler/backends/ent/backend/hashequal_test.go index 09842daedf..f512f1f6d3 100644 --- a/pkg/assembler/backends/ent/backend/hashequal_test.go +++ b/pkg/assembler/backends/ent/backend/hashequal_test.go @@ -19,6 +19,7 @@ package backend import ( "strconv" + "strings" "github.com/google/go-cmp/cmp" "github.com/guacsec/guac/internal/testing/ptrfrom" @@ -520,7 +521,7 @@ func (s *Suite) TestHashEqual() { if err != nil { return } - less := func(a, b *model.Artifact) bool { return a.Digest < b.Digest } + less := func(a, b *model.Artifact) int { return strings.Compare(a.Digest, b.Digest) } for _, he := range got { slices.SortFunc(he.Artifacts, less) } @@ -787,7 +788,7 @@ func (s *Suite) TestIngestHashEquals() { if err != nil { return } - less := func(a, b *model.Artifact) bool { return a.Digest < b.Digest } + less := func(a, b *model.Artifact) int { return strings.Compare(a.Digest, b.Digest) } for _, he := range got { slices.SortFunc(he.Artifacts, less) } diff --git a/pkg/assembler/backends/ent/backend/license_test.go b/pkg/assembler/backends/ent/backend/license_test.go index 41ab7e9074..333d3ba82b 100644 --- a/pkg/assembler/backends/ent/backend/license_test.go +++ b/pkg/assembler/backends/ent/backend/license_test.go @@ -19,6 +19,7 @@ package backend import ( "strconv" + "strings" "github.com/google/go-cmp/cmp" "github.com/guacsec/guac/internal/testing/ptrfrom" @@ -191,8 +192,8 @@ func (s *Suite) TestLicense() { } } -func lessLic(a, b *model.License) bool { - return a.Name < b.Name +func lessLic(a, b *model.License) int { + return strings.Compare(a.Name, b.Name) } func (s *Suite) TestIngestLicenses() { diff --git a/pkg/assembler/backends/ent/backend/package.go b/pkg/assembler/backends/ent/backend/package.go index c5da8e230b..7f5d4ea6f9 100644 --- a/pkg/assembler/backends/ent/backend/package.go +++ b/pkg/assembler/backends/ent/backend/package.go @@ -33,6 +33,7 @@ import ( "github.com/guacsec/guac/pkg/assembler/backends/helper" "github.com/guacsec/guac/pkg/assembler/graphql/model" "github.com/pkg/errors" + "golang.org/x/exp/slices" ) func (b *EntBackend) Packages(ctx context.Context, pkgSpec *model.PkgSpec) ([]*model.Package, error) { @@ -64,15 +65,15 @@ func (b *EntBackend) Packages(ctx context.Context, pkgSpec *model.PkgSpec) ([]*m ), ) - if PathContains(paths, "namespaces") || !isGQL { + if slices.Contains(paths, "namespaces") || !isGQL { query.WithNamespaces(func(q *ent.PackageNamespaceQuery) { q.Where(optionalPredicate(pkgSpec.Namespace, packagenamespace.NamespaceEQ)) - if PathContains(paths, "namespaces.names") || !isGQL { + if slices.Contains(paths, "namespaces.names") || !isGQL { q.WithNames(func(q *ent.PackageNameQuery) { q.Where(optionalPredicate(pkgSpec.Name, packagename.NameEQ)) - if PathContains(paths, "namespaces.names.versions") || !isGQL { + if slices.Contains(paths, "namespaces.names.versions") || !isGQL { q.WithVersions(func(q *ent.PackageVersionQuery) { q.Where( optionalPredicate(pkgSpec.ID, IDEQ), diff --git a/pkg/assembler/backends/ent/backend/vulnerability_test.go b/pkg/assembler/backends/ent/backend/vulnerability_test.go index 2595d88461..cfb5cbe5b6 100644 --- a/pkg/assembler/backends/ent/backend/vulnerability_test.go +++ b/pkg/assembler/backends/ent/backend/vulnerability_test.go @@ -108,8 +108,9 @@ var noVulnOut = &model.VulnerabilityID{ VulnerabilityID: "", } -func lessCve(a, b *model.Vulnerability) bool { - return a.VulnerabilityIDs[0].VulnerabilityID < b.VulnerabilityIDs[0].VulnerabilityID +func lessCve(a, b *model.Vulnerability) int { + return strings.Compare(a.VulnerabilityIDs[0].VulnerabilityID, + b.VulnerabilityIDs[0].VulnerabilityID) } func (s *Suite) TestVulnerability() { diff --git a/pkg/assembler/backends/inmem/hashEqual_test.go b/pkg/assembler/backends/inmem/hashEqual_test.go index 3ba41e3f10..481ea696dd 100644 --- a/pkg/assembler/backends/inmem/hashEqual_test.go +++ b/pkg/assembler/backends/inmem/hashEqual_test.go @@ -440,7 +440,7 @@ func TestHashEqual(t *testing.T) { if err != nil { return } - less := func(a, b *model.Artifact) bool { return a.Digest < b.Digest } + less := func(a, b *model.Artifact) int { return strings.Compare(a.Digest, b.Digest) } for _, he := range got { slices.SortFunc(he.Artifacts, less) } @@ -709,7 +709,7 @@ func TestIngestHashEquals(t *testing.T) { if err != nil { return } - less := func(a, b *model.Artifact) bool { return a.Digest < b.Digest } + less := func(a, b *model.Artifact) int { return strings.Compare(a.Digest, b.Digest) } for _, he := range got { slices.SortFunc(he.Artifacts, less) } diff --git a/pkg/assembler/backends/inmem/license_test.go b/pkg/assembler/backends/inmem/license_test.go index 3d8b55e7d0..1ad3f16a5d 100644 --- a/pkg/assembler/backends/inmem/license_test.go +++ b/pkg/assembler/backends/inmem/license_test.go @@ -179,8 +179,8 @@ func TestLicense(t *testing.T) { } } -func lessLic(a, b *model.License) bool { - return a.Name < b.Name +func lessLic(a, b *model.License) int { + return strings.Compare(a.Name, b.Name) } func TestIngestLicenses(t *testing.T) { diff --git a/pkg/assembler/backends/inmem/vulnerability_test.go b/pkg/assembler/backends/inmem/vulnerability_test.go index 64590ed6b0..f46cfa754d 100644 --- a/pkg/assembler/backends/inmem/vulnerability_test.go +++ b/pkg/assembler/backends/inmem/vulnerability_test.go @@ -107,8 +107,9 @@ var noVulnOut = &model.VulnerabilityID{ VulnerabilityID: "", } -func lessCve(a, b *model.Vulnerability) bool { - return a.VulnerabilityIDs[0].VulnerabilityID < b.VulnerabilityIDs[0].VulnerabilityID +func lessCve(a, b *model.Vulnerability) int { + return strings.Compare(a.VulnerabilityIDs[0].VulnerabilityID, + b.VulnerabilityIDs[0].VulnerabilityID) } func TestVulnerability(t *testing.T) { From d681a8db4013ede6b6ce0b2ba3fc1c421e2b95b9 Mon Sep 17 00:00:00 2001 From: Nathan Naveen <42319948+nathannaveen@users.noreply.github.com> Date: Wed, 4 Oct 2023 16:26:21 -0500 Subject: [PATCH 04/21] Include Timestamps for Verbs (#1338) * Included Timestamp for Verbs * Fixes https://github.com/guacsec/guac/issues/1030 Signed-off-by: nathannaveen <42319948+nathannaveen@users.noreply.github.com> * Edited GraphQL Signed-off-by: nathannaveen <42319948+nathannaveen@users.noreply.github.com> * Fixed some stuff Signed-off-by: nathannaveen <42319948+nathannaveen@users.noreply.github.com> * Fixed SBOM issue Signed-off-by: nathannaveen <42319948+nathannaveen@users.noreply.github.com> * Fixed tests Signed-off-by: nathannaveen <42319948+nathannaveen@users.noreply.github.com> * Stored Times to UTC Signed-off-by: nathannaveen <42319948+nathannaveen@users.noreply.github.com> * Updated spec Signed-off-by: nathannaveen <42319948+nathannaveen@users.noreply.github.com> --------- Signed-off-by: nathannaveen <42319948+nathannaveen@users.noreply.github.com> --- cmd/guacgql/cmd/ingest.go | 5 + cmd/guacone/cmd/certify.go | 3 + internal/testing/testdata/testdata.go | 14 ++ pkg/assembler/backends/inmem/certifyBad.go | 24 ++- .../backends/inmem/certifyBad_test.go | 40 +++++ pkg/assembler/backends/inmem/certifyGood.go | 23 ++- .../backends/inmem/certifyGood_test.go | 40 +++++ pkg/assembler/backends/inmem/hasSBOM.go | 11 +- pkg/assembler/backends/inmem/hasSBOM_test.go | 34 ++++ pkg/assembler/clients/generated/operations.go | 156 ++++++++++++++++-- .../clients/operations/trees.graphql | 3 + .../graphql/examples/certify_bad.gql | 1 + .../graphql/examples/certify_good.gql | 1 + pkg/assembler/graphql/examples/has_sbom.gql | 1 + .../graphql/generated/artifact.generated.go | 6 + .../graphql/generated/certifyBad.generated.go | 72 +++++++- .../generated/certifyGood.generated.go | 72 +++++++- .../graphql/generated/hasSBOM.generated.go | 72 +++++++- .../graphql/generated/root_.generated.go | 43 +++++ pkg/assembler/graphql/model/nodes.go | 43 +++-- .../graphql/resolvers/certifyBad.resolvers.go | 10 ++ .../resolvers/certifyBad.resolvers_test.go | 5 + .../resolvers/certifyGood.resolvers.go | 10 ++ .../resolvers/certifyGood.resolvers_test.go | 2 + .../graphql/resolvers/hasSBOM.resolvers.go | 10 ++ .../resolvers/hasSBOM.resolvers_test.go | 9 +- .../graphql/schema/certifyBad.graphql | 6 + .../graphql/schema/certifyGood.graphql | 6 + pkg/assembler/graphql/schema/hasSBOM.graphql | 7 + pkg/guacanalytics/patchPlanning_test.go | 2 + pkg/ingestor/parser/common/helpers.go | 4 +- .../parser/cyclonedx/parser_cyclonedx.go | 18 +- pkg/ingestor/parser/spdx/parse_spdx.go | 7 +- 33 files changed, 704 insertions(+), 56 deletions(-) diff --git a/cmd/guacgql/cmd/ingest.go b/cmd/guacgql/cmd/ingest.go index a460032051..2ab2b36676 100644 --- a/cmd/guacgql/cmd/ingest.go +++ b/cmd/guacgql/cmd/ingest.go @@ -1965,6 +1965,7 @@ func ingestHashEqual(ctx context.Context, client graphql.Client) { func ingestHasSBOM(ctx context.Context, client graphql.Client) { logger := logging.FromContext(ctx) + tm, _ := time.Parse(time.RFC3339, "2022-11-21T17:45:50.52Z") opensslNs := "openssl.org" opensslVersion := "3.0.3" ingestHasSBOM := []struct { @@ -1989,6 +1990,7 @@ func ingestHasSBOM(ctx context.Context, client graphql.Client) { DownloadLocation: "uri: download location of the SBOM", Origin: "Demo ingestion", Collector: "Demo ingestion", + KnownSince: tm, }, }, { @@ -2004,6 +2006,7 @@ func ingestHasSBOM(ctx context.Context, client graphql.Client) { DownloadLocation: "uri: download location of the SBOM", Origin: "Demo ingestion", Collector: "Demo ingestion", + KnownSince: tm, }, }, { @@ -2022,6 +2025,7 @@ func ingestHasSBOM(ctx context.Context, client graphql.Client) { DownloadLocation: "uri: download location of the SBOM", Origin: "Demo ingestion", Collector: "Demo ingestion", + KnownSince: tm, }, }, { @@ -2037,6 +2041,7 @@ func ingestHasSBOM(ctx context.Context, client graphql.Client) { DownloadLocation: "uri: download location of the SBOM", Origin: "Demo ingestion", Collector: "Demo ingestion", + KnownSince: tm, }, }, } diff --git a/cmd/guacone/cmd/certify.go b/cmd/guacone/cmd/certify.go index 595524cef4..76e8b8305c 100644 --- a/cmd/guacone/cmd/certify.go +++ b/cmd/guacone/cmd/certify.go @@ -20,6 +20,7 @@ import ( "fmt" "os" "strings" + "time" "github.com/guacsec/guac/pkg/assembler" model "github.com/guacsec/guac/pkg/assembler/clients/generated" @@ -122,6 +123,7 @@ var certifyCmd = &cobra.Command{ Justification: opts.justification, Origin: "GUAC Certify CLI", Collector: "GUAC", + KnownSince: time.Now().UTC(), } preds.CertifyGood = append(preds.CertifyGood, *certifyGood) } else { @@ -140,6 +142,7 @@ var certifyCmd = &cobra.Command{ Justification: opts.justification, Origin: "GUAC Certify CLI", Collector: "GUAC", + KnownSince: time.Now().UTC(), } preds.CertifyBad = append(preds.CertifyBad, *certifyBad) } diff --git a/internal/testing/testdata/testdata.go b/internal/testing/testdata/testdata.go index be340d8ef1..f5feaee027 100644 --- a/internal/testing/testdata/testdata.go +++ b/internal/testing/testdata/testdata.go @@ -725,6 +725,8 @@ var ( }, } + spdxTime, _ = time.Parse(time.RFC3339, "2022-09-24T17:27:55.556104Z") + SpdxHasSBOM = []assembler.HasSBOMIngest{ { Pkg: topLevelPack, @@ -733,6 +735,7 @@ var ( Algorithm: "sha256", Digest: "8b5e8212cae084f92ff91f8625a50ea1070738cfc68ecca08bf04d64f64b9feb", DownloadLocation: "TestSource", + KnownSince: spdxTime, }, }, } @@ -852,6 +855,8 @@ var ( }, } + cdxTime, _ = time.Parse(time.RFC3339, "2022-10-08T10:01:23-04:00") + CdxHasSBOM = []assembler.HasSBOMIngest{ { Pkg: cdxTopLevelPack, @@ -860,6 +865,7 @@ var ( Algorithm: "sha256", Digest: "01942b5eefd3c15b50318c66d8d16627be573197c877e8a286a8cb12de7939cb", DownloadLocation: "TestSource", + KnownSince: cdxTime, }, }, } @@ -942,6 +948,8 @@ var ( }, } + cdxQuarkusTime, _ = time.Parse(time.RFC3339, "2022-11-09T11:14:31Z") + CdxQuarkusHasSBOM = []assembler.HasSBOMIngest{ { Pkg: cdxTopQuarkusPack, @@ -950,6 +958,7 @@ var ( Algorithm: "sha256", Digest: "036a9f51468f5ce6eec7c310583164ed0ab9f58d7c03380a3fe19d420609e3de", DownloadLocation: "TestSource", + KnownSince: cdxQuarkusTime, }, }, } @@ -977,6 +986,8 @@ var ( }, } + cdxNpmTime, _ = time.Parse(time.RFC3339, "2022-11-22T17:14:57Z") + CdxNpmHasSBOM = []assembler.HasSBOMIngest{ { Pkg: cdxWebAppPackage, @@ -985,6 +996,7 @@ var ( Algorithm: "sha256", Digest: "35363f03c80f26a88db6f2400771bdcc6624bb7b61b96da8503be0f757605fde", DownloadLocation: "TestSource", + KnownSince: cdxNpmTime, }, }, } @@ -995,6 +1007,7 @@ var ( } quarkusParentPackage, _ = asmhelpers.PurlToPkg("pkg:maven/io.quarkus/quarkus-parent@999-SNAPSHOT?type=pom") + quarkusTime, _ = time.Parse(time.RFC3339, "2023-02-16T21:52:02Z") quarkusParentPackageHasSBOM = []assembler.HasSBOMIngest{ { @@ -1004,6 +1017,7 @@ var ( Algorithm: "sha256", Digest: "fcd4d1f9c83c274fbc2dabdca4e7de749b23fab1aa15dc2854880a13479fa74e", DownloadLocation: "TestSource", + KnownSince: quarkusTime, }, }, } diff --git a/pkg/assembler/backends/inmem/certifyBad.go b/pkg/assembler/backends/inmem/certifyBad.go index 8247c3ea05..92924388f4 100644 --- a/pkg/assembler/backends/inmem/certifyBad.go +++ b/pkg/assembler/backends/inmem/certifyBad.go @@ -18,12 +18,15 @@ package inmem import ( "context" "strconv" + "time" "github.com/vektah/gqlparser/v2/gqlerror" "github.com/guacsec/guac/pkg/assembler/graphql/model" ) +// TODO: update the other backends to handle the new timestamp fields beacuse of: https://github.com/guacsec/guac/pull/1338/files#r1343080326 + // Internal data: link that a package/source/artifact is bad type badList []*badLink type badLink struct { @@ -34,6 +37,7 @@ type badLink struct { justification string origin string collector string + knownSince time.Time } func (n *badLink) ID() uint32 { return n.id } @@ -157,7 +161,8 @@ func (c *demoClient) ingestCertifyBad(ctx context.Context, subject model.Package subjectMatch = true } if subjectMatch && certifyBad.Justification == v.justification && - certifyBad.Origin == v.origin && certifyBad.Collector == v.collector { + certifyBad.Origin == v.origin && certifyBad.Collector == v.collector && + certifyBad.KnownSince.Equal(v.knownSince) { collectedCertifyBadLink = *v duplicate = true @@ -180,6 +185,7 @@ func (c *demoClient) ingestCertifyBad(ctx context.Context, subject model.Package justification: certifyBad.Justification, origin: certifyBad.Origin, collector: certifyBad.Collector, + knownSince: certifyBad.KnownSince.UTC(), } c.index[collectedCertifyBadLink.id] = &collectedCertifyBadLink c.certifyBads = append(c.certifyBads, &collectedCertifyBadLink) @@ -282,14 +288,13 @@ func (c *demoClient) addCBIfMatch(out []*model.CertifyBad, filter *model.CertifyBadSpec, link *badLink) ( []*model.CertifyBad, error) { - if filter != nil && noMatch(filter.Justification, link.justification) { - return out, nil - } - if filter != nil && noMatch(filter.Collector, link.collector) { - return out, nil - } - if filter != nil && noMatch(filter.Origin, link.origin) { - return out, nil + if filter != nil { + if noMatch(filter.Justification, link.justification) || + noMatch(filter.Collector, link.collector) || + noMatch(filter.Origin, link.origin) || + (filter.KnownSince != nil && filter.KnownSince.After(link.knownSince)) { + return out, nil + } } foundCertifyBad, err := c.buildCertifyBad(link, filter, false) @@ -379,6 +384,7 @@ func (c *demoClient) buildCertifyBad(link *badLink, filter *model.CertifyBadSpec Justification: link.justification, Origin: link.origin, Collector: link.collector, + KnownSince: link.knownSince.UTC(), } return &certifyBad, nil } diff --git a/pkg/assembler/backends/inmem/certifyBad_test.go b/pkg/assembler/backends/inmem/certifyBad_test.go index 4d5dd1cf74..87cd3ccac5 100644 --- a/pkg/assembler/backends/inmem/certifyBad_test.go +++ b/pkg/assembler/backends/inmem/certifyBad_test.go @@ -19,6 +19,7 @@ import ( "context" "strings" "testing" + "time" "github.com/google/go-cmp/cmp" "github.com/guacsec/guac/internal/testing/ptrfrom" @@ -28,6 +29,8 @@ import ( ) func TestCertifyBad(t *testing.T) { + curTime := time.Now() + timeAfterOneSecond := curTime.Add(time.Second) type call struct { Sub model.PackageSourceOrArtifactInput Match *model.MatchFlags @@ -170,6 +173,43 @@ func TestCertifyBad(t *testing.T) { }, }, }, + { + Name: "Query on KnownSince", + InPkg: []*model.PkgInputSpec{p1}, + Calls: []call{ + { + Sub: model.PackageSourceOrArtifactInput{ + Package: p1, + }, + Match: &model.MatchFlags{ + Pkg: model.PkgMatchTypeSpecificVersion, + }, + CB: &model.CertifyBadInputSpec{ + KnownSince: timeAfterOneSecond, + }, + }, + { + Sub: model.PackageSourceOrArtifactInput{ + Package: p1, + }, + Match: &model.MatchFlags{ + Pkg: model.PkgMatchTypeSpecificVersion, + }, + CB: &model.CertifyBadInputSpec{ + KnownSince: curTime, + }, + }, + }, + Query: &model.CertifyBadSpec{ + KnownSince: &timeAfterOneSecond, + }, + ExpCB: []*model.CertifyBad{ + { + Subject: p1out, + KnownSince: timeAfterOneSecond, + }, + }, + }, { Name: "Query on Package", InPkg: []*model.PkgInputSpec{p1, p2}, diff --git a/pkg/assembler/backends/inmem/certifyGood.go b/pkg/assembler/backends/inmem/certifyGood.go index 8b56072f29..1d8b93a84e 100644 --- a/pkg/assembler/backends/inmem/certifyGood.go +++ b/pkg/assembler/backends/inmem/certifyGood.go @@ -18,6 +18,7 @@ package inmem import ( "context" "strconv" + "time" "github.com/vektah/gqlparser/v2/gqlerror" @@ -34,6 +35,7 @@ type goodLink struct { justification string origin string collector string + knownSince time.Time } func (n *goodLink) ID() uint32 { return n.id } @@ -158,7 +160,8 @@ func (c *demoClient) ingestCertifyGood(ctx context.Context, subject model.Packag subjectMatch = true } if subjectMatch && certifyGood.Justification == v.justification && - certifyGood.Origin == v.origin && certifyGood.Collector == v.collector { + certifyGood.Origin == v.origin && certifyGood.Collector == v.collector && + certifyGood.KnownSince.Equal(v.knownSince) { collectedCertifyGoodLink = *v duplicate = true @@ -181,6 +184,7 @@ func (c *demoClient) ingestCertifyGood(ctx context.Context, subject model.Packag justification: certifyGood.Justification, origin: certifyGood.Origin, collector: certifyGood.Collector, + knownSince: certifyGood.KnownSince.UTC(), } c.index[collectedCertifyGoodLink.id] = &collectedCertifyGoodLink c.certifyGoods = append(c.certifyGoods, &collectedCertifyGoodLink) @@ -283,14 +287,14 @@ func (c *demoClient) addCGIfMatch(out []*model.CertifyGood, filter *model.CertifyGoodSpec, link *goodLink) ( []*model.CertifyGood, error) { - if filter != nil && noMatch(filter.Justification, link.justification) { - return out, nil - } - if filter != nil && noMatch(filter.Collector, link.collector) { - return out, nil - } - if filter != nil && noMatch(filter.Origin, link.origin) { - return out, nil + if filter != nil { + if noMatch(filter.Justification, link.justification) || + noMatch(filter.Collector, link.collector) || + noMatch(filter.Collector, link.collector) || + noMatch(filter.Origin, link.origin) || + filter.KnownSince != nil && filter.KnownSince.After(link.knownSince) { + return out, nil + } } foundCertifyGood, err := c.buildCertifyGood(link, filter, false) @@ -380,6 +384,7 @@ func (c *demoClient) buildCertifyGood(link *goodLink, filter *model.CertifyGoodS Justification: link.justification, Origin: link.origin, Collector: link.collector, + KnownSince: link.knownSince.UTC(), } return &certifyGood, nil } diff --git a/pkg/assembler/backends/inmem/certifyGood_test.go b/pkg/assembler/backends/inmem/certifyGood_test.go index 1ca8b73ffe..bebd34d28f 100644 --- a/pkg/assembler/backends/inmem/certifyGood_test.go +++ b/pkg/assembler/backends/inmem/certifyGood_test.go @@ -19,6 +19,7 @@ import ( "context" "strings" "testing" + "time" "github.com/google/go-cmp/cmp" "github.com/guacsec/guac/internal/testing/ptrfrom" @@ -28,6 +29,8 @@ import ( ) func TestCertifyGood(t *testing.T) { + curTime := time.Now() + timeAfterOneSecond := curTime.Add(time.Second) type call struct { Sub model.PackageSourceOrArtifactInput Match *model.MatchFlags @@ -170,6 +173,43 @@ func TestCertifyGood(t *testing.T) { }, }, }, + { + Name: "Query on KnownSince", + InPkg: []*model.PkgInputSpec{p1}, + Calls: []call{ + { + Sub: model.PackageSourceOrArtifactInput{ + Package: p1, + }, + Match: &model.MatchFlags{ + Pkg: model.PkgMatchTypeSpecificVersion, + }, + CG: &model.CertifyGoodInputSpec{ + KnownSince: timeAfterOneSecond, + }, + }, + { + Sub: model.PackageSourceOrArtifactInput{ + Package: p1, + }, + Match: &model.MatchFlags{ + Pkg: model.PkgMatchTypeSpecificVersion, + }, + CG: &model.CertifyGoodInputSpec{ + KnownSince: curTime, + }, + }, + }, + Query: &model.CertifyGoodSpec{ + KnownSince: &timeAfterOneSecond, + }, + ExpCG: []*model.CertifyGood{ + { + Subject: p1out, + KnownSince: timeAfterOneSecond, + }, + }, + }, { Name: "Query on Package", InPkg: []*model.PkgInputSpec{p1, p2}, diff --git a/pkg/assembler/backends/inmem/hasSBOM.go b/pkg/assembler/backends/inmem/hasSBOM.go index 0a7fb52236..0b0f786631 100644 --- a/pkg/assembler/backends/inmem/hasSBOM.go +++ b/pkg/assembler/backends/inmem/hasSBOM.go @@ -19,6 +19,7 @@ import ( "context" "strconv" "strings" + "time" "github.com/vektah/gqlparser/v2/gqlerror" @@ -36,6 +37,7 @@ type hasSBOMStruct struct { downloadLocation string origin string collector string + knownSince time.Time } func (n *hasSBOMStruct) ID() uint32 { return n.id } @@ -131,7 +133,8 @@ func (c *demoClient) ingestHasSbom(ctx context.Context, subject model.PackageOrA h.digest == digest && h.downloadLocation == input.DownloadLocation && h.origin == input.Origin && - h.collector == input.Collector { + h.collector == input.Collector && + input.KnownSince.Equal(h.knownSince) { return c.convHasSBOM(h) } } @@ -153,6 +156,7 @@ func (c *demoClient) ingestHasSbom(ctx context.Context, subject model.PackageOrA downloadLocation: input.DownloadLocation, origin: input.Origin, collector: input.Collector, + knownSince: input.KnownSince.UTC(), } c.index[h.id] = h c.hasSBOMs = append(c.hasSBOMs, h) @@ -173,6 +177,7 @@ func (c *demoClient) convHasSBOM(in *hasSBOMStruct) (*model.HasSbom, error) { DownloadLocation: in.downloadLocation, Origin: in.origin, Collector: in.collector, + KnownSince: in.knownSince.UTC(), } if in.pkg != 0 { p, err := c.buildPackageResponse(in.pkg, nil) @@ -260,6 +265,7 @@ func (c *demoClient) HasSBOM(ctx context.Context, filter *model.HasSBOMSpec) ([] } } } + return out, nil } @@ -273,7 +279,8 @@ func (c *demoClient) addHasSBOMIfMatch(out []*model.HasSbom, noMatch(toLower(filter.Digest), link.digest) || noMatch(filter.DownloadLocation, link.downloadLocation) || noMatch(filter.Origin, link.origin) || - noMatch(filter.Collector, link.collector) { + noMatch(filter.Collector, link.collector) || + (filter.KnownSince != nil && filter.KnownSince.After(link.knownSince)) { return out, nil } if filter.Subject != nil { diff --git a/pkg/assembler/backends/inmem/hasSBOM_test.go b/pkg/assembler/backends/inmem/hasSBOM_test.go index 85a138d395..76472d24fa 100644 --- a/pkg/assembler/backends/inmem/hasSBOM_test.go +++ b/pkg/assembler/backends/inmem/hasSBOM_test.go @@ -19,6 +19,7 @@ import ( "context" "strings" "testing" + "time" "github.com/google/go-cmp/cmp" "github.com/guacsec/guac/internal/testing/ptrfrom" @@ -28,6 +29,8 @@ import ( ) func TestHasSBOM(t *testing.T) { + curTime := time.Now() + timeAfterOneSecond := curTime.Add(time.Second) type call struct { Sub model.PackageOrArtifactInput HS *model.HasSBOMInputSpec @@ -127,6 +130,37 @@ func TestHasSBOM(t *testing.T) { }, }, }, + { + Name: "Query on KnownSince", + InPkg: []*model.PkgInputSpec{p1}, + Calls: []call{ + { + Sub: model.PackageOrArtifactInput{ + Package: p1, + }, + HS: &model.HasSBOMInputSpec{ + KnownSince: timeAfterOneSecond, + }, + }, + { + Sub: model.PackageOrArtifactInput{ + Package: p1, + }, + HS: &model.HasSBOMInputSpec{ + KnownSince: curTime, + }, + }, + }, + Query: &model.HasSBOMSpec{ + KnownSince: &timeAfterOneSecond, + }, + ExpHS: []*model.HasSbom{ + { + Subject: p1out, + KnownSince: timeAfterOneSecond, + }, + }, + }, { Name: "Query on Package", InPkg: []*model.PkgInputSpec{p1, p2}, diff --git a/pkg/assembler/clients/generated/operations.go b/pkg/assembler/clients/generated/operations.go index 0081dc08dd..042595f677 100644 --- a/pkg/assembler/clients/generated/operations.go +++ b/pkg/assembler/clients/generated/operations.go @@ -70,6 +70,7 @@ func (v *AllBuilderTree) GetUri() string { return v.Uri } type AllCertifyBad struct { Id string `json:"id"` Justification string `json:"justification"` + KnownSince time.Time `json:"knownSince"` Subject AllCertifyBadSubjectPackageSourceOrArtifact `json:"-"` Origin string `json:"origin"` Collector string `json:"collector"` @@ -81,6 +82,9 @@ func (v *AllCertifyBad) GetId() string { return v.Id } // GetJustification returns AllCertifyBad.Justification, and is useful for accessing the field via an interface. func (v *AllCertifyBad) GetJustification() string { return v.Justification } +// GetKnownSince returns AllCertifyBad.KnownSince, and is useful for accessing the field via an interface. +func (v *AllCertifyBad) GetKnownSince() time.Time { return v.KnownSince } + // GetSubject returns AllCertifyBad.Subject, and is useful for accessing the field via an interface. func (v *AllCertifyBad) GetSubject() AllCertifyBadSubjectPackageSourceOrArtifact { return v.Subject } @@ -128,6 +132,8 @@ type __premarshalAllCertifyBad struct { Justification string `json:"justification"` + KnownSince time.Time `json:"knownSince"` + Subject json.RawMessage `json:"subject"` Origin string `json:"origin"` @@ -148,6 +154,7 @@ func (v *AllCertifyBad) __premarshalJSON() (*__premarshalAllCertifyBad, error) { retval.Id = v.Id retval.Justification = v.Justification + retval.KnownSince = v.KnownSince { dst := &retval.Subject @@ -539,6 +546,7 @@ func (v *AllCertifyBadSubjectSource) __premarshalJSON() (*__premarshalAllCertify type AllCertifyGood struct { Id string `json:"id"` Justification string `json:"justification"` + KnownSince time.Time `json:"knownSince"` Subject AllCertifyGoodSubjectPackageSourceOrArtifact `json:"-"` Origin string `json:"origin"` Collector string `json:"collector"` @@ -550,6 +558,9 @@ func (v *AllCertifyGood) GetId() string { return v.Id } // GetJustification returns AllCertifyGood.Justification, and is useful for accessing the field via an interface. func (v *AllCertifyGood) GetJustification() string { return v.Justification } +// GetKnownSince returns AllCertifyGood.KnownSince, and is useful for accessing the field via an interface. +func (v *AllCertifyGood) GetKnownSince() time.Time { return v.KnownSince } + // GetSubject returns AllCertifyGood.Subject, and is useful for accessing the field via an interface. func (v *AllCertifyGood) GetSubject() AllCertifyGoodSubjectPackageSourceOrArtifact { return v.Subject } @@ -597,6 +608,8 @@ type __premarshalAllCertifyGood struct { Justification string `json:"justification"` + KnownSince time.Time `json:"knownSince"` + Subject json.RawMessage `json:"subject"` Origin string `json:"origin"` @@ -617,6 +630,7 @@ func (v *AllCertifyGood) __premarshalJSON() (*__premarshalAllCertifyGood, error) retval.Id = v.Id retval.Justification = v.Justification + retval.KnownSince = v.KnownSince { dst := &retval.Subject @@ -3060,6 +3074,8 @@ type AllHasSBOMTree struct { Origin string `json:"origin"` // GUAC collector for the document Collector string `json:"collector"` + // Timestamp for SBOM creation + KnownSince time.Time `json:"knownSince"` } // GetId returns AllHasSBOMTree.Id, and is useful for accessing the field via an interface. @@ -3086,6 +3102,9 @@ func (v *AllHasSBOMTree) GetOrigin() string { return v.Origin } // GetCollector returns AllHasSBOMTree.Collector, and is useful for accessing the field via an interface. func (v *AllHasSBOMTree) GetCollector() string { return v.Collector } +// GetKnownSince returns AllHasSBOMTree.KnownSince, and is useful for accessing the field via an interface. +func (v *AllHasSBOMTree) GetKnownSince() time.Time { return v.KnownSince } + func (v *AllHasSBOMTree) UnmarshalJSON(b []byte) error { if string(b) == "null" { @@ -3135,6 +3154,8 @@ type __premarshalAllHasSBOMTree struct { Origin string `json:"origin"` Collector string `json:"collector"` + + KnownSince time.Time `json:"knownSince"` } func (v *AllHasSBOMTree) MarshalJSON() ([]byte, error) { @@ -3167,6 +3188,7 @@ func (v *AllHasSBOMTree) __premarshalJSON() (*__premarshalAllHasSBOMTree, error) retval.DownloadLocation = v.DownloadLocation retval.Origin = v.Origin retval.Collector = v.Collector + retval.KnownSince = v.KnownSince return &retval, nil } @@ -6060,9 +6082,10 @@ func (v *CertifyBadArtifactsResponse) GetIngestCertifyBads() []string { return v // CertifyBadInputSpec represents the mutation input to ingest a CertifyBad // evidence. type CertifyBadInputSpec struct { - Justification string `json:"justification"` - Origin string `json:"origin"` - Collector string `json:"collector"` + Justification string `json:"justification"` + Origin string `json:"origin"` + Collector string `json:"collector"` + KnownSince time.Time `json:"knownSince"` } // GetJustification returns CertifyBadInputSpec.Justification, and is useful for accessing the field via an interface. @@ -6074,6 +6097,9 @@ func (v *CertifyBadInputSpec) GetOrigin() string { return v.Origin } // GetCollector returns CertifyBadInputSpec.Collector, and is useful for accessing the field via an interface. func (v *CertifyBadInputSpec) GetCollector() string { return v.Collector } +// GetKnownSince returns CertifyBadInputSpec.KnownSince, and is useful for accessing the field via an interface. +func (v *CertifyBadInputSpec) GetKnownSince() time.Time { return v.KnownSince } + // CertifyBadPkgResponse is returned by CertifyBadPkg on success. type CertifyBadPkgResponse struct { // Adds a certification that a package, source or artifact is considered bad. The returned ID can be empty string. @@ -6101,12 +6127,16 @@ func (v *CertifyBadPkgsResponse) GetIngestCertifyBads() []string { return v.Inge // // If a source is specified in the subject filter, then it must specify a name, // and optionally a tag and a commit. +// +// If KnownSince is specified, the returned value will be after or equal to the specified time. +// Any nodes time that is before KnownSince is excluded. type CertifyBadSpec struct { Id *string `json:"id"` Subject *PackageSourceOrArtifactSpec `json:"subject"` Justification *string `json:"justification"` Origin *string `json:"origin"` Collector *string `json:"collector"` + KnownSince *time.Time `json:"knownSince"` } // GetId returns CertifyBadSpec.Id, and is useful for accessing the field via an interface. @@ -6124,6 +6154,9 @@ func (v *CertifyBadSpec) GetOrigin() *string { return v.Origin } // GetCollector returns CertifyBadSpec.Collector, and is useful for accessing the field via an interface. func (v *CertifyBadSpec) GetCollector() *string { return v.Collector } +// GetKnownSince returns CertifyBadSpec.KnownSince, and is useful for accessing the field via an interface. +func (v *CertifyBadSpec) GetKnownSince() *time.Time { return v.KnownSince } + // CertifyBadSrcResponse is returned by CertifyBadSrc on success. type CertifyBadSrcResponse struct { // Adds a certification that a package, source or artifact is considered bad. The returned ID can be empty string. @@ -6166,6 +6199,9 @@ func (v *CertifyBadsCertifyBad) GetId() string { return v.AllCertifyBad.Id } // GetJustification returns CertifyBadsCertifyBad.Justification, and is useful for accessing the field via an interface. func (v *CertifyBadsCertifyBad) GetJustification() string { return v.AllCertifyBad.Justification } +// GetKnownSince returns CertifyBadsCertifyBad.KnownSince, and is useful for accessing the field via an interface. +func (v *CertifyBadsCertifyBad) GetKnownSince() time.Time { return v.AllCertifyBad.KnownSince } + // GetSubject returns CertifyBadsCertifyBad.Subject, and is useful for accessing the field via an interface. func (v *CertifyBadsCertifyBad) GetSubject() AllCertifyBadSubjectPackageSourceOrArtifact { return v.AllCertifyBad.Subject @@ -6207,6 +6243,8 @@ type __premarshalCertifyBadsCertifyBad struct { Justification string `json:"justification"` + KnownSince time.Time `json:"knownSince"` + Subject json.RawMessage `json:"subject"` Origin string `json:"origin"` @@ -6227,6 +6265,7 @@ func (v *CertifyBadsCertifyBad) __premarshalJSON() (*__premarshalCertifyBadsCert retval.Id = v.AllCertifyBad.Id retval.Justification = v.AllCertifyBad.Justification + retval.KnownSince = v.AllCertifyBad.KnownSince { dst := &retval.Subject @@ -6273,9 +6312,10 @@ func (v *CertifyGoodArtifactsResponse) GetIngestCertifyGoods() []string { return // CertifyGoodInputSpec represents the mutation input to ingest a CertifyGood evidence. type CertifyGoodInputSpec struct { - Justification string `json:"justification"` - Origin string `json:"origin"` - Collector string `json:"collector"` + Justification string `json:"justification"` + Origin string `json:"origin"` + Collector string `json:"collector"` + KnownSince time.Time `json:"knownSince"` } // GetJustification returns CertifyGoodInputSpec.Justification, and is useful for accessing the field via an interface. @@ -6287,6 +6327,9 @@ func (v *CertifyGoodInputSpec) GetOrigin() string { return v.Origin } // GetCollector returns CertifyGoodInputSpec.Collector, and is useful for accessing the field via an interface. func (v *CertifyGoodInputSpec) GetCollector() string { return v.Collector } +// GetKnownSince returns CertifyGoodInputSpec.KnownSince, and is useful for accessing the field via an interface. +func (v *CertifyGoodInputSpec) GetKnownSince() time.Time { return v.KnownSince } + // CertifyGoodPkgResponse is returned by CertifyGoodPkg on success. type CertifyGoodPkgResponse struct { // Adds a certification that a package, source or artifact is considered good. The returned ID can be empty string. @@ -7339,12 +7382,13 @@ func (v *HasSBOMArtifactsResponse) GetIngestHasSBOMs() []string { return v.Inges // HasSBOMInputSpec is the same as HasSBOM but for mutation input. type HasSBOMInputSpec struct { - Uri string `json:"uri"` - Algorithm string `json:"algorithm"` - Digest string `json:"digest"` - DownloadLocation string `json:"downloadLocation"` - Origin string `json:"origin"` - Collector string `json:"collector"` + Uri string `json:"uri"` + Algorithm string `json:"algorithm"` + Digest string `json:"digest"` + DownloadLocation string `json:"downloadLocation"` + Origin string `json:"origin"` + Collector string `json:"collector"` + KnownSince time.Time `json:"knownSince"` } // GetUri returns HasSBOMInputSpec.Uri, and is useful for accessing the field via an interface. @@ -7365,6 +7409,9 @@ func (v *HasSBOMInputSpec) GetOrigin() string { return v.Origin } // GetCollector returns HasSBOMInputSpec.Collector, and is useful for accessing the field via an interface. func (v *HasSBOMInputSpec) GetCollector() string { return v.Collector } +// GetKnownSince returns HasSBOMInputSpec.KnownSince, and is useful for accessing the field via an interface. +func (v *HasSBOMInputSpec) GetKnownSince() time.Time { return v.KnownSince } + // HasSBOMPkgResponse is returned by HasSBOMPkg on success. type HasSBOMPkgResponse struct { // Certifies that a package or artifact has an SBOM. The returned ID can be empty string. @@ -8027,6 +8074,9 @@ func (v *NeighborsNeighborsCertifyBad) GetJustification() string { return v.AllCertifyBad.Justification } +// GetKnownSince returns NeighborsNeighborsCertifyBad.KnownSince, and is useful for accessing the field via an interface. +func (v *NeighborsNeighborsCertifyBad) GetKnownSince() time.Time { return v.AllCertifyBad.KnownSince } + // GetSubject returns NeighborsNeighborsCertifyBad.Subject, and is useful for accessing the field via an interface. func (v *NeighborsNeighborsCertifyBad) GetSubject() AllCertifyBadSubjectPackageSourceOrArtifact { return v.AllCertifyBad.Subject @@ -8070,6 +8120,8 @@ type __premarshalNeighborsNeighborsCertifyBad struct { Justification string `json:"justification"` + KnownSince time.Time `json:"knownSince"` + Subject json.RawMessage `json:"subject"` Origin string `json:"origin"` @@ -8091,6 +8143,7 @@ func (v *NeighborsNeighborsCertifyBad) __premarshalJSON() (*__premarshalNeighbor retval.Typename = v.Typename retval.Id = v.AllCertifyBad.Id retval.Justification = v.AllCertifyBad.Justification + retval.KnownSince = v.AllCertifyBad.KnownSince { dst := &retval.Subject @@ -8138,6 +8191,9 @@ func (v *NeighborsNeighborsCertifyGood) GetJustification() string { return v.AllCertifyGood.Justification } +// GetKnownSince returns NeighborsNeighborsCertifyGood.KnownSince, and is useful for accessing the field via an interface. +func (v *NeighborsNeighborsCertifyGood) GetKnownSince() time.Time { return v.AllCertifyGood.KnownSince } + // GetSubject returns NeighborsNeighborsCertifyGood.Subject, and is useful for accessing the field via an interface. func (v *NeighborsNeighborsCertifyGood) GetSubject() AllCertifyGoodSubjectPackageSourceOrArtifact { return v.AllCertifyGood.Subject @@ -8181,6 +8237,8 @@ type __premarshalNeighborsNeighborsCertifyGood struct { Justification string `json:"justification"` + KnownSince time.Time `json:"knownSince"` + Subject json.RawMessage `json:"subject"` Origin string `json:"origin"` @@ -8202,6 +8260,7 @@ func (v *NeighborsNeighborsCertifyGood) __premarshalJSON() (*__premarshalNeighbo retval.Typename = v.Typename retval.Id = v.AllCertifyGood.Id retval.Justification = v.AllCertifyGood.Justification + retval.KnownSince = v.AllCertifyGood.KnownSince { dst := &retval.Subject @@ -8867,6 +8926,9 @@ func (v *NeighborsNeighborsHasSBOM) GetOrigin() string { return v.AllHasSBOMTree // GetCollector returns NeighborsNeighborsHasSBOM.Collector, and is useful for accessing the field via an interface. func (v *NeighborsNeighborsHasSBOM) GetCollector() string { return v.AllHasSBOMTree.Collector } +// GetKnownSince returns NeighborsNeighborsHasSBOM.KnownSince, and is useful for accessing the field via an interface. +func (v *NeighborsNeighborsHasSBOM) GetKnownSince() time.Time { return v.AllHasSBOMTree.KnownSince } + func (v *NeighborsNeighborsHasSBOM) UnmarshalJSON(b []byte) error { if string(b) == "null" { @@ -8910,6 +8972,8 @@ type __premarshalNeighborsNeighborsHasSBOM struct { Origin string `json:"origin"` Collector string `json:"collector"` + + KnownSince time.Time `json:"knownSince"` } func (v *NeighborsNeighborsHasSBOM) MarshalJSON() ([]byte, error) { @@ -8943,6 +9007,7 @@ func (v *NeighborsNeighborsHasSBOM) __premarshalJSON() (*__premarshalNeighborsNe retval.DownloadLocation = v.AllHasSBOMTree.DownloadLocation retval.Origin = v.AllHasSBOMTree.Origin retval.Collector = v.AllHasSBOMTree.Collector + retval.KnownSince = v.AllHasSBOMTree.KnownSince return &retval, nil } @@ -11427,6 +11492,9 @@ func (v *NodeNodeCertifyBad) GetId() string { return v.AllCertifyBad.Id } // GetJustification returns NodeNodeCertifyBad.Justification, and is useful for accessing the field via an interface. func (v *NodeNodeCertifyBad) GetJustification() string { return v.AllCertifyBad.Justification } +// GetKnownSince returns NodeNodeCertifyBad.KnownSince, and is useful for accessing the field via an interface. +func (v *NodeNodeCertifyBad) GetKnownSince() time.Time { return v.AllCertifyBad.KnownSince } + // GetSubject returns NodeNodeCertifyBad.Subject, and is useful for accessing the field via an interface. func (v *NodeNodeCertifyBad) GetSubject() AllCertifyBadSubjectPackageSourceOrArtifact { return v.AllCertifyBad.Subject @@ -11470,6 +11538,8 @@ type __premarshalNodeNodeCertifyBad struct { Justification string `json:"justification"` + KnownSince time.Time `json:"knownSince"` + Subject json.RawMessage `json:"subject"` Origin string `json:"origin"` @@ -11491,6 +11561,7 @@ func (v *NodeNodeCertifyBad) __premarshalJSON() (*__premarshalNodeNodeCertifyBad retval.Typename = v.Typename retval.Id = v.AllCertifyBad.Id retval.Justification = v.AllCertifyBad.Justification + retval.KnownSince = v.AllCertifyBad.KnownSince { dst := &retval.Subject @@ -11536,6 +11607,9 @@ func (v *NodeNodeCertifyGood) GetId() string { return v.AllCertifyGood.Id } // GetJustification returns NodeNodeCertifyGood.Justification, and is useful for accessing the field via an interface. func (v *NodeNodeCertifyGood) GetJustification() string { return v.AllCertifyGood.Justification } +// GetKnownSince returns NodeNodeCertifyGood.KnownSince, and is useful for accessing the field via an interface. +func (v *NodeNodeCertifyGood) GetKnownSince() time.Time { return v.AllCertifyGood.KnownSince } + // GetSubject returns NodeNodeCertifyGood.Subject, and is useful for accessing the field via an interface. func (v *NodeNodeCertifyGood) GetSubject() AllCertifyGoodSubjectPackageSourceOrArtifact { return v.AllCertifyGood.Subject @@ -11579,6 +11653,8 @@ type __premarshalNodeNodeCertifyGood struct { Justification string `json:"justification"` + KnownSince time.Time `json:"knownSince"` + Subject json.RawMessage `json:"subject"` Origin string `json:"origin"` @@ -11600,6 +11676,7 @@ func (v *NodeNodeCertifyGood) __premarshalJSON() (*__premarshalNodeNodeCertifyGo retval.Typename = v.Typename retval.Id = v.AllCertifyGood.Id retval.Justification = v.AllCertifyGood.Justification + retval.KnownSince = v.AllCertifyGood.KnownSince { dst := &retval.Subject @@ -12247,6 +12324,9 @@ func (v *NodeNodeHasSBOM) GetOrigin() string { return v.AllHasSBOMTree.Origin } // GetCollector returns NodeNodeHasSBOM.Collector, and is useful for accessing the field via an interface. func (v *NodeNodeHasSBOM) GetCollector() string { return v.AllHasSBOMTree.Collector } +// GetKnownSince returns NodeNodeHasSBOM.KnownSince, and is useful for accessing the field via an interface. +func (v *NodeNodeHasSBOM) GetKnownSince() time.Time { return v.AllHasSBOMTree.KnownSince } + func (v *NodeNodeHasSBOM) UnmarshalJSON(b []byte) error { if string(b) == "null" { @@ -12290,6 +12370,8 @@ type __premarshalNodeNodeHasSBOM struct { Origin string `json:"origin"` Collector string `json:"collector"` + + KnownSince time.Time `json:"knownSince"` } func (v *NodeNodeHasSBOM) MarshalJSON() ([]byte, error) { @@ -12323,6 +12405,7 @@ func (v *NodeNodeHasSBOM) __premarshalJSON() (*__premarshalNodeNodeHasSBOM, erro retval.DownloadLocation = v.AllHasSBOMTree.DownloadLocation retval.Origin = v.AllHasSBOMTree.Origin retval.Collector = v.AllHasSBOMTree.Collector + retval.KnownSince = v.AllHasSBOMTree.KnownSince return &retval, nil } @@ -13875,6 +13958,9 @@ func (v *NodesNodesCertifyBad) GetId() string { return v.AllCertifyBad.Id } // GetJustification returns NodesNodesCertifyBad.Justification, and is useful for accessing the field via an interface. func (v *NodesNodesCertifyBad) GetJustification() string { return v.AllCertifyBad.Justification } +// GetKnownSince returns NodesNodesCertifyBad.KnownSince, and is useful for accessing the field via an interface. +func (v *NodesNodesCertifyBad) GetKnownSince() time.Time { return v.AllCertifyBad.KnownSince } + // GetSubject returns NodesNodesCertifyBad.Subject, and is useful for accessing the field via an interface. func (v *NodesNodesCertifyBad) GetSubject() AllCertifyBadSubjectPackageSourceOrArtifact { return v.AllCertifyBad.Subject @@ -13918,6 +14004,8 @@ type __premarshalNodesNodesCertifyBad struct { Justification string `json:"justification"` + KnownSince time.Time `json:"knownSince"` + Subject json.RawMessage `json:"subject"` Origin string `json:"origin"` @@ -13939,6 +14027,7 @@ func (v *NodesNodesCertifyBad) __premarshalJSON() (*__premarshalNodesNodesCertif retval.Typename = v.Typename retval.Id = v.AllCertifyBad.Id retval.Justification = v.AllCertifyBad.Justification + retval.KnownSince = v.AllCertifyBad.KnownSince { dst := &retval.Subject @@ -13984,6 +14073,9 @@ func (v *NodesNodesCertifyGood) GetId() string { return v.AllCertifyGood.Id } // GetJustification returns NodesNodesCertifyGood.Justification, and is useful for accessing the field via an interface. func (v *NodesNodesCertifyGood) GetJustification() string { return v.AllCertifyGood.Justification } +// GetKnownSince returns NodesNodesCertifyGood.KnownSince, and is useful for accessing the field via an interface. +func (v *NodesNodesCertifyGood) GetKnownSince() time.Time { return v.AllCertifyGood.KnownSince } + // GetSubject returns NodesNodesCertifyGood.Subject, and is useful for accessing the field via an interface. func (v *NodesNodesCertifyGood) GetSubject() AllCertifyGoodSubjectPackageSourceOrArtifact { return v.AllCertifyGood.Subject @@ -14027,6 +14119,8 @@ type __premarshalNodesNodesCertifyGood struct { Justification string `json:"justification"` + KnownSince time.Time `json:"knownSince"` + Subject json.RawMessage `json:"subject"` Origin string `json:"origin"` @@ -14048,6 +14142,7 @@ func (v *NodesNodesCertifyGood) __premarshalJSON() (*__premarshalNodesNodesCerti retval.Typename = v.Typename retval.Id = v.AllCertifyGood.Id retval.Justification = v.AllCertifyGood.Justification + retval.KnownSince = v.AllCertifyGood.KnownSince { dst := &retval.Subject @@ -14697,6 +14792,9 @@ func (v *NodesNodesHasSBOM) GetOrigin() string { return v.AllHasSBOMTree.Origin // GetCollector returns NodesNodesHasSBOM.Collector, and is useful for accessing the field via an interface. func (v *NodesNodesHasSBOM) GetCollector() string { return v.AllHasSBOMTree.Collector } +// GetKnownSince returns NodesNodesHasSBOM.KnownSince, and is useful for accessing the field via an interface. +func (v *NodesNodesHasSBOM) GetKnownSince() time.Time { return v.AllHasSBOMTree.KnownSince } + func (v *NodesNodesHasSBOM) UnmarshalJSON(b []byte) error { if string(b) == "null" { @@ -14740,6 +14838,8 @@ type __premarshalNodesNodesHasSBOM struct { Origin string `json:"origin"` Collector string `json:"collector"` + + KnownSince time.Time `json:"knownSince"` } func (v *NodesNodesHasSBOM) MarshalJSON() ([]byte, error) { @@ -14773,6 +14873,7 @@ func (v *NodesNodesHasSBOM) __premarshalJSON() (*__premarshalNodesNodesHasSBOM, retval.DownloadLocation = v.AllHasSBOMTree.DownloadLocation retval.Origin = v.AllHasSBOMTree.Origin retval.Collector = v.AllHasSBOMTree.Collector + retval.KnownSince = v.AllHasSBOMTree.KnownSince return &retval, nil } @@ -17329,6 +17430,9 @@ func (v *PathPathCertifyBad) GetId() string { return v.AllCertifyBad.Id } // GetJustification returns PathPathCertifyBad.Justification, and is useful for accessing the field via an interface. func (v *PathPathCertifyBad) GetJustification() string { return v.AllCertifyBad.Justification } +// GetKnownSince returns PathPathCertifyBad.KnownSince, and is useful for accessing the field via an interface. +func (v *PathPathCertifyBad) GetKnownSince() time.Time { return v.AllCertifyBad.KnownSince } + // GetSubject returns PathPathCertifyBad.Subject, and is useful for accessing the field via an interface. func (v *PathPathCertifyBad) GetSubject() AllCertifyBadSubjectPackageSourceOrArtifact { return v.AllCertifyBad.Subject @@ -17372,6 +17476,8 @@ type __premarshalPathPathCertifyBad struct { Justification string `json:"justification"` + KnownSince time.Time `json:"knownSince"` + Subject json.RawMessage `json:"subject"` Origin string `json:"origin"` @@ -17393,6 +17499,7 @@ func (v *PathPathCertifyBad) __premarshalJSON() (*__premarshalPathPathCertifyBad retval.Typename = v.Typename retval.Id = v.AllCertifyBad.Id retval.Justification = v.AllCertifyBad.Justification + retval.KnownSince = v.AllCertifyBad.KnownSince { dst := &retval.Subject @@ -17438,6 +17545,9 @@ func (v *PathPathCertifyGood) GetId() string { return v.AllCertifyGood.Id } // GetJustification returns PathPathCertifyGood.Justification, and is useful for accessing the field via an interface. func (v *PathPathCertifyGood) GetJustification() string { return v.AllCertifyGood.Justification } +// GetKnownSince returns PathPathCertifyGood.KnownSince, and is useful for accessing the field via an interface. +func (v *PathPathCertifyGood) GetKnownSince() time.Time { return v.AllCertifyGood.KnownSince } + // GetSubject returns PathPathCertifyGood.Subject, and is useful for accessing the field via an interface. func (v *PathPathCertifyGood) GetSubject() AllCertifyGoodSubjectPackageSourceOrArtifact { return v.AllCertifyGood.Subject @@ -17481,6 +17591,8 @@ type __premarshalPathPathCertifyGood struct { Justification string `json:"justification"` + KnownSince time.Time `json:"knownSince"` + Subject json.RawMessage `json:"subject"` Origin string `json:"origin"` @@ -17502,6 +17614,7 @@ func (v *PathPathCertifyGood) __premarshalJSON() (*__premarshalPathPathCertifyGo retval.Typename = v.Typename retval.Id = v.AllCertifyGood.Id retval.Justification = v.AllCertifyGood.Justification + retval.KnownSince = v.AllCertifyGood.KnownSince { dst := &retval.Subject @@ -18149,6 +18262,9 @@ func (v *PathPathHasSBOM) GetOrigin() string { return v.AllHasSBOMTree.Origin } // GetCollector returns PathPathHasSBOM.Collector, and is useful for accessing the field via an interface. func (v *PathPathHasSBOM) GetCollector() string { return v.AllHasSBOMTree.Collector } +// GetKnownSince returns PathPathHasSBOM.KnownSince, and is useful for accessing the field via an interface. +func (v *PathPathHasSBOM) GetKnownSince() time.Time { return v.AllHasSBOMTree.KnownSince } + func (v *PathPathHasSBOM) UnmarshalJSON(b []byte) error { if string(b) == "null" { @@ -18192,6 +18308,8 @@ type __premarshalPathPathHasSBOM struct { Origin string `json:"origin"` Collector string `json:"collector"` + + KnownSince time.Time `json:"knownSince"` } func (v *PathPathHasSBOM) MarshalJSON() ([]byte, error) { @@ -18225,6 +18343,7 @@ func (v *PathPathHasSBOM) __premarshalJSON() (*__premarshalPathPathHasSBOM, erro retval.DownloadLocation = v.AllHasSBOMTree.DownloadLocation retval.Origin = v.AllHasSBOMTree.Origin retval.Collector = v.AllHasSBOMTree.Collector + retval.KnownSince = v.AllHasSBOMTree.KnownSince return &retval, nil } @@ -22277,6 +22396,7 @@ query CertifyBads ($filter: CertifyBadSpec!) { fragment AllCertifyBad on CertifyBad { id justification + knownSince subject { __typename ... on Package { @@ -24748,6 +24868,7 @@ fragment AllSLSATree on HasSLSA { fragment AllCertifyBad on CertifyBad { id justification + knownSince subject { __typename ... on Package { @@ -24766,6 +24887,7 @@ fragment AllCertifyBad on CertifyBad { fragment AllCertifyGood on CertifyGood { id justification + knownSince subject { __typename ... on Package { @@ -24807,6 +24929,7 @@ fragment AllHasSBOMTree on HasSBOM { downloadLocation origin collector + knownSince } fragment AllHasSourceAt on HasSourceAt { id @@ -25207,6 +25330,7 @@ fragment AllSLSATree on HasSLSA { fragment AllCertifyBad on CertifyBad { id justification + knownSince subject { __typename ... on Package { @@ -25225,6 +25349,7 @@ fragment AllCertifyBad on CertifyBad { fragment AllCertifyGood on CertifyGood { id justification + knownSince subject { __typename ... on Package { @@ -25266,6 +25391,7 @@ fragment AllHasSBOMTree on HasSBOM { downloadLocation origin collector + knownSince } fragment AllHasSourceAt on HasSourceAt { id @@ -25664,6 +25790,7 @@ fragment AllSLSATree on HasSLSA { fragment AllCertifyBad on CertifyBad { id justification + knownSince subject { __typename ... on Package { @@ -25682,6 +25809,7 @@ fragment AllCertifyBad on CertifyBad { fragment AllCertifyGood on CertifyGood { id justification + knownSince subject { __typename ... on Package { @@ -25723,6 +25851,7 @@ fragment AllHasSBOMTree on HasSBOM { downloadLocation origin collector + knownSince } fragment AllHasSourceAt on HasSourceAt { id @@ -26350,6 +26479,7 @@ fragment AllSLSATree on HasSLSA { fragment AllCertifyBad on CertifyBad { id justification + knownSince subject { __typename ... on Package { @@ -26368,6 +26498,7 @@ fragment AllCertifyBad on CertifyBad { fragment AllCertifyGood on CertifyGood { id justification + knownSince subject { __typename ... on Package { @@ -26409,6 +26540,7 @@ fragment AllHasSBOMTree on HasSBOM { downloadLocation origin collector + knownSince } fragment AllHasSourceAt on HasSourceAt { id diff --git a/pkg/assembler/clients/operations/trees.graphql b/pkg/assembler/clients/operations/trees.graphql index 6e10474b9f..866004363a 100644 --- a/pkg/assembler/clients/operations/trees.graphql +++ b/pkg/assembler/clients/operations/trees.graphql @@ -208,6 +208,7 @@ fragment AllCertifyLegalTree on CertifyLegal { fragment AllCertifyBad on CertifyBad { id justification + knownSince subject { __typename ... on Package { @@ -227,6 +228,7 @@ fragment AllCertifyBad on CertifyBad { fragment AllCertifyGood on CertifyGood { id justification + knownSince subject { __typename ... on Package { @@ -270,6 +272,7 @@ fragment AllHasSBOMTree on HasSBOM { downloadLocation origin collector + knownSince } fragment AllHasSourceAt on HasSourceAt { diff --git a/pkg/assembler/graphql/examples/certify_bad.gql b/pkg/assembler/graphql/examples/certify_bad.gql index 961cba58ed..944f6c1bcc 100644 --- a/pkg/assembler/graphql/examples/certify_bad.gql +++ b/pkg/assembler/graphql/examples/certify_bad.gql @@ -1,6 +1,7 @@ fragment allCertifyBadTree on CertifyBad { id justification + knownSince subject { __typename ... on Package { diff --git a/pkg/assembler/graphql/examples/certify_good.gql b/pkg/assembler/graphql/examples/certify_good.gql index 7a1c2147bd..0d7d4ab993 100644 --- a/pkg/assembler/graphql/examples/certify_good.gql +++ b/pkg/assembler/graphql/examples/certify_good.gql @@ -1,6 +1,7 @@ fragment allCertifyGoodTree on CertifyGood { id justification + knownSince subject { __typename ... on Package { diff --git a/pkg/assembler/graphql/examples/has_sbom.gql b/pkg/assembler/graphql/examples/has_sbom.gql index cc61c8171f..56d8502c64 100644 --- a/pkg/assembler/graphql/examples/has_sbom.gql +++ b/pkg/assembler/graphql/examples/has_sbom.gql @@ -35,6 +35,7 @@ fragment allHasSBOMTree on HasSBOM { downloadLocation origin collector + knownSince } query HasSBOMQ1 { diff --git a/pkg/assembler/graphql/generated/artifact.generated.go b/pkg/assembler/graphql/generated/artifact.generated.go index 84495986fb..1e9ec2c04e 100644 --- a/pkg/assembler/graphql/generated/artifact.generated.go +++ b/pkg/assembler/graphql/generated/artifact.generated.go @@ -4735,6 +4735,8 @@ func (ec *executionContext) fieldContext_Query_CertifyBad(ctx context.Context, f return ec.fieldContext_CertifyBad_origin(ctx, field) case "collector": return ec.fieldContext_CertifyBad_collector(ctx, field) + case "knownSince": + return ec.fieldContext_CertifyBad_knownSince(ctx, field) } return nil, fmt.Errorf("no field named %q was found under type CertifyBad", field.Name) }, @@ -4802,6 +4804,8 @@ func (ec *executionContext) fieldContext_Query_CertifyGood(ctx context.Context, return ec.fieldContext_CertifyGood_origin(ctx, field) case "collector": return ec.fieldContext_CertifyGood_collector(ctx, field) + case "knownSince": + return ec.fieldContext_CertifyGood_knownSince(ctx, field) } return nil, fmt.Errorf("no field named %q was found under type CertifyGood", field.Name) }, @@ -5232,6 +5236,8 @@ func (ec *executionContext) fieldContext_Query_HasSBOM(ctx context.Context, fiel return ec.fieldContext_HasSBOM_origin(ctx, field) case "collector": return ec.fieldContext_HasSBOM_collector(ctx, field) + case "knownSince": + return ec.fieldContext_HasSBOM_knownSince(ctx, field) } return nil, fmt.Errorf("no field named %q was found under type HasSBOM", field.Name) }, diff --git a/pkg/assembler/graphql/generated/certifyBad.generated.go b/pkg/assembler/graphql/generated/certifyBad.generated.go index ea433e4885..3619990082 100644 --- a/pkg/assembler/graphql/generated/certifyBad.generated.go +++ b/pkg/assembler/graphql/generated/certifyBad.generated.go @@ -9,6 +9,7 @@ import ( "strconv" "sync" "sync/atomic" + "time" "github.com/99designs/gqlgen/graphql" "github.com/guacsec/guac/pkg/assembler/graphql/model" @@ -249,6 +250,50 @@ func (ec *executionContext) fieldContext_CertifyBad_collector(ctx context.Contex return fc, nil } +func (ec *executionContext) _CertifyBad_knownSince(ctx context.Context, field graphql.CollectedField, obj *model.CertifyBad) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_CertifyBad_knownSince(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return obj.KnownSince, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(time.Time) + fc.Result = res + return ec.marshalNTime2timeᚐTime(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_CertifyBad_knownSince(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "CertifyBad", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + return nil, errors.New("field of type Time does not have child fields") + }, + } + return fc, nil +} + // endregion **************************** field.gotpl ***************************** // region **************************** input.gotpl ***************************** @@ -260,7 +305,7 @@ func (ec *executionContext) unmarshalInputCertifyBadInputSpec(ctx context.Contex asMap[k] = v } - fieldsInOrder := [...]string{"justification", "origin", "collector"} + fieldsInOrder := [...]string{"justification", "origin", "collector", "knownSince"} for _, k := range fieldsInOrder { v, ok := asMap[k] if !ok { @@ -294,6 +339,15 @@ func (ec *executionContext) unmarshalInputCertifyBadInputSpec(ctx context.Contex return it, err } it.Collector = data + case "knownSince": + var err error + + ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("knownSince")) + data, err := ec.unmarshalNTime2timeᚐTime(ctx, v) + if err != nil { + return it, err + } + it.KnownSince = data } } @@ -307,7 +361,7 @@ func (ec *executionContext) unmarshalInputCertifyBadSpec(ctx context.Context, ob asMap[k] = v } - fieldsInOrder := [...]string{"id", "subject", "justification", "origin", "collector"} + fieldsInOrder := [...]string{"id", "subject", "justification", "origin", "collector", "knownSince"} for _, k := range fieldsInOrder { v, ok := asMap[k] if !ok { @@ -359,6 +413,15 @@ func (ec *executionContext) unmarshalInputCertifyBadSpec(ctx context.Context, ob return it, err } it.Collector = data + case "knownSince": + var err error + + ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("knownSince")) + data, err := ec.unmarshalOTime2ᚖtimeᚐTime(ctx, v) + if err != nil { + return it, err + } + it.KnownSince = data } } @@ -609,6 +672,11 @@ func (ec *executionContext) _CertifyBad(ctx context.Context, sel ast.SelectionSe if out.Values[i] == graphql.Null { out.Invalids++ } + case "knownSince": + out.Values[i] = ec._CertifyBad_knownSince(ctx, field, obj) + if out.Values[i] == graphql.Null { + out.Invalids++ + } default: panic("unknown field " + strconv.Quote(field.Name)) } diff --git a/pkg/assembler/graphql/generated/certifyGood.generated.go b/pkg/assembler/graphql/generated/certifyGood.generated.go index ab14df762e..9e88812286 100644 --- a/pkg/assembler/graphql/generated/certifyGood.generated.go +++ b/pkg/assembler/graphql/generated/certifyGood.generated.go @@ -8,6 +8,7 @@ import ( "strconv" "sync" "sync/atomic" + "time" "github.com/99designs/gqlgen/graphql" "github.com/guacsec/guac/pkg/assembler/graphql/model" @@ -248,6 +249,50 @@ func (ec *executionContext) fieldContext_CertifyGood_collector(ctx context.Conte return fc, nil } +func (ec *executionContext) _CertifyGood_knownSince(ctx context.Context, field graphql.CollectedField, obj *model.CertifyGood) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_CertifyGood_knownSince(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return obj.KnownSince, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(time.Time) + fc.Result = res + return ec.marshalNTime2timeᚐTime(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_CertifyGood_knownSince(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "CertifyGood", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + return nil, errors.New("field of type Time does not have child fields") + }, + } + return fc, nil +} + // endregion **************************** field.gotpl ***************************** // region **************************** input.gotpl ***************************** @@ -259,7 +304,7 @@ func (ec *executionContext) unmarshalInputCertifyGoodInputSpec(ctx context.Conte asMap[k] = v } - fieldsInOrder := [...]string{"justification", "origin", "collector"} + fieldsInOrder := [...]string{"justification", "origin", "collector", "knownSince"} for _, k := range fieldsInOrder { v, ok := asMap[k] if !ok { @@ -293,6 +338,15 @@ func (ec *executionContext) unmarshalInputCertifyGoodInputSpec(ctx context.Conte return it, err } it.Collector = data + case "knownSince": + var err error + + ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("knownSince")) + data, err := ec.unmarshalNTime2timeᚐTime(ctx, v) + if err != nil { + return it, err + } + it.KnownSince = data } } @@ -306,7 +360,7 @@ func (ec *executionContext) unmarshalInputCertifyGoodSpec(ctx context.Context, o asMap[k] = v } - fieldsInOrder := [...]string{"id", "subject", "justification", "origin", "collector"} + fieldsInOrder := [...]string{"id", "subject", "justification", "origin", "collector", "knownSince"} for _, k := range fieldsInOrder { v, ok := asMap[k] if !ok { @@ -358,6 +412,15 @@ func (ec *executionContext) unmarshalInputCertifyGoodSpec(ctx context.Context, o return it, err } it.Collector = data + case "knownSince": + var err error + + ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("knownSince")) + data, err := ec.unmarshalOTime2ᚖtimeᚐTime(ctx, v) + if err != nil { + return it, err + } + it.KnownSince = data } } @@ -408,6 +471,11 @@ func (ec *executionContext) _CertifyGood(ctx context.Context, sel ast.SelectionS if out.Values[i] == graphql.Null { out.Invalids++ } + case "knownSince": + out.Values[i] = ec._CertifyGood_knownSince(ctx, field, obj) + if out.Values[i] == graphql.Null { + out.Invalids++ + } default: panic("unknown field " + strconv.Quote(field.Name)) } diff --git a/pkg/assembler/graphql/generated/hasSBOM.generated.go b/pkg/assembler/graphql/generated/hasSBOM.generated.go index 0b9714487b..529f4d6fbf 100644 --- a/pkg/assembler/graphql/generated/hasSBOM.generated.go +++ b/pkg/assembler/graphql/generated/hasSBOM.generated.go @@ -8,6 +8,7 @@ import ( "strconv" "sync" "sync/atomic" + "time" "github.com/99designs/gqlgen/graphql" "github.com/guacsec/guac/pkg/assembler/graphql/model" @@ -380,6 +381,50 @@ func (ec *executionContext) fieldContext_HasSBOM_collector(ctx context.Context, return fc, nil } +func (ec *executionContext) _HasSBOM_knownSince(ctx context.Context, field graphql.CollectedField, obj *model.HasSbom) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_HasSBOM_knownSince(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return obj.KnownSince, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(time.Time) + fc.Result = res + return ec.marshalNTime2timeᚐTime(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_HasSBOM_knownSince(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "HasSBOM", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + return nil, errors.New("field of type Time does not have child fields") + }, + } + return fc, nil +} + // endregion **************************** field.gotpl ***************************** // region **************************** input.gotpl ***************************** @@ -391,7 +436,7 @@ func (ec *executionContext) unmarshalInputHasSBOMInputSpec(ctx context.Context, asMap[k] = v } - fieldsInOrder := [...]string{"uri", "algorithm", "digest", "downloadLocation", "origin", "collector"} + fieldsInOrder := [...]string{"uri", "algorithm", "digest", "downloadLocation", "origin", "collector", "knownSince"} for _, k := range fieldsInOrder { v, ok := asMap[k] if !ok { @@ -452,6 +497,15 @@ func (ec *executionContext) unmarshalInputHasSBOMInputSpec(ctx context.Context, return it, err } it.Collector = data + case "knownSince": + var err error + + ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("knownSince")) + data, err := ec.unmarshalNTime2timeᚐTime(ctx, v) + if err != nil { + return it, err + } + it.KnownSince = data } } @@ -465,7 +519,7 @@ func (ec *executionContext) unmarshalInputHasSBOMSpec(ctx context.Context, obj i asMap[k] = v } - fieldsInOrder := [...]string{"id", "subject", "uri", "algorithm", "digest", "downloadLocation", "origin", "collector"} + fieldsInOrder := [...]string{"id", "subject", "uri", "algorithm", "digest", "downloadLocation", "origin", "collector", "knownSince"} for _, k := range fieldsInOrder { v, ok := asMap[k] if !ok { @@ -544,6 +598,15 @@ func (ec *executionContext) unmarshalInputHasSBOMSpec(ctx context.Context, obj i return it, err } it.Collector = data + case "knownSince": + var err error + + ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("knownSince")) + data, err := ec.unmarshalOTime2ᚖtimeᚐTime(ctx, v) + if err != nil { + return it, err + } + it.KnownSince = data } } @@ -609,6 +672,11 @@ func (ec *executionContext) _HasSBOM(ctx context.Context, sel ast.SelectionSet, if out.Values[i] == graphql.Null { out.Invalids++ } + case "knownSince": + out.Values[i] = ec._HasSBOM_knownSince(ctx, field, obj) + if out.Values[i] == graphql.Null { + out.Invalids++ + } default: panic("unknown field " + strconv.Quote(field.Name)) } diff --git a/pkg/assembler/graphql/generated/root_.generated.go b/pkg/assembler/graphql/generated/root_.generated.go index 7665457db5..99da8d1ff7 100644 --- a/pkg/assembler/graphql/generated/root_.generated.go +++ b/pkg/assembler/graphql/generated/root_.generated.go @@ -54,6 +54,7 @@ type ComplexityRoot struct { Collector func(childComplexity int) int ID func(childComplexity int) int Justification func(childComplexity int) int + KnownSince func(childComplexity int) int Origin func(childComplexity int) int Subject func(childComplexity int) int } @@ -62,6 +63,7 @@ type ComplexityRoot struct { Collector func(childComplexity int) int ID func(childComplexity int) int Justification func(childComplexity int) int + KnownSince func(childComplexity int) int Origin func(childComplexity int) int Subject func(childComplexity int) int } @@ -123,6 +125,7 @@ type ComplexityRoot struct { Digest func(childComplexity int) int DownloadLocation func(childComplexity int) int ID func(childComplexity int) int + KnownSince func(childComplexity int) int Origin func(childComplexity int) int Subject func(childComplexity int) int URI func(childComplexity int) int @@ -470,6 +473,13 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.CertifyBad.Justification(childComplexity), true + case "CertifyBad.knownSince": + if e.complexity.CertifyBad.KnownSince == nil { + break + } + + return e.complexity.CertifyBad.KnownSince(childComplexity), true + case "CertifyBad.origin": if e.complexity.CertifyBad.Origin == nil { break @@ -505,6 +515,13 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.CertifyGood.Justification(childComplexity), true + case "CertifyGood.knownSince": + if e.complexity.CertifyGood.KnownSince == nil { + break + } + + return e.complexity.CertifyGood.KnownSince(childComplexity), true + case "CertifyGood.origin": if e.complexity.CertifyGood.Origin == nil { break @@ -806,6 +823,13 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.HasSBOM.ID(childComplexity), true + case "HasSBOM.knownSince": + if e.complexity.HasSBOM.KnownSince == nil { + break + } + + return e.complexity.HasSBOM.KnownSince(childComplexity), true + case "HasSBOM.origin": if e.complexity.HasSBOM.Origin == nil { break @@ -2883,6 +2907,7 @@ type CertifyBad { justification: String! origin: String! collector: String! + knownSince: Time! } """ @@ -2895,6 +2920,9 @@ name and one of version, qualifiers, or subpath. If a source is specified in the subject filter, then it must specify a name, and optionally a tag and a commit. + +If KnownSince is specified, the returned value will be after or equal to the specified time. +Any nodes time that is before KnownSince is excluded. """ input CertifyBadSpec { id: ID @@ -2902,6 +2930,7 @@ input CertifyBadSpec { justification: String origin: String collector: String + knownSince: Time } """ @@ -2912,6 +2941,7 @@ input CertifyBadInputSpec { justification: String! origin: String! collector: String! + knownSince: Time! } """ @@ -2986,6 +3016,7 @@ type CertifyGood { justification: String! origin: String! collector: String! + knownSince: Time! } """ @@ -2998,6 +3029,9 @@ name and one of version, qualifiers, or subpath. If a source is specified in the subject filter, then it must specify a name, and optionally a tag and a commit. + +If KnownSince is specified, the returned value will be after or equal to the specified time. +Any nodes time that is before KnownSince is excluded. """ input CertifyGoodSpec { id: ID @@ -3005,6 +3039,7 @@ input CertifyGoodSpec { justification: String origin: String collector: String + knownSince: Time } """ @@ -3014,6 +3049,7 @@ input CertifyGoodInputSpec { justification: String! origin: String! collector: String! + knownSince: Time! } extend type Query { @@ -3686,12 +3722,17 @@ type HasSBOM { origin: String! "GUAC collector for the document" collector: String! + "Timestamp for SBOM creation" + knownSince: Time! } """ HasSBOMSpec allows filtering the list of HasSBOM to return. Only the package or artifact can be added, not both. + +If KnownSince is specified, the returned value will be after or equal to the specified time. +Any nodes time that is before KnownSince is excluded. """ input HasSBOMSpec { id: ID @@ -3702,6 +3743,7 @@ input HasSBOMSpec { downloadLocation: String origin: String collector: String + knownSince: Time } "HasSBOMInputSpec is the same as HasSBOM but for mutation input." @@ -3712,6 +3754,7 @@ input HasSBOMInputSpec { downloadLocation: String! origin: String! collector: String! + knownSince: Time! } extend type Query { diff --git a/pkg/assembler/graphql/model/nodes.go b/pkg/assembler/graphql/model/nodes.go index a68007b6e9..84aca2b89e 100644 --- a/pkg/assembler/graphql/model/nodes.go +++ b/pkg/assembler/graphql/model/nodes.go @@ -107,6 +107,7 @@ type CertifyBad struct { Justification string `json:"justification"` Origin string `json:"origin"` Collector string `json:"collector"` + KnownSince time.Time `json:"knownSince"` } func (CertifyBad) IsNode() {} @@ -114,9 +115,10 @@ func (CertifyBad) IsNode() {} // CertifyBadInputSpec represents the mutation input to ingest a CertifyBad // evidence. type CertifyBadInputSpec struct { - Justification string `json:"justification"` - Origin string `json:"origin"` - Collector string `json:"collector"` + Justification string `json:"justification"` + Origin string `json:"origin"` + Collector string `json:"collector"` + KnownSince time.Time `json:"knownSince"` } // CertifyBadSpec allows filtering the list of CertifyBad evidence to return in a @@ -128,12 +130,16 @@ type CertifyBadInputSpec struct { // // If a source is specified in the subject filter, then it must specify a name, // and optionally a tag and a commit. +// +// If KnownSince is specified, the returned value will be after or equal to the specified time. +// Any nodes time that is before KnownSince is excluded. type CertifyBadSpec struct { ID *string `json:"id,omitempty"` Subject *PackageSourceOrArtifactSpec `json:"subject,omitempty"` Justification *string `json:"justification,omitempty"` Origin *string `json:"origin,omitempty"` Collector *string `json:"collector,omitempty"` + KnownSince *time.Time `json:"knownSince,omitempty"` } // CertifyGood is an attestation that a package, source, or artifact is considered @@ -153,15 +159,17 @@ type CertifyGood struct { Justification string `json:"justification"` Origin string `json:"origin"` Collector string `json:"collector"` + KnownSince time.Time `json:"knownSince"` } func (CertifyGood) IsNode() {} // CertifyGoodInputSpec represents the mutation input to ingest a CertifyGood evidence. type CertifyGoodInputSpec struct { - Justification string `json:"justification"` - Origin string `json:"origin"` - Collector string `json:"collector"` + Justification string `json:"justification"` + Origin string `json:"origin"` + Collector string `json:"collector"` + KnownSince time.Time `json:"knownSince"` } // CertifyBadSpec allows filtering the list of CertifyBad evidence to return in a @@ -173,12 +181,16 @@ type CertifyGoodInputSpec struct { // // If a source is specified in the subject filter, then it must specify a name, // and optionally a tag and a commit. +// +// If KnownSince is specified, the returned value will be after or equal to the specified time. +// Any nodes time that is before KnownSince is excluded. type CertifyGoodSpec struct { ID *string `json:"id,omitempty"` Subject *PackageSourceOrArtifactSpec `json:"subject,omitempty"` Justification *string `json:"justification,omitempty"` Origin *string `json:"origin,omitempty"` Collector *string `json:"collector,omitempty"` + KnownSince *time.Time `json:"knownSince,omitempty"` } // CertifyLegal is an attestation to attach legal information to a package or source. @@ -435,23 +447,29 @@ type HasSbom struct { Origin string `json:"origin"` // GUAC collector for the document Collector string `json:"collector"` + // Timestamp for SBOM creation + KnownSince time.Time `json:"knownSince"` } func (HasSbom) IsNode() {} // HasSBOMInputSpec is the same as HasSBOM but for mutation input. type HasSBOMInputSpec struct { - URI string `json:"uri"` - Algorithm string `json:"algorithm"` - Digest string `json:"digest"` - DownloadLocation string `json:"downloadLocation"` - Origin string `json:"origin"` - Collector string `json:"collector"` + URI string `json:"uri"` + Algorithm string `json:"algorithm"` + Digest string `json:"digest"` + DownloadLocation string `json:"downloadLocation"` + Origin string `json:"origin"` + Collector string `json:"collector"` + KnownSince time.Time `json:"knownSince"` } // HasSBOMSpec allows filtering the list of HasSBOM to return. // // Only the package or artifact can be added, not both. +// +// If KnownSince is specified, the returned value will be after or equal to the specified time. +// Any nodes time that is before KnownSince is excluded. type HasSBOMSpec struct { ID *string `json:"id,omitempty"` Subject *PackageOrArtifactSpec `json:"subject,omitempty"` @@ -461,6 +479,7 @@ type HasSBOMSpec struct { DownloadLocation *string `json:"downloadLocation,omitempty"` Origin *string `json:"origin,omitempty"` Collector *string `json:"collector,omitempty"` + KnownSince *time.Time `json:"knownSince,omitempty"` } // HasSLSA records that a subject node has a SLSA attestation. diff --git a/pkg/assembler/graphql/resolvers/certifyBad.resolvers.go b/pkg/assembler/graphql/resolvers/certifyBad.resolvers.go index da61dafac5..8945816eb7 100644 --- a/pkg/assembler/graphql/resolvers/certifyBad.resolvers.go +++ b/pkg/assembler/graphql/resolvers/certifyBad.resolvers.go @@ -18,6 +18,9 @@ func (r *mutationResolver) IngestCertifyBad(ctx context.Context, subject model.P if err := helper.ValidatePackageSourceOrArtifactInput(&subject, funcName); err != nil { return "", gqlerror.Errorf("%v :: %s", funcName, err) } + if certifyBad.KnownSince.IsZero() { + return "", gqlerror.Errorf("certifyBad.KnownSince is a zero time") + } ingestedCertifyBad, err := r.Backend.IngestCertifyBad(ctx, subject, &pkgMatchType, certifyBad) if err != nil { return "", err @@ -51,6 +54,13 @@ func (r *mutationResolver) IngestCertifyBads(ctx context.Context, subjects model if valuesDefined != 1 { return ingestedCertifyBadsIDS, gqlerror.Errorf("%v :: must specify at most packages, artifacts or sources", funcName) } + + for _, certifyBad := range certifyBads { + if certifyBad.KnownSince.IsZero() { + return ingestedCertifyBadsIDS, gqlerror.Errorf("certifyBads contains a zero time") + } + } + ingestedCertifyBads, err := r.Backend.IngestCertifyBads(ctx, subjects, &pkgMatchType, certifyBads) if err == nil { for _, certifybad := range ingestedCertifyBads { diff --git a/pkg/assembler/graphql/resolvers/certifyBad.resolvers_test.go b/pkg/assembler/graphql/resolvers/certifyBad.resolvers_test.go index 9aa7ec2579..35dd7792ae 100644 --- a/pkg/assembler/graphql/resolvers/certifyBad.resolvers_test.go +++ b/pkg/assembler/graphql/resolvers/certifyBad.resolvers_test.go @@ -18,6 +18,7 @@ package resolvers_test import ( "context" "testing" + "time" "github.com/golang/mock/gomock" "github.com/guacsec/guac/internal/testing/mocks" @@ -27,6 +28,8 @@ import ( "github.com/guacsec/guac/pkg/assembler/graphql/resolvers" ) +var ZeroTime = time.Unix(0, 0) + func TestIngestCertifyBad(t *testing.T) { type call struct { Sub model.PackageSourceOrArtifactInput @@ -62,6 +65,7 @@ func TestIngestCertifyBad(t *testing.T) { }, CB: &model.CertifyBadInputSpec{ Justification: "test justification", + KnownSince: ZeroTime, }, }, }, @@ -189,6 +193,7 @@ func TestIngestCertifyBads(t *testing.T) { CB: []*model.CertifyBadInputSpec{ { Justification: "test justification", + KnownSince: ZeroTime, }, }, }, diff --git a/pkg/assembler/graphql/resolvers/certifyGood.resolvers.go b/pkg/assembler/graphql/resolvers/certifyGood.resolvers.go index a8b64b9ddd..2a53e37f28 100644 --- a/pkg/assembler/graphql/resolvers/certifyGood.resolvers.go +++ b/pkg/assembler/graphql/resolvers/certifyGood.resolvers.go @@ -18,6 +18,9 @@ func (r *mutationResolver) IngestCertifyGood(ctx context.Context, subject model. if err := helper.ValidatePackageSourceOrArtifactInput(&subject, funcName); err != nil { return "", err } + if certifyGood.KnownSince.IsZero() { + return "", gqlerror.Errorf("certifyGood.KnownSince is a zero time") + } ingestedCertifyGood, err := r.Backend.IngestCertifyGood(ctx, subject, &pkgMatchType, certifyGood) if err != nil { return "", err @@ -51,6 +54,13 @@ func (r *mutationResolver) IngestCertifyGoods(ctx context.Context, subjects mode if valuesDefined != 1 { return ingestedCertifyGoodsIDS, gqlerror.Errorf("%v :: must specify at most packages, artifacts or sources", funcName) } + + for _, certifyGood := range certifyGoods { + if certifyGood.KnownSince.IsZero() { + return ingestedCertifyGoodsIDS, gqlerror.Errorf("certifyGoods contains a zero time") + } + } + ingestedCertifyGoods, err := r.Backend.IngestCertifyGoods(ctx, subjects, &pkgMatchType, certifyGoods) if err == nil { for _, certifyGood := range ingestedCertifyGoods { diff --git a/pkg/assembler/graphql/resolvers/certifyGood.resolvers_test.go b/pkg/assembler/graphql/resolvers/certifyGood.resolvers_test.go index 644c673670..f728853884 100644 --- a/pkg/assembler/graphql/resolvers/certifyGood.resolvers_test.go +++ b/pkg/assembler/graphql/resolvers/certifyGood.resolvers_test.go @@ -62,6 +62,7 @@ func TestIngestCertifyGood(t *testing.T) { }, CG: &model.CertifyGoodInputSpec{ Justification: "test justification", + KnownSince: ZeroTime, }, }, }, @@ -189,6 +190,7 @@ func TestIngestCertifyGoods(t *testing.T) { CG: []*model.CertifyGoodInputSpec{ { Justification: "test justification", + KnownSince: ZeroTime, }, }, }, diff --git a/pkg/assembler/graphql/resolvers/hasSBOM.resolvers.go b/pkg/assembler/graphql/resolvers/hasSBOM.resolvers.go index 222d9196fb..7c8d2f37fc 100644 --- a/pkg/assembler/graphql/resolvers/hasSBOM.resolvers.go +++ b/pkg/assembler/graphql/resolvers/hasSBOM.resolvers.go @@ -18,6 +18,10 @@ func (r *mutationResolver) IngestHasSbom(ctx context.Context, subject model.Pack if err := helper.ValidatePackageOrArtifactInput(&subject, funcName); err != nil { return "", gqlerror.Errorf("%v :: %s", funcName, err) } + if hasSbom.KnownSince.IsZero() { + return "", gqlerror.Errorf("hasSbom.KnownSince is a zero time") + } + ingestedHasSbom, err := r.Backend.IngestHasSbom(ctx, subject, hasSbom) if err != nil { return "", err @@ -46,6 +50,12 @@ func (r *mutationResolver) IngestHasSBOMs(ctx context.Context, subjects model.Pa return ingestedHasSBOMSIDS, gqlerror.Errorf("%v :: must specify at most packages or artifacts for ingestion", funcName) } + for _, hasSbom := range hasSBOMs { + if hasSbom.KnownSince.IsZero() { + return ingestedHasSBOMSIDS, gqlerror.Errorf("hasSBOMS contains a zero time") + } + } + ingestedHasSBOMs, err := r.Backend.IngestHasSBOMs(ctx, subjects, hasSBOMs) if err == nil { for _, hasSBOM := range ingestedHasSBOMs { diff --git a/pkg/assembler/graphql/resolvers/hasSBOM.resolvers_test.go b/pkg/assembler/graphql/resolvers/hasSBOM.resolvers_test.go index cbec603564..a1feed2244 100644 --- a/pkg/assembler/graphql/resolvers/hasSBOM.resolvers_test.go +++ b/pkg/assembler/graphql/resolvers/hasSBOM.resolvers_test.go @@ -60,7 +60,8 @@ func TestIngestHasSbom(t *testing.T) { Package: testdata.P1, }, HS: &model.HasSBOMInputSpec{ - URI: "test uri", + URI: "test uri", + KnownSince: ZeroTime, }, }, }, @@ -147,7 +148,8 @@ func TestIngestHasSBOMs(t *testing.T) { }, HS: []*model.HasSBOMInputSpec{ { - URI: "test uri", + URI: "test uri", + KnownSince: ZeroTime, }, }, }, @@ -163,7 +165,8 @@ func TestIngestHasSBOMs(t *testing.T) { }, HS: []*model.HasSBOMInputSpec{ { - URI: "test uri", + URI: "test uri", + KnownSince: ZeroTime, }, }, }, diff --git a/pkg/assembler/graphql/schema/certifyBad.graphql b/pkg/assembler/graphql/schema/certifyBad.graphql index 91738b367c..4562b36a58 100644 --- a/pkg/assembler/graphql/schema/certifyBad.graphql +++ b/pkg/assembler/graphql/schema/certifyBad.graphql @@ -75,6 +75,7 @@ type CertifyBad { justification: String! origin: String! collector: String! + knownSince: Time! } """ @@ -87,6 +88,9 @@ name and one of version, qualifiers, or subpath. If a source is specified in the subject filter, then it must specify a name, and optionally a tag and a commit. + +If KnownSince is specified, the returned value will be after or equal to the specified time. +Any nodes time that is before KnownSince is excluded. """ input CertifyBadSpec { id: ID @@ -94,6 +98,7 @@ input CertifyBadSpec { justification: String origin: String collector: String + knownSince: Time } """ @@ -104,6 +109,7 @@ input CertifyBadInputSpec { justification: String! origin: String! collector: String! + knownSince: Time! } """ diff --git a/pkg/assembler/graphql/schema/certifyGood.graphql b/pkg/assembler/graphql/schema/certifyGood.graphql index 9427355d78..52e393b671 100644 --- a/pkg/assembler/graphql/schema/certifyGood.graphql +++ b/pkg/assembler/graphql/schema/certifyGood.graphql @@ -36,6 +36,7 @@ type CertifyGood { justification: String! origin: String! collector: String! + knownSince: Time! } """ @@ -48,6 +49,9 @@ name and one of version, qualifiers, or subpath. If a source is specified in the subject filter, then it must specify a name, and optionally a tag and a commit. + +If KnownSince is specified, the returned value will be after or equal to the specified time. +Any nodes time that is before KnownSince is excluded. """ input CertifyGoodSpec { id: ID @@ -55,6 +59,7 @@ input CertifyGoodSpec { justification: String origin: String collector: String + knownSince: Time } """ @@ -64,6 +69,7 @@ input CertifyGoodInputSpec { justification: String! origin: String! collector: String! + knownSince: Time! } extend type Query { diff --git a/pkg/assembler/graphql/schema/hasSBOM.graphql b/pkg/assembler/graphql/schema/hasSBOM.graphql index 757b14192a..5c1525e30d 100644 --- a/pkg/assembler/graphql/schema/hasSBOM.graphql +++ b/pkg/assembler/graphql/schema/hasSBOM.graphql @@ -33,12 +33,17 @@ type HasSBOM { origin: String! "GUAC collector for the document" collector: String! + "Timestamp for SBOM creation" + knownSince: Time! } """ HasSBOMSpec allows filtering the list of HasSBOM to return. Only the package or artifact can be added, not both. + +If KnownSince is specified, the returned value will be after or equal to the specified time. +Any nodes time that is before KnownSince is excluded. """ input HasSBOMSpec { id: ID @@ -49,6 +54,7 @@ input HasSBOMSpec { downloadLocation: String origin: String collector: String + knownSince: Time } "HasSBOMInputSpec is the same as HasSBOM but for mutation input." @@ -59,6 +65,7 @@ input HasSBOMInputSpec { downloadLocation: String! origin: String! collector: String! + knownSince: Time! } extend type Query { diff --git a/pkg/guacanalytics/patchPlanning_test.go b/pkg/guacanalytics/patchPlanning_test.go index 29860a60cf..533225b317 100644 --- a/pkg/guacanalytics/patchPlanning_test.go +++ b/pkg/guacanalytics/patchPlanning_test.go @@ -452,6 +452,7 @@ var ( }, CertifyGood: &model.CertifyGoodInputSpec{ Justification: "good package", + KnownSince: tm, }, }, { @@ -466,6 +467,7 @@ var ( }, CertifyGood: &model.CertifyGoodInputSpec{ Justification: "good package", + KnownSince: tm, }, }, }, diff --git a/pkg/ingestor/parser/common/helpers.go b/pkg/ingestor/parser/common/helpers.go index 0bc366c299..99cd866502 100644 --- a/pkg/ingestor/parser/common/helpers.go +++ b/pkg/ingestor/parser/common/helpers.go @@ -19,6 +19,7 @@ import ( "crypto/sha256" "encoding/hex" "reflect" + "time" "github.com/guacsec/guac/pkg/assembler" model "github.com/guacsec/guac/pkg/assembler/clients/generated" @@ -100,7 +101,7 @@ func CreateTopLevelIsDeps(topLevel *model.PkgInputSpec, packages map[string][]*m return isDeps } -func CreateTopLevelHasSBOM(topLevel *model.PkgInputSpec, sbomDoc *processor.Document) assembler.HasSBOMIngest { +func CreateTopLevelHasSBOM(topLevel *model.PkgInputSpec, sbomDoc *processor.Document, timeStamp time.Time) assembler.HasSBOMIngest { sha256sum := sha256.Sum256(sbomDoc.Blob) hash := hex.EncodeToString(sha256sum[:]) return assembler.HasSBOMIngest{ @@ -110,6 +111,7 @@ func CreateTopLevelHasSBOM(topLevel *model.PkgInputSpec, sbomDoc *processor.Docu Algorithm: "sha256", Digest: hash, DownloadLocation: sbomDoc.SourceInformation.Source, + KnownSince: timeStamp, }, } } diff --git a/pkg/ingestor/parser/cyclonedx/parser_cyclonedx.go b/pkg/ingestor/parser/cyclonedx/parser_cyclonedx.go index 5dc2b65ec6..346b5113d6 100644 --- a/pkg/ingestor/parser/cyclonedx/parser_cyclonedx.go +++ b/pkg/ingestor/parser/cyclonedx/parser_cyclonedx.go @@ -21,6 +21,7 @@ import ( "fmt" "reflect" "strings" + "time" jsoniter "github.com/json-iterator/go" @@ -37,6 +38,8 @@ var json = jsoniter.ConfigCompatibleWithStandardLibrary const topCdxPurlGuac string = "pkg:guac/cdx/" +var zeroTime = time.Unix(0, 0) + type cyclonedxParser struct { doc *processor.Document packagePackages map[string][]*model.PkgInputSpec @@ -214,8 +217,21 @@ func (c *cyclonedxParser) GetPredicates(ctx context.Context) *assembler.IngestPr // TODO: This is not based on the relationship so that can be inaccurate (can capture both direct and in-direct)...Remove this and be done below by the *c.cdxBom.Dependencies? // see https://github.com/CycloneDX/specification/issues/33 if toplevel != nil { + var timestamp time.Time + var err error + if c.cdxBom.Metadata.Timestamp == "" { + // set the time to zero time if timestamp is not provided + timestamp = zeroTime + } else { + timestamp, err = time.Parse(time.RFC3339, c.cdxBom.Metadata.Timestamp) + if err != nil { + logger.Errorf("SPDX document had invalid created time %q : %v", c.cdxBom.Metadata.Timestamp, err) + return nil + } + } + preds.IsDependency = append(preds.IsDependency, common.CreateTopLevelIsDeps(toplevel[0], c.packagePackages, nil, "top-level package GUAC heuristic connecting to each file/package")...) - preds.HasSBOM = append(preds.HasSBOM, common.CreateTopLevelHasSBOM(toplevel[0], c.doc)) + preds.HasSBOM = append(preds.HasSBOM, common.CreateTopLevelHasSBOM(toplevel[0], c.doc, timestamp)) } for id := range c.packagePackages { diff --git a/pkg/ingestor/parser/spdx/parse_spdx.go b/pkg/ingestor/parser/spdx/parse_spdx.go index c6dd87cb6b..a7479d7b08 100644 --- a/pkg/ingestor/parser/spdx/parse_spdx.go +++ b/pkg/ingestor/parser/spdx/parse_spdx.go @@ -243,8 +243,13 @@ func (s *spdxParser) GetPredicates(ctx context.Context) *assembler.IngestPredica return preds } else { // adding top level package edge manually for all depends on package + timestamp, err := time.Parse(time.RFC3339, s.spdxDoc.CreationInfo.Created) + if err != nil { + logger.Errorf("SPDX document had invalid created time %q : %w", s.spdxDoc.CreationInfo.Created, err) + return nil + } for _, topLevelPkg := range topLevel { - preds.HasSBOM = append(preds.HasSBOM, common.CreateTopLevelHasSBOM(topLevelPkg, s.doc)) + preds.HasSBOM = append(preds.HasSBOM, common.CreateTopLevelHasSBOM(topLevelPkg, s.doc, timestamp)) } if s.topLevelIsHeuristic { From ffadd34be70f6190ce9f82e6c76bf31284644b54 Mon Sep 17 00:00:00 2001 From: Jeff Mendoza Date: Wed, 4 Oct 2023 14:27:07 -0700 Subject: [PATCH 05/21] Add a developer readme to the cli commands. (#1324) * Add a developer readme to the cli commands. Signed-off-by: Jeff Mendoza * Update README.md Signed-off-by: Jeff Mendoza --------- Signed-off-by: Jeff Mendoza --- cmd/README.md | 127 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 cmd/README.md diff --git a/cmd/README.md b/cmd/README.md new file mode 100644 index 0000000000..154c7dfdf2 --- /dev/null +++ b/cmd/README.md @@ -0,0 +1,127 @@ +# Developer notes for cli commands + +Divide the commands into two categories: + +- User-facing: Made to be run by a user / tester to accomplish something. Should + be consistent and intuitive CLI UX. One powerful command makes sense for the + user to learn. + +- Service: Things to be run as a service, ex: in a container, as a cron, as a + systemd service, etc. Separate single-purpose binaries make sense. + +## Commands + +user-facing: + +**guacone** + +- what it does: performs user-facing operations that require setting up a + processor / ingestor / assembler in one binary, only talks to GQL, no nats +- options: + - gql endpoint +- commands: + - certify - runs the certifier once (once by default, optional + poll) + - collector - runs the collector once, includes "files" (once by + default, optional poll) + - query - runs the canned query. + +services: + +**guacgql** + +- what it does: runs a GraphQL server +- options: + - backend: inmem, neo4j, arango, ent, or future DB + - backend-specific options: neo4j connection options + - playground / debug: also start playground + +**guaccsub** + +- what it does: runs the collector-subscriber service +- options: + - listening port + +**guacingest** + +- what it does: runs the ingestor connected to nats and GraphQL +- options: + - nats addr + - gql endpoint + +**guaccollect** + +- what it does: runs the named collector or certifier connected to GraphQL and + nats +- options: + - nats addr + - gql addr + - colsub addr + - collector/certifier name + - polling options + +## Collectors and Certifiers + +These appear both in `guacone` and in `guaccollect`. The difference is that +`guacone` uses the all-in-one processor-ingestor-assembler, and only depends on +`guacgql` being up. Conversely `guaccollect` depends on the nats ingestion +pipeline (or future ingestor services) being up and running. + +Collectors and Certifiers that are intended to be run by a user can be added to +`guacone` first, and should default to run-once. A polling option can be +included, though is not required.. + +Collectors and Certifiers that will be eventually run as part of a guac +deployment should be added to `guaccollect`. This is not required for initial +implementations / contributions. These should default to running as a service, +polling, a "watch", etc. An option to disable polling can be included, but is +not required. + +## Flag names: + +- Consistent name - For example, the Graph QL address is needed in most + commands, the flag name should be the same across all commands. + +- Consistent style - whatever it is, make is consistent. Use dash-style, + therefore don't use camelCase anywhere. + +- Descriptive on its own - The flag names are also used in the guac.yaml config + file. Therefore a name should be self descriptive. Good: `nats-addr`, Bad: + `type`. If it is something that has the same meaning everywhere, it is ok to + be short: ex: interval. + +- Namespaced - If appropriate, group a group of flags that go together with a + prefix, ex: neo4j. + +- Short versions - Service oriented flags don't need short versions, user + oriented flags should. + +- User oriented bools default false - Name bools so that the default is false. + Because long names must be descriptive and possibly namespaced (for the config + file), they are more cumbersome to type. You can't set a bool to false with + the short version, only with long (--long-name=false), so default to false, + and the short version can be used to enable it. + +- Required args - required args should be positional. Options should be optional + with good defaults. + +All the current flags are in +https://github.com/guacsec/guac/blob/main/pkg/cli/store.go This helps with the +consistent name, and to make sure the flags are not used for different meanings +in two places. + +## Other notes: + +Prefix all the binaries with guac. Binaries could eventually be installed in +/usr/bin. Avoid collisions by namespacing, ex: `ingestor` or `collector` are too +generic. + +Service-oriented CLI commands should exit gracefully. This means catching +SIGINT/SIGTERM and canceling contexts and/or calling Shutdown() on http servers. +`time.Sleep` should not be used anywhere for polling. + +## Background / history: + +https://github.com/guacsec/guac/issues/719 +https://github.com/guacsec/guac/issues/762 +https://github.com/guacsec/guac/issues/809 From a172069dc20ac7abdf2deabb6318d81e75393675 Mon Sep 17 00:00:00 2001 From: Juan Manuel Leflet Estrada Date: Fri, 22 Sep 2023 13:38:26 +0200 Subject: [PATCH 06/21] Initial commit Signed-off-by: Juan Manuel Leflet Estrada --- cmd/guacone/cmd/s3.go | 154 +++++++++++++++ go.mod | 21 +++ go.sum | 15 ++ pkg/handler/collector/s3/bucket/bucket.go | 115 ++++++++++++ pkg/handler/collector/s3/bucket/encoding.go | 51 +++++ pkg/handler/collector/s3/messaging/kafka.go | 107 +++++++++++ pkg/handler/collector/s3/messaging/message.go | 67 +++++++ pkg/handler/collector/s3/messaging/sqs.go | 156 ++++++++++++++++ pkg/handler/collector/s3/s3.go | 175 ++++++++++++++++++ pkg/handler/collector/s3/s3_test.go | 171 +++++++++++++++++ 10 files changed, 1032 insertions(+) create mode 100644 cmd/guacone/cmd/s3.go create mode 100644 pkg/handler/collector/s3/bucket/bucket.go create mode 100644 pkg/handler/collector/s3/bucket/encoding.go create mode 100644 pkg/handler/collector/s3/messaging/kafka.go create mode 100644 pkg/handler/collector/s3/messaging/message.go create mode 100644 pkg/handler/collector/s3/messaging/sqs.go create mode 100644 pkg/handler/collector/s3/s3.go create mode 100644 pkg/handler/collector/s3/s3_test.go diff --git a/cmd/guacone/cmd/s3.go b/cmd/guacone/cmd/s3.go new file mode 100644 index 0000000000..b1d2f1bf1f --- /dev/null +++ b/cmd/guacone/cmd/s3.go @@ -0,0 +1,154 @@ +// +// Copyright 2023 The GUAC Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "context" + "fmt" + "os" + + csub_client "github.com/guacsec/guac/pkg/collectsub/client" + "github.com/guacsec/guac/pkg/handler/collector" + "github.com/guacsec/guac/pkg/handler/collector/s3" + "github.com/guacsec/guac/pkg/handler/processor" + "github.com/guacsec/guac/pkg/ingestor" + "github.com/guacsec/guac/pkg/logging" + "github.com/spf13/cobra" + "github.com/spf13/viper" + "golang.org/x/sync/errgroup" +) + +// s3Options flags for configuring the command +type s3Options struct { + s3hostname string // hostname of the s3 provider + s3port string // port of the s3 provider + region string // AWS region, for s3/sqs configuration (defaults to us-east-1) + queues string // comma-separated list of queues/topics + mp string // message provider name (sqs or kafka, will default to kafka) + mpHostname string // hostname for the message provider + mpPort string // port for the message provider +} + +var s3Opts s3Options + +var s3Cmd = &cobra.Command{ + Use: "s3 [flags]", + Short: "listens to kafka/sqs s3 events to download documents and add them to the GUAC graph", + Args: cobra.MinimumNArgs(0), + Run: func(cmd *cobra.Command, args []string) { + ctx := logging.WithLogger(context.Background()) + logger := logging.FromContext(ctx) + + s3Opts, err := validateS3Opts(s3Opts.s3hostname, s3Opts.s3port, s3Opts.mp, s3Opts.mpHostname, s3Opts.mpPort, s3Opts.queues, s3Opts.region) + if err != nil { + os.Exit(1) + } + + s3Collector, err := s3.NewS3Collector(s3.S3CollectorConfig{ + S3Host: s3Opts.s3hostname, + S3Port: s3Opts.s3port, + MessageProvider: s3Opts.mp, + MessageProviderHost: s3Opts.mpHostname, + MessageProviderPort: s3Opts.mpPort, + Queues: s3Opts.queues, + Region: s3Opts.region, + }) + if err != nil { + logger.Errorf("unable to create s3 collector: %v", err) + os.Exit(1) + } + + err = collector.RegisterDocumentCollector(s3Collector, s3.S3CollectorType) + if err != nil { + logger.Errorf("unable to register s3 collector: %v\n", err) + os.Exit(1) + } + + csubClient, err := csub_client.NewClient(viper.GetString("csub-addr")) + if err != nil { + logger.Infof("collectsub client initialization failed, this ingestion will not pull in any additional data through the collectsub service: %v", err) + csubClient = nil + } else { + defer csubClient.Close() + } + + errFound := false + + _, s3ctx := errgroup.WithContext(ctx) + emit := func(d *processor.Document) error { + err := ingestor.Ingest(s3ctx, d, viper.GetString("gql-addr"), csubClient) + + if err != nil { + errFound = true + return fmt.Errorf("unable to ingest document: %w", err) + } + return nil + } + + errHandler := func(err error) bool { + if err == nil { + logger.Info("collector ended gracefully") + return true + } + logger.Errorf("collector ended with error: %v", err) + return false + } + + if err := collector.Collect(ctx, emit, errHandler); err != nil { + logger.Fatal(err) + } + + if errFound { + logger.Fatalf("completed ingestion with error") + } else { + logger.Infof("completed ingestion") + } + }, +} + +func validateS3Opts(s3hostname string, s3port string, mp string, mpHostname string, mpPort string, queues string, region string) (s3Options, error) { + var opts s3Options + if len(s3hostname) == 0 { + return opts, fmt.Errorf("expected s3 hostname") + } + if len(s3port) == 0 { + return opts, fmt.Errorf("expected s3 port") + } + if len(mpHostname) == 0 { + return opts, fmt.Errorf("expected hostname for message provider") + } + if len(mpPort) == 0 { + return opts, fmt.Errorf("expected port for message provider") + } + if len(queues) == 0 { + return opts, fmt.Errorf("expected at least one queue") + } + + opts = s3Options{s3hostname, s3port, region, queues, mp, mpHostname, mpPort} + + return opts, nil +} + +func init() { + s3Cmd.Flags().StringVarP(&s3Opts.s3hostname, "s3-host", "", "", "hostname for s3") + s3Cmd.Flags().StringVarP(&s3Opts.s3port, "s3-port", "", "", "port for s3") + s3Cmd.Flags().StringVarP(&s3Opts.mp, "mp", "m", "", "message provider (sqs or kafka, defaults to kafka)") + s3Cmd.Flags().StringVarP(&s3Opts.mpHostname, "mp-host", "", "", "hostname for the message provider (sqs/kafka)") + s3Cmd.Flags().StringVarP(&s3Opts.mpPort, "mp-port", "", "", "port for the message provider (sqs/kafka)") + s3Cmd.Flags().StringVarP(&s3Opts.queues, "queues", "", "", "comma-separated list of queue/topic names") + s3Cmd.Flags().StringVarP(&s3Opts.region, "region", "", "us-east-1", "aws region, defaults to us-east-1") + collectCmd.AddCommand(s3Cmd) +} diff --git a/go.mod b/go.mod index 7c9f67a016..573da3512c 100644 --- a/go.mod +++ b/go.mod @@ -62,6 +62,21 @@ require ( github.com/anchore/go-struct-converter v0.0.0-20221221214134-65614c61201e // indirect github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.11 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.13.31 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.7 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.37 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.31 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.38 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.12 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.32 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.31 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.13.1 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.21.1 // indirect + github.com/aws/smithy-go v1.14.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bombsimon/logrusr/v2 v2.0.1 // indirect github.com/bradleyfalzon/ghinstallation/v2 v2.6.0 // indirect @@ -127,6 +142,7 @@ require ( github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0-rc3 // indirect github.com/pelletier/go-toml/v2 v2.0.8 // indirect + github.com/pierrec/lz4/v4 v4.1.15 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect @@ -184,6 +200,10 @@ require ( github.com/Masterminds/semver v1.5.0 github.com/arangodb/go-driver v1.6.0 github.com/aws/aws-sdk-go v1.45.20 + github.com/aws/aws-sdk-go-v2 v1.20.0 + github.com/aws/aws-sdk-go-v2/config v1.18.32 + github.com/aws/aws-sdk-go-v2/service/s3 v1.38.1 + github.com/aws/aws-sdk-go-v2/service/sqs v1.24.1 github.com/fsnotify/fsnotify v1.6.0 github.com/go-git/go-git/v5 v5.9.0 github.com/gobwas/glob v0.2.3 @@ -208,6 +228,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.17.0 github.com/regclient/regclient v0.5.1 + github.com/segmentio/kafka-go v0.4.42 github.com/segmentio/ksuid v1.0.4 github.com/sigstore/sigstore v1.7.3 github.com/spdx/tools-golang v0.5.3 diff --git a/go.sum b/go.sum index 1377d4d4c9..baf8b914a2 100644 --- a/go.sum +++ b/go.sum @@ -143,6 +143,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.0 h1:Wgjft9X4W5pMeu github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.0/go.mod h1:FWNzS4+zcWAP05IF7TDYTY1ysZAzIvogxWaDT9p8fsA= github.com/aws/aws-sdk-go-v2/service/s3 v1.38.1 h1:mTgFVlfQT8gikc5+/HwD8UL9jnUro5MGv8n/VEYF12I= github.com/aws/aws-sdk-go-v2/service/s3 v1.38.1/go.mod h1:6SOWLiobcZZshbmECRTADIRYliPL0etqFSigauQEeT0= +github.com/aws/aws-sdk-go-v2/service/sqs v1.24.1 h1:KbGaxApdPOT2ZWqJiQY5ApnpNhUGbGTjYiKAidlFwp8= +github.com/aws/aws-sdk-go-v2/service/sqs v1.24.1/go.mod h1:+phkm4aFvcM4jbsDRGoZ+mD8MMvksHF459Xpy5Z90f0= github.com/aws/aws-sdk-go-v2/service/sso v1.13.1 h1:DSNpSbfEgFXRV+IfEcKE5kTbqxm+MeF5WgyeRlsLnHY= github.com/aws/aws-sdk-go-v2/service/sso v1.13.1/go.mod h1:TC9BubuFMVScIU+TLKamO6VZiYTkYoEHqlSQwAe2omw= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.1 h1:hd0SKLMdOL/Sl6Z0np1PX9LeH2gqNtBe0MhTedA8MGI= @@ -445,6 +447,7 @@ github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4 github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM= github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= @@ -549,6 +552,8 @@ github.com/package-url/packageurl-go v0.1.1 h1:KTRE0bK3sKbFKAk3yy63DpeskU7Cvs/x/ github.com/package-url/packageurl-go v0.1.1/go.mod h1:uQd4a7Rh3ZsVg5j0lNyAfyxIeGde9yrlhjF78GzeW0c= github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= +github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= +github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -587,6 +592,8 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/secure-systems-lab/go-securesystemslib v0.7.0 h1:OwvJ5jQf9LnIAS83waAjPbcMsODrTQUpJ02eNLUoxBg= github.com/secure-systems-lab/go-securesystemslib v0.7.0/go.mod h1:/2gYnlnHVQ6xeGtfIqFy7Do03K4cdCY0A/GlJLDKLHI= +github.com/segmentio/kafka-go v0.4.42 h1:qffhBZCz4WcWyNuHEclHjIMLs2slp6mZO8px+5W5tfU= +github.com/segmentio/kafka-go v0.4.42/go.mod h1:d0g15xPMqoUookug0OU75DhGZxXwCFxSLeJ4uphwJzg= github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c= github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= @@ -668,6 +675,12 @@ github.com/xanzy/go-gitlab v0.89.0 h1:yJuy1Pw+to/NqHzVIiopt/VApoHvGDB5SEGuRs3EJp github.com/xanzy/go-gitlab v0.89.0/go.mod h1:5ryv+MnpZStBH8I/77HuQBsMbBGANtVpLWC15qOjWAw= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= +github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= +github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= +github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= +github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= @@ -799,6 +812,7 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= @@ -916,6 +930,7 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= diff --git a/pkg/handler/collector/s3/bucket/bucket.go b/pkg/handler/collector/s3/bucket/bucket.go new file mode 100644 index 0000000000..8838177e6c --- /dev/null +++ b/pkg/handler/collector/s3/bucket/bucket.go @@ -0,0 +1,115 @@ +// +// Copyright 2023 The GUAC Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bucket + +import ( + "bytes" + "context" + "fmt" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" + "io" +) + +type BuildBucket interface { + GetDownloader(hostname string, port string, region string) Bucket +} + +type BucketBuilder struct { +} + +func (bd BucketBuilder) GetBucket(hostname string, port string, region string) Bucket { + return S3Bucket{ + hostname, + port, + region, + } +} + +type Bucket interface { + DownloadFile(ctx context.Context, bucket string, item string) ([]byte, error) + GetEncoding(ctx context.Context, bucket string, item string) (string, error) +} + +type S3Bucket struct { + hostname string + port string + region string +} + +func GetDefaultBucket(hostname string, port string, region string) Bucket { + return S3Bucket{hostname, port, region} +} + +func (d S3Bucket) DownloadFile(ctx context.Context, bucket string, item string) ([]byte, error) { + cfg, err := config.LoadDefaultConfig(ctx) + if err != nil { + return nil, fmt.Errorf("error loading AWS SDK config: %v", err) + } + + addr := fmt.Sprintf("http://%s:%s/%s/", d.hostname, d.port, bucket) + cfg.Region = d.region + + client := s3.NewFromConfig(cfg, func(o *s3.Options) { + o.BaseEndpoint = aws.String(addr) + }) + + // Create a GetObjectInput with the bucket name and object key. + input := &s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(item), + } + + resp, err := client.GetObject(ctx, input) + if err != nil { + return nil, fmt.Errorf("unable to download file: %v", err) + } + defer resp.Body.Close() + + buf := new(bytes.Buffer) + n, err := io.Copy(buf, resp.Body) + if err != nil || n == 0 { + return nil, fmt.Errorf("unable to read file contents: %v", err) + } + + return buf.Bytes(), err +} + +func (d S3Bucket) GetEncoding(ctx context.Context, bucket string, item string) (string, error) { + cfg, err := config.LoadDefaultConfig(context.TODO()) + if err != nil { + return "", fmt.Errorf("error loading AWS SDK config: %v", err) + } + + addr := fmt.Sprintf("http://%s:%s/%s/", d.hostname, d.port, bucket) + cfg.Region = d.region + + client := s3.NewFromConfig(cfg, func(o *s3.Options) { + o.BaseEndpoint = aws.String(addr) + }) + + headObject, err := client.HeadObject(context.Background(), &s3.HeadObjectInput{Bucket: aws.String(bucket), Key: aws.String(item)}) + if err != nil { + return "", fmt.Errorf("could not get head object: %v", err) + } + + if headObject.ContentEncoding == nil { + return "", nil + } + + return *headObject.ContentEncoding, nil +} diff --git a/pkg/handler/collector/s3/bucket/encoding.go b/pkg/handler/collector/s3/bucket/encoding.go new file mode 100644 index 0000000000..17a3683217 --- /dev/null +++ b/pkg/handler/collector/s3/bucket/encoding.go @@ -0,0 +1,51 @@ +// +// Copyright 2023 The GUAC Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bucket + +import ( + "github.com/guacsec/guac/pkg/handler/processor" + "strings" +) + +const ( + BZIP2 = "BZIP2" + ZSTD = "ZSTD" + UNKNOWN = "UNKNOWN" +) + +func ExtractEncoding(encoding string, filename string) processor.EncodingType { + switch encoding { + case BZIP2: + return BZIP2 + case ZSTD: + return ZSTD + default: + return FromFile(filename) + } +} + +func FromFile(file string) processor.EncodingType { + strs := strings.Split(file, ".") + extension := strs[len(strs)-1] + switch extension { + case "bz2": + return BZIP2 + case "zst": + return ZSTD + default: + return UNKNOWN + } +} diff --git a/pkg/handler/collector/s3/messaging/kafka.go b/pkg/handler/collector/s3/messaging/kafka.go new file mode 100644 index 0000000000..8fcbe7f11f --- /dev/null +++ b/pkg/handler/collector/s3/messaging/kafka.go @@ -0,0 +1,107 @@ +// +// Copyright 2023 The GUAC Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package messaging + +import ( + "context" + "encoding/json" + "fmt" + "github.com/guacsec/guac/pkg/logging" + "github.com/segmentio/kafka-go" + "strings" +) + +type KafkaProvider struct { + // Kafka-specific configuration fields + reader *kafka.Reader +} + +type KafkaMessage struct { + EventName string `json:"EventName"` + Key string `json:"Key"` +} + +func (m KafkaMessage) GetEvent() (EventName, error) { + if m.EventName == "s3:ObjectCreated:Put" { + return PUT, nil + } + return "", nil +} + +func (m KafkaMessage) GetBucket() (string, error) { + info := strings.Split(m.Key, "/") + if len(info) < 2 { + return "", fmt.Errorf("invalid format of key: %s", m.Key) + } + return info[0], nil +} + +func (m KafkaMessage) GetItem() (string, error) { + info := strings.Split(m.Key, "/") + if len(info) < 2 { + return "", fmt.Errorf("invalid format of item: %s", m.Key) + } + return info[1], nil +} + +func NewKafkaProvider(mpConfig MessageProviderConfig) (KafkaProvider, error) { + kafkaHostname := mpConfig.Host + kafkaPort := mpConfig.Port + kafkaTopic := mpConfig.Queue + + kafkaProvider := KafkaProvider{} + kafkaProvider.reader = kafka.NewReader(kafka.ReaderConfig{ + Brokers: []string{fmt.Sprintf("%s:%s", kafkaHostname, kafkaPort)}, + Topic: kafkaTopic, + Partition: 0, + MaxBytes: 10e6, + }) + err := kafkaProvider.reader.SetOffset(kafka.LastOffset) + if err != nil { + return KafkaProvider{}, err + } + + return kafkaProvider, nil +} + +func (k KafkaProvider) ReceiveMessage(ctx context.Context) (Message, error) { + logger := logging.FromContext(ctx) + + m, err := k.reader.ReadMessage(context.Background()) + if err != nil { + fmt.Println(err.Error()) + } + logger.Debugf("Message at offset %d: %s = %s\n", m.Offset, string(m.Key), string(m.Value)) + + msg := KafkaMessage{} + err = json.Unmarshal(m.Value, &msg) + if err != nil { + return msg, fmt.Errorf("error parsing JSON: %v", err) + } + + return msg, err +} + +func (k KafkaProvider) Close(ctx context.Context) error { + logger := logging.FromContext(ctx) + + if err := k.reader.Close(); err != nil { + logger.Errorf("failed to close reader:", err) + return err + } + + return nil +} diff --git a/pkg/handler/collector/s3/messaging/message.go b/pkg/handler/collector/s3/messaging/message.go new file mode 100644 index 0000000000..2f664d1626 --- /dev/null +++ b/pkg/handler/collector/s3/messaging/message.go @@ -0,0 +1,67 @@ +// +// Copyright 2023 The GUAC Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package messaging + +import "context" + +type EventName string + +const ( + PUT EventName = "PUT" +) + +// Message A generic message related to an S3 bucket and item +type Message interface { + GetEvent() (EventName, error) + GetBucket() (string, error) + GetItem() (string, error) +} + +// MessageProvider Reads and returns messages from a given queue (or topic) +type MessageProvider interface { + ReceiveMessage(ctx context.Context) (Message, error) + Close(ctx context.Context) error +} + +type MessageProviderConfig struct { + Queue string + Host string + Port string + Provider string + Region string +} + +// MessageProviderBuilder Returns a builder for a MessageProvider +type MessageProviderBuilder interface { + GetMessageProvider(config MessageProviderConfig) (MessageProvider, error) +} + +type MpBuilder struct { +} + +// GetMessageProvider Returns a MessageProvider with the given config. Defaults to Kafka provider if no MESSAGE_PROVIDER environment variable is found +func (mb MpBuilder) GetMessageProvider(config MessageProviderConfig) (MessageProvider, error) { + switch config.Provider { + case "sqs": + return NewSqsProvider(config) + default: + return NewKafkaProvider(config) + } +} + +func GetDefaultMessageProviderBuilder() (MessageProviderBuilder, error) { + return MpBuilder{}, nil +} diff --git a/pkg/handler/collector/s3/messaging/sqs.go b/pkg/handler/collector/s3/messaging/sqs.go new file mode 100644 index 0000000000..e8b927646b --- /dev/null +++ b/pkg/handler/collector/s3/messaging/sqs.go @@ -0,0 +1,156 @@ +// +// Copyright 2023 The GUAC Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package messaging + +import ( + "context" + "encoding/json" + "fmt" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/sqs" + "github.com/guacsec/guac/pkg/logging" +) + +type SqsProvider struct { + client *sqs.Client + hostname string + port string + queue string +} + +type SqsBucket struct { + Name string `json:"name"` +} + +type SqsObject struct { + Key string `json:"key"` +} + +type SqsS3 struct { + Object SqsObject `json:"object"` + Bucket SqsBucket `json:"bucket"` +} + +type SqsRecord struct { + EventName string `json:"eventName"` + S3 SqsS3 `json:"s3"` +} + +type SqsMessage struct { + Records []SqsRecord `json:"Records"` +} + +func (m SqsMessage) GetEvent() (EventName, error) { + if len(m.Records) == 0 { + return "", fmt.Errorf("error getting event from message %s", m) + } + + if m.Records[0].EventName == "ObjectCreated:Put" { + return PUT, nil + } + + return "", nil +} + +func (m SqsMessage) GetBucket() (string, error) { + if len(m.Records) == 0 { + return "", fmt.Errorf("error getting bucket from message %s", m) + } + return m.Records[0].S3.Bucket.Name, nil +} + +func (m SqsMessage) GetItem() (string, error) { + if len(m.Records) == 0 { + return "", fmt.Errorf("error getting item from message %s", m) + } + return m.Records[0].S3.Object.Key, nil +} + +func NewSqsProvider(mpConfig MessageProviderConfig) (SqsProvider, error) { + sqsHostname := mpConfig.Host + sqsPort := mpConfig.Port + sqsQueue := mpConfig.Queue + sqsProvider := SqsProvider{} + + cfg, err := config.LoadDefaultConfig(context.TODO()) + if err != nil { + return SqsProvider{}, fmt.Errorf("error loading AWS SDK config: %v", err) + } + + client := sqs.NewFromConfig(cfg, func(o *sqs.Options) { + o.BaseEndpoint = aws.String(fmt.Sprintf("http://%s:%s", sqsHostname, sqsPort)) + o.Region = mpConfig.Region + }) + + sqsProvider.client = client + sqsProvider.hostname = sqsHostname + sqsProvider.port = sqsPort + sqsProvider.queue = sqsQueue + + return sqsProvider, nil +} + +func (s SqsProvider) ReceiveMessage(ctx context.Context) (Message, error) { + logger := logging.FromContext(ctx) + + addr := fmt.Sprintf("http://%s:%s/000000000000/%s", s.hostname, s.port, s.queue) + + // Receive messages from the queue + receiveInput := &sqs.ReceiveMessageInput{ + QueueUrl: &addr, + MaxNumberOfMessages: 1, + WaitTimeSeconds: 10, + } + + for { + receiveOutput, err := s.client.ReceiveMessage(ctx, receiveInput) + if err != nil { + fmt.Printf("error receiving message, skipping: %v\n", err) + continue + } + + messages := receiveOutput.Messages + if len(messages) > 0 { + message := receiveOutput.Messages[0] + logger.Debugf("Received message: %v\n", *message.Body) + + var msg SqsMessage + err := json.Unmarshal([]byte(*message.Body), &msg) + if err != nil { + fmt.Println("Error:", err) + return SqsMessage{}, err + } + + // Delete the received message from the queue (stardard sqs procedure) + deleteInput := &sqs.DeleteMessageInput{ + QueueUrl: &addr, + ReceiptHandle: message.ReceiptHandle, + } + _, err = s.client.DeleteMessage(context.TODO(), deleteInput) + if err != nil { + logger.Errorf("error deleting message: %v\n", err) + } + logger.Debugf("Message deleted from the queue") + + return msg, nil + } + } +} + +func (s SqsProvider) Close(ctx context.Context) error { + return nil +} diff --git a/pkg/handler/collector/s3/s3.go b/pkg/handler/collector/s3/s3.go new file mode 100644 index 0000000000..868671e38b --- /dev/null +++ b/pkg/handler/collector/s3/s3.go @@ -0,0 +1,175 @@ +// +// Copyright 2023 The GUAC Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s3 + +import ( + "context" + "fmt" + "github.com/guacsec/guac/pkg/handler/collector/s3/bucket" + "github.com/guacsec/guac/pkg/handler/collector/s3/messaging" + "github.com/guacsec/guac/pkg/handler/processor" + "github.com/guacsec/guac/pkg/logging" + "os" + "strings" + "sync" +) + +const ( + S3CollectorType = "S3CollectorType" +) + +type S3Collector struct { + config S3CollectorConfig +} + +type S3CollectorConfig struct { + MessageProvider string + MessageProviderHost string + MessageProviderPort string + S3Host string + S3Port string + Region string // optional (defaults to us-east-1, assumes same region for s3 and sqs) + Queues string + MpBuilder messaging.MessageProviderBuilder // optional + BucketBuilder bucket.BuildBucket // optional + SigChan chan os.Signal // optional +} + +func NewS3Collector(cfg S3CollectorConfig) (*S3Collector, error) { + s3collector := &S3Collector{ + config: cfg, + } + return s3collector, nil +} + +func (s S3Collector) RetrieveArtifacts(ctx context.Context, docChannel chan<- *processor.Document) error { + logger := logging.FromContext(ctx) + queues := strings.Split(s.config.Queues, ",") + downloader := getDownloader(s) + sigChan := s.config.SigChan + + var wg sync.WaitGroup + for _, queue := range queues { + wg.Add(1) + + go func(queue string) { + mp, err := getMessageProvider(s, queue) + if err != nil { + logger.Errorf("error getting message provider for queue %v: %v", queue, err) + return + } + + for { + select { + case <-sigChan: + logger.Infof("Shutting down collector for queue %s...\n", queue) + break + default: + m, err := mp.ReceiveMessage(ctx) + if err != nil { + logger.Infof("error while receiving message, skipping: %v\n", err) + continue + } + + if e, er := m.GetEvent(); e != messaging.PUT { + if er != nil { + logger.Debugf("skipping message: %v\n", er) + } + continue + } + bucketName, err := m.GetBucket() + if err != nil { + logger.Errorf("skipping message: %v\n", err) + continue + } + item, err := m.GetItem() + if err != nil { + logger.Errorf("skipping message: %v\n", err) + } + + blob, err := downloader.DownloadFile(ctx, bucketName, item) + if err != nil { + logger.Errorf("could not download item %v, skipping: %v", item, err) + continue + } + + enc, err := downloader.GetEncoding(ctx, bucketName, item) + if err != nil { + logger.Errorf("could not get encoding for item %v, skipping: %v", item, err) + continue + } + + doc := &processor.Document{ + Blob: blob, + Type: processor.DocumentUnknown, + Format: processor.FormatUnknown, + Encoding: bucket.ExtractEncoding(enc, item), + SourceInformation: processor.SourceInformation{ + Collector: S3CollectorType, + Source: "S3", + }, + } + docChannel <- doc + } + } + + }(queue) + } + + wg.Wait() + + return nil +} + +func getMessageProvider(s S3Collector, queue string) (messaging.MessageProvider, error) { + var err error + var mpBuilder messaging.MessageProviderBuilder + if s.config.MpBuilder != nil { + mpBuilder = s.config.MpBuilder + } else { + mpBuilder, err = messaging.GetDefaultMessageProviderBuilder() + if err != nil { + return nil, fmt.Errorf("error getting message provider: %v", err) + } + } + + mp, err := mpBuilder.GetMessageProvider(messaging.MessageProviderConfig{ + Queue: queue, + Host: s.config.MessageProviderHost, + Port: s.config.MessageProviderPort, + Provider: s.config.MessageProvider, + Region: s.config.Region, + }) + if err != nil { + return nil, fmt.Errorf("error creating message provider: %s", err) + } + + return mp, nil +} + +func getDownloader(s S3Collector) bucket.Bucket { + var downloader bucket.Bucket + if s.config.BucketBuilder != nil { + downloader = s.config.BucketBuilder.GetDownloader(s.config.S3Host, s.config.S3Port, s.config.Region) + } else { + downloader = bucket.GetDefaultBucket(s.config.S3Host, s.config.S3Port, s.config.Region) + } + return downloader +} + +func (s S3Collector) Type() string { + return S3CollectorType +} diff --git a/pkg/handler/collector/s3/s3_test.go b/pkg/handler/collector/s3/s3_test.go new file mode 100644 index 0000000000..61e5460aa3 --- /dev/null +++ b/pkg/handler/collector/s3/s3_test.go @@ -0,0 +1,171 @@ +// +// Copyright 2023 The GUAC Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s3 + +import ( + "bytes" + "context" + "errors" + "fmt" + "github.com/guacsec/guac/pkg/handler/collector" + "github.com/guacsec/guac/pkg/handler/collector/s3/bucket" + "github.com/guacsec/guac/pkg/handler/collector/s3/messaging" + "github.com/guacsec/guac/pkg/handler/processor" + "io" + "os" + "os/signal" + "strings" + "testing" + "time" +) + +// Test message +type TestMessage struct { + item string + bucket string + event messaging.EventName +} + +func (msg TestMessage) GetEvent() (messaging.EventName, error) { + return msg.event, nil +} + +func (msg TestMessage) GetBucket() (string, error) { + return msg.bucket, nil +} + +func (msg TestMessage) GetItem() (string, error) { + return msg.item, nil +} + +// Test Message Provider +type TestProvider struct { + queue string +} + +func NewTestProvider(queue string) TestProvider { + return TestProvider{queue} +} + +func (t TestProvider) ReceiveMessage(context.Context) (messaging.Message, error) { + fmt.Printf("returning message for queue %s\n", t.queue) + time.Sleep(2 * time.Second) + + return TestMessage{ + item: "item", + bucket: t.queue, + event: messaging.PUT, + }, nil +} + +func (t TestProvider) Close(ctx context.Context) error { + return nil +} + +// Test Message Provider builder +type TestMpBuilder struct { +} + +func (tb TestMpBuilder) GetMessageProvider(config messaging.MessageProviderConfig) (messaging.MessageProvider, error) { + return NewTestProvider(config.Queue), nil +} + +// Test Bucket +type TestBucket struct { +} + +func (td TestBucket) DownloadFile(ctx context.Context, bucket string, item string) ([]byte, error) { + fmt.Printf("downloading file with bucket %s and name %s\n", bucket, item) + return []byte{1, 2, 3}, nil +} + +func (td TestBucket) GetEncoding(ctx context.Context, bucket string, item string) (string, error) { + return "", nil +} + +type TestBucketBuilder struct { +} + +func (td TestBucketBuilder) GetDownloader(hostname string, port string, region string) bucket.Bucket { + return TestBucket{} +} + +func TestQueuesSplit(t *testing.T) { + ctx := context.Background() + + sigChan := make(chan os.Signal, 1) + s3Collector, _ := NewS3Collector(S3CollectorConfig{ + Queues: "q1,q2", + MpBuilder: TestMpBuilder{}, + BucketBuilder: TestBucketBuilder{}, + SigChan: sigChan, + }) + + if err := collector.RegisterDocumentCollector(s3Collector, S3CollectorType); err != nil && + !errors.Is(err, collector.ErrCollectorOverwrite) { + t.Fatalf("could not register collector: %v", err) + } + + oldStdout := os.Stdout + r, w, _ := os.Pipe() + os.Stdout = w + outC := make(chan string) + go func() { + var buf bytes.Buffer + _, err := io.Copy(&buf, r) + if err != nil { + return + } + outC <- buf.String() + }() + + var s []*processor.Document + em := func(d *processor.Document) error { + s = append(s, d) + return nil + } + eh := func(err error) bool { + return true + } + + go func() { + err := collector.Collect(ctx, em, eh) + if err != nil { + fmt.Printf("error collecting: %v", err) + } + }() + time.Sleep(5 * time.Second) + signal.Notify(sigChan, os.Interrupt) + + w.Close() + os.Stdout = oldStdout // restoring the real stdout + out := <-outC + + fmt.Println(out) + + if !strings.Contains(out, "returning message for queue q1") { + t.Errorf("message for q1 not returned") + } + if !strings.Contains(out, "returning message for queue q2") { + t.Errorf("message for q2 not returned") + } + if !strings.Contains(out, "downloading file with bucket q1 and name item") { + t.Errorf("not downloading from bucket q1") + } + if !strings.Contains(out, "downloading file with bucket q2 and name item") { + t.Errorf("not downloading from bucket q2") + } +} From c6efb2d91f4ff7cbdb8fab6f9121c0685f066c6f Mon Sep 17 00:00:00 2001 From: Juan Manuel Leflet Estrada Date: Fri, 22 Sep 2023 14:33:52 +0200 Subject: [PATCH 07/21] goimport files Signed-off-by: Juan Manuel Leflet Estrada --- pkg/handler/collector/s3/bucket/bucket.go | 3 ++- pkg/handler/collector/s3/bucket/encoding.go | 3 ++- pkg/handler/collector/s3/messaging/kafka.go | 3 ++- pkg/handler/collector/s3/messaging/sqs.go | 1 + pkg/handler/collector/s3/s3.go | 7 ++++--- pkg/handler/collector/s3/s3_test.go | 9 +++++---- 6 files changed, 16 insertions(+), 10 deletions(-) diff --git a/pkg/handler/collector/s3/bucket/bucket.go b/pkg/handler/collector/s3/bucket/bucket.go index 8838177e6c..bba093f72c 100644 --- a/pkg/handler/collector/s3/bucket/bucket.go +++ b/pkg/handler/collector/s3/bucket/bucket.go @@ -19,10 +19,11 @@ import ( "bytes" "context" "fmt" + "io" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/s3" - "io" ) type BuildBucket interface { diff --git a/pkg/handler/collector/s3/bucket/encoding.go b/pkg/handler/collector/s3/bucket/encoding.go index 17a3683217..efb35b6e20 100644 --- a/pkg/handler/collector/s3/bucket/encoding.go +++ b/pkg/handler/collector/s3/bucket/encoding.go @@ -16,8 +16,9 @@ package bucket import ( - "github.com/guacsec/guac/pkg/handler/processor" "strings" + + "github.com/guacsec/guac/pkg/handler/processor" ) const ( diff --git a/pkg/handler/collector/s3/messaging/kafka.go b/pkg/handler/collector/s3/messaging/kafka.go index 8fcbe7f11f..d6208ee1e3 100644 --- a/pkg/handler/collector/s3/messaging/kafka.go +++ b/pkg/handler/collector/s3/messaging/kafka.go @@ -19,9 +19,10 @@ import ( "context" "encoding/json" "fmt" + "strings" + "github.com/guacsec/guac/pkg/logging" "github.com/segmentio/kafka-go" - "strings" ) type KafkaProvider struct { diff --git a/pkg/handler/collector/s3/messaging/sqs.go b/pkg/handler/collector/s3/messaging/sqs.go index e8b927646b..f28bbdc312 100644 --- a/pkg/handler/collector/s3/messaging/sqs.go +++ b/pkg/handler/collector/s3/messaging/sqs.go @@ -19,6 +19,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/sqs" diff --git a/pkg/handler/collector/s3/s3.go b/pkg/handler/collector/s3/s3.go index 868671e38b..76c6e98fd0 100644 --- a/pkg/handler/collector/s3/s3.go +++ b/pkg/handler/collector/s3/s3.go @@ -18,13 +18,14 @@ package s3 import ( "context" "fmt" + "os" + "strings" + "sync" + "github.com/guacsec/guac/pkg/handler/collector/s3/bucket" "github.com/guacsec/guac/pkg/handler/collector/s3/messaging" "github.com/guacsec/guac/pkg/handler/processor" "github.com/guacsec/guac/pkg/logging" - "os" - "strings" - "sync" ) const ( diff --git a/pkg/handler/collector/s3/s3_test.go b/pkg/handler/collector/s3/s3_test.go index 61e5460aa3..34fd90ff40 100644 --- a/pkg/handler/collector/s3/s3_test.go +++ b/pkg/handler/collector/s3/s3_test.go @@ -20,16 +20,17 @@ import ( "context" "errors" "fmt" - "github.com/guacsec/guac/pkg/handler/collector" - "github.com/guacsec/guac/pkg/handler/collector/s3/bucket" - "github.com/guacsec/guac/pkg/handler/collector/s3/messaging" - "github.com/guacsec/guac/pkg/handler/processor" "io" "os" "os/signal" "strings" "testing" "time" + + "github.com/guacsec/guac/pkg/handler/collector" + "github.com/guacsec/guac/pkg/handler/collector/s3/bucket" + "github.com/guacsec/guac/pkg/handler/collector/s3/messaging" + "github.com/guacsec/guac/pkg/handler/processor" ) // Test message From ef06aaa40da720cef02f5392b33274eb6ee518ab Mon Sep 17 00:00:00 2001 From: Juan Manuel Leflet Estrada Date: Mon, 25 Sep 2023 17:58:42 +0200 Subject: [PATCH 08/21] improve signal and error handling, correctly handle flags Signed-off-by: Juan Manuel Leflet Estrada --- cmd/guacone/cmd/s3.go | 51 ++++++++++------ pkg/cli/store.go | 9 +++ pkg/handler/collector/s3/bucket/bucket.go | 14 ++--- pkg/handler/collector/s3/messaging/kafka.go | 6 +- pkg/handler/collector/s3/messaging/message.go | 4 +- pkg/handler/collector/s3/messaging/sqs.go | 59 ++++++++++--------- pkg/handler/collector/s3/s3.go | 40 +++++++++---- pkg/handler/collector/s3/s3_test.go | 5 +- 8 files changed, 118 insertions(+), 70 deletions(-) diff --git a/cmd/guacone/cmd/s3.go b/cmd/guacone/cmd/s3.go index b1d2f1bf1f..e219c70864 100644 --- a/cmd/guacone/cmd/s3.go +++ b/cmd/guacone/cmd/s3.go @@ -19,7 +19,10 @@ import ( "context" "fmt" "os" + "os/signal" + "syscall" + "github.com/guacsec/guac/pkg/cli" csub_client "github.com/guacsec/guac/pkg/collectsub/client" "github.com/guacsec/guac/pkg/handler/collector" "github.com/guacsec/guac/pkg/handler/collector/s3" @@ -28,7 +31,6 @@ import ( "github.com/guacsec/guac/pkg/logging" "github.com/spf13/cobra" "github.com/spf13/viper" - "golang.org/x/sync/errgroup" ) // s3Options flags for configuring the command @@ -52,12 +54,26 @@ var s3Cmd = &cobra.Command{ ctx := logging.WithLogger(context.Background()) logger := logging.FromContext(ctx) - s3Opts, err := validateS3Opts(s3Opts.s3hostname, s3Opts.s3port, s3Opts.mp, s3Opts.mpHostname, s3Opts.mpPort, s3Opts.queues, s3Opts.region) + //s3Opts, err := validateS3Opts(s3Opts.s3hostname, s3Opts.s3port, s3Opts.mp, s3Opts.mpHostname, s3Opts.mpPort, s3Opts.queues, s3Opts.region) + s3Opts, err := validateS3Opts( + viper.GetString("s3-host"), + viper.GetString("s3-port"), + viper.GetString("s3-mp"), + viper.GetString("s3-mp-host"), + viper.GetString("s3-mp-port"), + viper.GetString("s3-queues"), + viper.GetString("s3-region"), + ) if err != nil { + logger.Errorf("failed to validate flags: %v", err) + _ = cmd.Help() os.Exit(1) } - s3Collector, err := s3.NewS3Collector(s3.S3CollectorConfig{ + signals := make(chan os.Signal, 1) + signal.Notify(signals, syscall.SIGINT) + + s3Collector := s3.NewS3Collector(s3.S3CollectorConfig{ S3Host: s3Opts.s3hostname, S3Port: s3Opts.s3port, MessageProvider: s3Opts.mp, @@ -65,14 +81,10 @@ var s3Cmd = &cobra.Command{ MessageProviderPort: s3Opts.mpPort, Queues: s3Opts.queues, Region: s3Opts.region, + SigChan: signals, }) - if err != nil { - logger.Errorf("unable to create s3 collector: %v", err) - os.Exit(1) - } - err = collector.RegisterDocumentCollector(s3Collector, s3.S3CollectorType) - if err != nil { + if err := collector.RegisterDocumentCollector(s3Collector, s3.S3CollectorType); err != nil { logger.Errorf("unable to register s3 collector: %v\n", err) os.Exit(1) } @@ -87,9 +99,8 @@ var s3Cmd = &cobra.Command{ errFound := false - _, s3ctx := errgroup.WithContext(ctx) emit := func(d *processor.Document) error { - err := ingestor.Ingest(s3ctx, d, viper.GetString("gql-addr"), csubClient) + err := ingestor.Ingest(ctx, d, viper.GetString("gql-addr"), csubClient) if err != nil { errFound = true @@ -143,12 +154,16 @@ func validateS3Opts(s3hostname string, s3port string, mp string, mpHostname stri } func init() { - s3Cmd.Flags().StringVarP(&s3Opts.s3hostname, "s3-host", "", "", "hostname for s3") - s3Cmd.Flags().StringVarP(&s3Opts.s3port, "s3-port", "", "", "port for s3") - s3Cmd.Flags().StringVarP(&s3Opts.mp, "mp", "m", "", "message provider (sqs or kafka, defaults to kafka)") - s3Cmd.Flags().StringVarP(&s3Opts.mpHostname, "mp-host", "", "", "hostname for the message provider (sqs/kafka)") - s3Cmd.Flags().StringVarP(&s3Opts.mpPort, "mp-port", "", "", "port for the message provider (sqs/kafka)") - s3Cmd.Flags().StringVarP(&s3Opts.queues, "queues", "", "", "comma-separated list of queue/topic names") - s3Cmd.Flags().StringVarP(&s3Opts.region, "region", "", "us-east-1", "aws region, defaults to us-east-1") + set, err := cli.BuildFlags([]string{"s3-host", "s3-port", "s3-mp", "s3-mp-host", "s3-mp-port", "s3-queues", "s3-region"}) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to setup flag: %s", err) + os.Exit(1) + } + s3Cmd.Flags().AddFlagSet(set) + if err := viper.BindPFlags(s3Cmd.Flags()); err != nil { + fmt.Fprintf(os.Stderr, "failed to bind flags: %s", err) + os.Exit(1) + } + collectCmd.AddCommand(s3Cmd) } diff --git a/pkg/cli/store.go b/pkg/cli/store.go index d1ac9103e4..440b4cf063 100644 --- a/pkg/cli/store.go +++ b/pkg/cli/store.go @@ -87,6 +87,15 @@ func init() { // Google Cloud platform flags set.String("gcp-credentials-path", "", "Path to the Google Cloud service account credentials json file.\nAlternatively you can set GOOGLE_APPLICATION_CREDENTIALS= in your environment.") + // S3 flags + set.String("s3-host", "", "hostname for the s3 provider") + set.String("s3-port", "", "port for the s3 provider") + set.String("s3-mp", "", "message provider (sqs or kafka, defaults to kafka)") + set.String("s3-mp-host", "", "hostname for the message provider (sqs/kafka)") + set.String("s3-mp-port", "", "port for the message provider (sqs/kafka)") + set.String("s3-queues", "", "comma-separated list of queue/topic names") + set.String("s3-region", "us-east-1", "aws region, defaults to us-east-1") + set.VisitAll(func(f *pflag.Flag) { flagStore[f.Name] = f }) diff --git a/pkg/handler/collector/s3/bucket/bucket.go b/pkg/handler/collector/s3/bucket/bucket.go index bba093f72c..39aff440c7 100644 --- a/pkg/handler/collector/s3/bucket/bucket.go +++ b/pkg/handler/collector/s3/bucket/bucket.go @@ -59,7 +59,7 @@ func GetDefaultBucket(hostname string, port string, region string) Bucket { func (d S3Bucket) DownloadFile(ctx context.Context, bucket string, item string) ([]byte, error) { cfg, err := config.LoadDefaultConfig(ctx) if err != nil { - return nil, fmt.Errorf("error loading AWS SDK config: %v", err) + return nil, fmt.Errorf("error loading AWS SDK config: %w", err) } addr := fmt.Sprintf("http://%s:%s/%s/", d.hostname, d.port, bucket) @@ -77,23 +77,23 @@ func (d S3Bucket) DownloadFile(ctx context.Context, bucket string, item string) resp, err := client.GetObject(ctx, input) if err != nil { - return nil, fmt.Errorf("unable to download file: %v", err) + return nil, fmt.Errorf("unable to download file: %w", err) } defer resp.Body.Close() buf := new(bytes.Buffer) n, err := io.Copy(buf, resp.Body) if err != nil || n == 0 { - return nil, fmt.Errorf("unable to read file contents: %v", err) + return nil, fmt.Errorf("unable to read file contents: %w", err) } - return buf.Bytes(), err + return buf.Bytes(), nil } func (d S3Bucket) GetEncoding(ctx context.Context, bucket string, item string) (string, error) { - cfg, err := config.LoadDefaultConfig(context.TODO()) + cfg, err := config.LoadDefaultConfig(ctx) if err != nil { - return "", fmt.Errorf("error loading AWS SDK config: %v", err) + return "", fmt.Errorf("error loading AWS SDK config: %w", err) } addr := fmt.Sprintf("http://%s:%s/%s/", d.hostname, d.port, bucket) @@ -105,7 +105,7 @@ func (d S3Bucket) GetEncoding(ctx context.Context, bucket string, item string) ( headObject, err := client.HeadObject(context.Background(), &s3.HeadObjectInput{Bucket: aws.String(bucket), Key: aws.String(item)}) if err != nil { - return "", fmt.Errorf("could not get head object: %v", err) + return "", fmt.Errorf("could not get head object: %w", err) } if headObject.ContentEncoding == nil { diff --git a/pkg/handler/collector/s3/messaging/kafka.go b/pkg/handler/collector/s3/messaging/kafka.go index d6208ee1e3..0267240d82 100644 --- a/pkg/handler/collector/s3/messaging/kafka.go +++ b/pkg/handler/collector/s3/messaging/kafka.go @@ -81,7 +81,7 @@ func NewKafkaProvider(mpConfig MessageProviderConfig) (KafkaProvider, error) { func (k KafkaProvider) ReceiveMessage(ctx context.Context) (Message, error) { logger := logging.FromContext(ctx) - m, err := k.reader.ReadMessage(context.Background()) + m, err := k.reader.ReadMessage(ctx) if err != nil { fmt.Println(err.Error()) } @@ -90,7 +90,7 @@ func (k KafkaProvider) ReceiveMessage(ctx context.Context) (Message, error) { msg := KafkaMessage{} err = json.Unmarshal(m.Value, &msg) if err != nil { - return msg, fmt.Errorf("error parsing JSON: %v", err) + return msg, fmt.Errorf("error parsing JSON: %w", err) } return msg, err @@ -100,7 +100,7 @@ func (k KafkaProvider) Close(ctx context.Context) error { logger := logging.FromContext(ctx) if err := k.reader.Close(); err != nil { - logger.Errorf("failed to close reader:", err) + logger.Errorf("failed to close reader: %v", err) return err } diff --git a/pkg/handler/collector/s3/messaging/message.go b/pkg/handler/collector/s3/messaging/message.go index 2f664d1626..43f7fbcda7 100644 --- a/pkg/handler/collector/s3/messaging/message.go +++ b/pkg/handler/collector/s3/messaging/message.go @@ -62,6 +62,6 @@ func (mb MpBuilder) GetMessageProvider(config MessageProviderConfig) (MessagePro } } -func GetDefaultMessageProviderBuilder() (MessageProviderBuilder, error) { - return MpBuilder{}, nil +func GetDefaultMessageProviderBuilder() MessageProviderBuilder { + return MpBuilder{} } diff --git a/pkg/handler/collector/s3/messaging/sqs.go b/pkg/handler/collector/s3/messaging/sqs.go index f28bbdc312..b665250c11 100644 --- a/pkg/handler/collector/s3/messaging/sqs.go +++ b/pkg/handler/collector/s3/messaging/sqs.go @@ -89,7 +89,7 @@ func NewSqsProvider(mpConfig MessageProviderConfig) (SqsProvider, error) { cfg, err := config.LoadDefaultConfig(context.TODO()) if err != nil { - return SqsProvider{}, fmt.Errorf("error loading AWS SDK config: %v", err) + return SqsProvider{}, fmt.Errorf("error loading AWS SDK config: %w", err) } client := sqs.NewFromConfig(cfg, func(o *sqs.Options) { @@ -118,36 +118,41 @@ func (s SqsProvider) ReceiveMessage(ctx context.Context) (Message, error) { } for { - receiveOutput, err := s.client.ReceiveMessage(ctx, receiveInput) - if err != nil { - fmt.Printf("error receiving message, skipping: %v\n", err) - continue - } - - messages := receiveOutput.Messages - if len(messages) > 0 { - message := receiveOutput.Messages[0] - logger.Debugf("Received message: %v\n", *message.Body) - - var msg SqsMessage - err := json.Unmarshal([]byte(*message.Body), &msg) + select { + case <-ctx.Done(): + return SqsMessage{}, nil + default: + receiveOutput, err := s.client.ReceiveMessage(ctx, receiveInput) if err != nil { - fmt.Println("Error:", err) - return SqsMessage{}, err + fmt.Printf("error receiving message, skipping: %v\n", err) + continue } - // Delete the received message from the queue (stardard sqs procedure) - deleteInput := &sqs.DeleteMessageInput{ - QueueUrl: &addr, - ReceiptHandle: message.ReceiptHandle, + messages := receiveOutput.Messages + if len(messages) > 0 { + message := receiveOutput.Messages[0] + logger.Debugf("Received message: %v\n", *message.Body) + + var msg SqsMessage + err := json.Unmarshal([]byte(*message.Body), &msg) + if err != nil { + logger.Errorf("error unmarshalling message: %v", err) + return SqsMessage{}, err + } + + // Delete the received message from the queue (stardard sqs procedure) + deleteInput := &sqs.DeleteMessageInput{ + QueueUrl: &addr, + ReceiptHandle: message.ReceiptHandle, + } + _, err = s.client.DeleteMessage(context.TODO(), deleteInput) + if err != nil { + logger.Errorf("error deleting message: %v\n", err) + } + logger.Debugf("Message deleted from the queue") + + return msg, nil } - _, err = s.client.DeleteMessage(context.TODO(), deleteInput) - if err != nil { - logger.Errorf("error deleting message: %v\n", err) - } - logger.Debugf("Message deleted from the queue") - - return msg, nil } } } diff --git a/pkg/handler/collector/s3/s3.go b/pkg/handler/collector/s3/s3.go index 76c6e98fd0..ecc6f056cd 100644 --- a/pkg/handler/collector/s3/s3.go +++ b/pkg/handler/collector/s3/s3.go @@ -49,11 +49,11 @@ type S3CollectorConfig struct { SigChan chan os.Signal // optional } -func NewS3Collector(cfg S3CollectorConfig) (*S3Collector, error) { +func NewS3Collector(cfg S3CollectorConfig) *S3Collector { s3collector := &S3Collector{ config: cfg, } - return s3collector, nil + return s3collector } func (s S3Collector) RetrieveArtifacts(ctx context.Context, docChannel chan<- *processor.Document) error { @@ -62,11 +62,28 @@ func (s S3Collector) RetrieveArtifacts(ctx context.Context, docChannel chan<- *p downloader := getDownloader(s) sigChan := s.config.SigChan + cancelCtx, cancel := context.WithCancel(ctx) + defer cancel() + + // Send cancellation in case of receiving SIGINT + go func(cancel context.CancelFunc) { + <-sigChan + cancel() + }(cancel) + var wg sync.WaitGroup for _, queue := range queues { wg.Add(1) - go func(queue string) { + go func(wg *sync.WaitGroup, cncCtx context.Context, queue string) { + defer wg.Done() + + defer func() { + if r := recover(); r != nil { + fmt.Println("recovered from panic:", r) + } + }() + mp, err := getMessageProvider(s, queue) if err != nil { logger.Errorf("error getting message provider for queue %v: %v", queue, err) @@ -75,11 +92,11 @@ func (s S3Collector) RetrieveArtifacts(ctx context.Context, docChannel chan<- *p for { select { - case <-sigChan: + case <-cncCtx.Done(): logger.Infof("Shutting down collector for queue %s...\n", queue) - break + return default: - m, err := mp.ReceiveMessage(ctx) + m, err := mp.ReceiveMessage(cncCtx) if err != nil { logger.Infof("error while receiving message, skipping: %v\n", err) continue @@ -99,15 +116,16 @@ func (s S3Collector) RetrieveArtifacts(ctx context.Context, docChannel chan<- *p item, err := m.GetItem() if err != nil { logger.Errorf("skipping message: %v\n", err) + continue } - blob, err := downloader.DownloadFile(ctx, bucketName, item) + blob, err := downloader.DownloadFile(cncCtx, bucketName, item) if err != nil { logger.Errorf("could not download item %v, skipping: %v", item, err) continue } - enc, err := downloader.GetEncoding(ctx, bucketName, item) + enc, err := downloader.GetEncoding(cncCtx, bucketName, item) if err != nil { logger.Errorf("could not get encoding for item %v, skipping: %v", item, err) continue @@ -127,7 +145,7 @@ func (s S3Collector) RetrieveArtifacts(ctx context.Context, docChannel chan<- *p } } - }(queue) + }(&wg, cancelCtx, queue) } wg.Wait() @@ -141,9 +159,9 @@ func getMessageProvider(s S3Collector, queue string) (messaging.MessageProvider, if s.config.MpBuilder != nil { mpBuilder = s.config.MpBuilder } else { - mpBuilder, err = messaging.GetDefaultMessageProviderBuilder() + mpBuilder = messaging.GetDefaultMessageProviderBuilder() if err != nil { - return nil, fmt.Errorf("error getting message provider: %v", err) + return nil, fmt.Errorf("error getting message provider: %w", err) } } diff --git a/pkg/handler/collector/s3/s3_test.go b/pkg/handler/collector/s3/s3_test.go index 34fd90ff40..4492b38b8e 100644 --- a/pkg/handler/collector/s3/s3_test.go +++ b/pkg/handler/collector/s3/s3_test.go @@ -24,6 +24,7 @@ import ( "os" "os/signal" "strings" + "syscall" "testing" "time" @@ -108,7 +109,7 @@ func TestQueuesSplit(t *testing.T) { ctx := context.Background() sigChan := make(chan os.Signal, 1) - s3Collector, _ := NewS3Collector(S3CollectorConfig{ + s3Collector := NewS3Collector(S3CollectorConfig{ Queues: "q1,q2", MpBuilder: TestMpBuilder{}, BucketBuilder: TestBucketBuilder{}, @@ -149,7 +150,7 @@ func TestQueuesSplit(t *testing.T) { } }() time.Sleep(5 * time.Second) - signal.Notify(sigChan, os.Interrupt) + signal.Notify(sigChan, syscall.SIGINT) w.Close() os.Stdout = oldStdout // restoring the real stdout From 04801e8a522cf3da733d3384ed96e99888d67e0d Mon Sep 17 00:00:00 2001 From: Juan Manuel Leflet Estrada Date: Wed, 27 Sep 2023 13:11:41 +0200 Subject: [PATCH 09/21] add non-polling behaviour, make it default Signed-off-by: Juan Manuel Leflet Estrada --- cmd/guacone/cmd/s3.go | 49 ++++++++++++++++++--------- pkg/cli/store.go | 11 ++++--- pkg/handler/collector/s3/s3.go | 51 +++++++++++++++++++++++++++-- pkg/handler/collector/s3/s3_test.go | 48 ++++++++++++++++++++++++++- 4 files changed, 135 insertions(+), 24 deletions(-) diff --git a/cmd/guacone/cmd/s3.go b/cmd/guacone/cmd/s3.go index e219c70864..2979f49cd2 100644 --- a/cmd/guacone/cmd/s3.go +++ b/cmd/guacone/cmd/s3.go @@ -37,11 +37,14 @@ import ( type s3Options struct { s3hostname string // hostname of the s3 provider s3port string // port of the s3 provider + s3bucket string // s3 bucket (only for non-polling behaviour) + s3item string // s3 item (only for non-polling behaviour) region string // AWS region, for s3/sqs configuration (defaults to us-east-1) - queues string // comma-separated list of queues/topics + queues string // comma-separated list of queues/topics (only for polling behaviour) mp string // message provider name (sqs or kafka, will default to kafka) - mpHostname string // hostname for the message provider - mpPort string // port for the message provider + mpHostname string // hostname for the message provider (only for polling behaviour) + mpPort string // port for the message provider (only for polling behaviour) + poll bool // polling or non-polling behaviour? (defaults to non-polling) } var s3Opts s3Options @@ -54,15 +57,17 @@ var s3Cmd = &cobra.Command{ ctx := logging.WithLogger(context.Background()) logger := logging.FromContext(ctx) - //s3Opts, err := validateS3Opts(s3Opts.s3hostname, s3Opts.s3port, s3Opts.mp, s3Opts.mpHostname, s3Opts.mpPort, s3Opts.queues, s3Opts.region) s3Opts, err := validateS3Opts( viper.GetString("s3-host"), viper.GetString("s3-port"), + viper.GetString("s3-bucket"), + viper.GetString("s3-item"), viper.GetString("s3-mp"), viper.GetString("s3-mp-host"), viper.GetString("s3-mp-port"), viper.GetString("s3-queues"), viper.GetString("s3-region"), + viper.GetBool("s3-poll"), ) if err != nil { logger.Errorf("failed to validate flags: %v", err) @@ -71,17 +76,20 @@ var s3Cmd = &cobra.Command{ } signals := make(chan os.Signal, 1) - signal.Notify(signals, syscall.SIGINT) + signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM) s3Collector := s3.NewS3Collector(s3.S3CollectorConfig{ S3Host: s3Opts.s3hostname, S3Port: s3Opts.s3port, + S3Bucket: s3Opts.s3bucket, + S3Item: s3Opts.s3item, MessageProvider: s3Opts.mp, MessageProviderHost: s3Opts.mpHostname, MessageProviderPort: s3Opts.mpPort, Queues: s3Opts.queues, Region: s3Opts.region, SigChan: signals, + Poll: s3Opts.poll, }) if err := collector.RegisterDocumentCollector(s3Collector, s3.S3CollectorType); err != nil { @@ -130,7 +138,7 @@ var s3Cmd = &cobra.Command{ }, } -func validateS3Opts(s3hostname string, s3port string, mp string, mpHostname string, mpPort string, queues string, region string) (s3Options, error) { +func validateS3Opts(s3hostname string, s3port string, s3bucket string, s3item string, mp string, mpHostname string, mpPort string, queues string, region string, poll bool) (s3Options, error) { var opts s3Options if len(s3hostname) == 0 { return opts, fmt.Errorf("expected s3 hostname") @@ -138,23 +146,32 @@ func validateS3Opts(s3hostname string, s3port string, mp string, mpHostname stri if len(s3port) == 0 { return opts, fmt.Errorf("expected s3 port") } - if len(mpHostname) == 0 { - return opts, fmt.Errorf("expected hostname for message provider") - } - if len(mpPort) == 0 { - return opts, fmt.Errorf("expected port for message provider") - } - if len(queues) == 0 { - return opts, fmt.Errorf("expected at least one queue") + if poll { + if len(mpHostname) == 0 { + return opts, fmt.Errorf("expected hostname for message provider") + } + if len(mpPort) == 0 { + return opts, fmt.Errorf("expected port for message provider") + } + if len(queues) == 0 { + return opts, fmt.Errorf("expected at least one queue") + } + } else { + if len(s3bucket) == 0 { + return opts, fmt.Errorf("expected s3 bucket") + } + if len(s3item) == 0 { + return opts, fmt.Errorf("expected s3 item") + } } - opts = s3Options{s3hostname, s3port, region, queues, mp, mpHostname, mpPort} + opts = s3Options{s3hostname, s3port, s3bucket, s3item, region, queues, mp, mpHostname, mpPort, poll} return opts, nil } func init() { - set, err := cli.BuildFlags([]string{"s3-host", "s3-port", "s3-mp", "s3-mp-host", "s3-mp-port", "s3-queues", "s3-region"}) + set, err := cli.BuildFlags([]string{"s3-host", "s3-port", "s3-bucket", "s3-item", "s3-mp", "s3-mp-host", "s3-mp-port", "s3-queues", "s3-region", "s3-poll"}) if err != nil { fmt.Fprintf(os.Stderr, "failed to setup flag: %s", err) os.Exit(1) diff --git a/pkg/cli/store.go b/pkg/cli/store.go index 440b4cf063..18e6f720e5 100644 --- a/pkg/cli/store.go +++ b/pkg/cli/store.go @@ -90,11 +90,14 @@ func init() { // S3 flags set.String("s3-host", "", "hostname for the s3 provider") set.String("s3-port", "", "port for the s3 provider") - set.String("s3-mp", "", "message provider (sqs or kafka, defaults to kafka)") - set.String("s3-mp-host", "", "hostname for the message provider (sqs/kafka)") - set.String("s3-mp-port", "", "port for the message provider (sqs/kafka)") + set.String("s3-bucket", "", "bucket in the s3 provider") + set.String("s3-item", "", "item in the s3 provider") + set.String("s3-mp", "kafka", "message provider (sqs or kafka)") + set.String("s3-mp-host", "", "hostname for the message provider") + set.String("s3-mp-port", "", "port for the message provider") set.String("s3-queues", "", "comma-separated list of queue/topic names") - set.String("s3-region", "us-east-1", "aws region, defaults to us-east-1") + set.String("s3-region", "us-east-1", "aws region") + set.Bool("s3-poll", false, "polling or non-polling behaviour") set.VisitAll(func(f *pflag.Flag) { flagStore[f.Name] = f diff --git a/pkg/handler/collector/s3/s3.go b/pkg/handler/collector/s3/s3.go index ecc6f056cd..2326be9b3a 100644 --- a/pkg/handler/collector/s3/s3.go +++ b/pkg/handler/collector/s3/s3.go @@ -42,11 +42,14 @@ type S3CollectorConfig struct { MessageProviderPort string S3Host string S3Port string + S3Bucket string + S3Item string Region string // optional (defaults to us-east-1, assumes same region for s3 and sqs) Queues string MpBuilder messaging.MessageProviderBuilder // optional BucketBuilder bucket.BuildBucket // optional SigChan chan os.Signal // optional + Poll bool } func NewS3Collector(cfg S3CollectorConfig) *S3Collector { @@ -57,9 +60,53 @@ func NewS3Collector(cfg S3CollectorConfig) *S3Collector { } func (s S3Collector) RetrieveArtifacts(ctx context.Context, docChannel chan<- *processor.Document) error { + + if s.config.Poll { + retrieveWithPoll(s, ctx, docChannel) + } else { + retrieve(s, ctx, docChannel) + } + + return nil +} + +func retrieve(s S3Collector, ctx context.Context, docChannel chan<- *processor.Document) { logger := logging.FromContext(ctx) - queues := strings.Split(s.config.Queues, ",") downloader := getDownloader(s) + + bckt := s.config.S3Bucket + item := s.config.S3Item + + blob, err := downloader.DownloadFile(ctx, bckt, item) + if err != nil { + logger.Errorf("could not download item %v: %v", item, err) + return + } + + enc, err := downloader.GetEncoding(ctx, bckt, item) + if err != nil { + logger.Errorf("could not get encoding for item %v: %v", item, err) + return + } + + doc := &processor.Document{ + Blob: blob, + Type: processor.DocumentUnknown, + Format: processor.FormatUnknown, + Encoding: bucket.ExtractEncoding(enc, item), + SourceInformation: processor.SourceInformation{ + Collector: S3CollectorType, + Source: "S3", + }, + } + docChannel <- doc + +} + +func retrieveWithPoll(s S3Collector, ctx context.Context, docChannel chan<- *processor.Document) { + logger := logging.FromContext(ctx) + downloader := getDownloader(s) + queues := strings.Split(s.config.Queues, ",") sigChan := s.config.SigChan cancelCtx, cancel := context.WithCancel(ctx) @@ -149,8 +196,6 @@ func (s S3Collector) RetrieveArtifacts(ctx context.Context, docChannel chan<- *p } wg.Wait() - - return nil } func getMessageProvider(s S3Collector, queue string) (messaging.MessageProvider, error) { diff --git a/pkg/handler/collector/s3/s3_test.go b/pkg/handler/collector/s3/s3_test.go index 4492b38b8e..c98de31d0b 100644 --- a/pkg/handler/collector/s3/s3_test.go +++ b/pkg/handler/collector/s3/s3_test.go @@ -105,7 +105,7 @@ func (td TestBucketBuilder) GetDownloader(hostname string, port string, region s return TestBucket{} } -func TestQueuesSplit(t *testing.T) { +func TestQueuesSplitPolling(t *testing.T) { ctx := context.Background() sigChan := make(chan os.Signal, 1) @@ -114,6 +114,7 @@ func TestQueuesSplit(t *testing.T) { MpBuilder: TestMpBuilder{}, BucketBuilder: TestBucketBuilder{}, SigChan: sigChan, + Poll: true, }) if err := collector.RegisterDocumentCollector(s3Collector, S3CollectorType); err != nil && @@ -121,6 +122,7 @@ func TestQueuesSplit(t *testing.T) { t.Fatalf("could not register collector: %v", err) } + // redirect stdout to read it oldStdout := os.Stdout r, w, _ := os.Pipe() os.Stdout = w @@ -134,6 +136,7 @@ func TestQueuesSplit(t *testing.T) { outC <- buf.String() }() + // create fake emitter and handler var s []*processor.Document em := func(d *processor.Document) error { s = append(s, d) @@ -143,13 +146,17 @@ func TestQueuesSplit(t *testing.T) { return true } + // spawn collector go func() { err := collector.Collect(ctx, em, eh) if err != nil { fmt.Printf("error collecting: %v", err) } }() + + // wait for a while to get some messages time.Sleep(5 * time.Second) + // shut down collector signal.Notify(sigChan, syscall.SIGINT) w.Close() @@ -171,3 +178,42 @@ func TestQueuesSplit(t *testing.T) { t.Errorf("not downloading from bucket q2") } } + +func TestNoPolling(t *testing.T) { + ctx := context.Background() + + s3Collector := NewS3Collector(S3CollectorConfig{ + BucketBuilder: TestBucketBuilder{}, + S3Bucket: "no-poll-bucket", + S3Item: "no-poll-item", + Poll: false, + }) + + if err := collector.RegisterDocumentCollector(s3Collector, S3CollectorType); err != nil && + !errors.Is(err, collector.ErrCollectorOverwrite) { + t.Fatalf("could not register collector: %v", err) + } + + // create fake emitter and handler + var s []*processor.Document + em := func(d *processor.Document) error { + s = append(s, d) + return nil + } + eh := func(err error) bool { + return true + } + + err := collector.Collect(ctx, em, eh) + if err != nil { + fmt.Printf("error collecting: %v", err) + } + + if len(s) == 0 { + t.Errorf("no documents returned") + } + if len(s[0].Blob) != 3 { + t.Errorf("wrong document returned") + } + +} From 698e370570dd8fc9e6ee1645f632f13bf591eba1 Mon Sep 17 00:00:00 2001 From: Juan Manuel Leflet Estrada Date: Mon, 2 Oct 2023 10:23:56 +0200 Subject: [PATCH 10/21] Use pointer receivers Signed-off-by: Juan Manuel Leflet Estrada --- pkg/handler/collector/s3/bucket/bucket.go | 10 +++--- pkg/handler/collector/s3/messaging/kafka.go | 14 ++++----- pkg/handler/collector/s3/messaging/message.go | 10 +++--- pkg/handler/collector/s3/messaging/sqs.go | 16 +++++----- pkg/handler/collector/s3/s3.go | 8 ++--- pkg/handler/collector/s3/s3_test.go | 31 ++++++++++--------- 6 files changed, 46 insertions(+), 43 deletions(-) diff --git a/pkg/handler/collector/s3/bucket/bucket.go b/pkg/handler/collector/s3/bucket/bucket.go index 39aff440c7..b96fcf6ad5 100644 --- a/pkg/handler/collector/s3/bucket/bucket.go +++ b/pkg/handler/collector/s3/bucket/bucket.go @@ -33,8 +33,8 @@ type BuildBucket interface { type BucketBuilder struct { } -func (bd BucketBuilder) GetBucket(hostname string, port string, region string) Bucket { - return S3Bucket{ +func (bd *BucketBuilder) GetBucket(hostname string, port string, region string) Bucket { + return &S3Bucket{ hostname, port, region, @@ -53,10 +53,10 @@ type S3Bucket struct { } func GetDefaultBucket(hostname string, port string, region string) Bucket { - return S3Bucket{hostname, port, region} + return &S3Bucket{hostname, port, region} } -func (d S3Bucket) DownloadFile(ctx context.Context, bucket string, item string) ([]byte, error) { +func (d *S3Bucket) DownloadFile(ctx context.Context, bucket string, item string) ([]byte, error) { cfg, err := config.LoadDefaultConfig(ctx) if err != nil { return nil, fmt.Errorf("error loading AWS SDK config: %w", err) @@ -90,7 +90,7 @@ func (d S3Bucket) DownloadFile(ctx context.Context, bucket string, item string) return buf.Bytes(), nil } -func (d S3Bucket) GetEncoding(ctx context.Context, bucket string, item string) (string, error) { +func (d *S3Bucket) GetEncoding(ctx context.Context, bucket string, item string) (string, error) { cfg, err := config.LoadDefaultConfig(ctx) if err != nil { return "", fmt.Errorf("error loading AWS SDK config: %w", err) diff --git a/pkg/handler/collector/s3/messaging/kafka.go b/pkg/handler/collector/s3/messaging/kafka.go index 0267240d82..ed1ca38dbb 100644 --- a/pkg/handler/collector/s3/messaging/kafka.go +++ b/pkg/handler/collector/s3/messaging/kafka.go @@ -35,14 +35,14 @@ type KafkaMessage struct { Key string `json:"Key"` } -func (m KafkaMessage) GetEvent() (EventName, error) { +func (m *KafkaMessage) GetEvent() (EventName, error) { if m.EventName == "s3:ObjectCreated:Put" { return PUT, nil } return "", nil } -func (m KafkaMessage) GetBucket() (string, error) { +func (m *KafkaMessage) GetBucket() (string, error) { info := strings.Split(m.Key, "/") if len(info) < 2 { return "", fmt.Errorf("invalid format of key: %s", m.Key) @@ -50,7 +50,7 @@ func (m KafkaMessage) GetBucket() (string, error) { return info[0], nil } -func (m KafkaMessage) GetItem() (string, error) { +func (m *KafkaMessage) GetItem() (string, error) { info := strings.Split(m.Key, "/") if len(info) < 2 { return "", fmt.Errorf("invalid format of item: %s", m.Key) @@ -78,7 +78,7 @@ func NewKafkaProvider(mpConfig MessageProviderConfig) (KafkaProvider, error) { return kafkaProvider, nil } -func (k KafkaProvider) ReceiveMessage(ctx context.Context) (Message, error) { +func (k *KafkaProvider) ReceiveMessage(ctx context.Context) (Message, error) { logger := logging.FromContext(ctx) m, err := k.reader.ReadMessage(ctx) @@ -90,13 +90,13 @@ func (k KafkaProvider) ReceiveMessage(ctx context.Context) (Message, error) { msg := KafkaMessage{} err = json.Unmarshal(m.Value, &msg) if err != nil { - return msg, fmt.Errorf("error parsing JSON: %w", err) + return &msg, fmt.Errorf("error parsing JSON: %w", err) } - return msg, err + return &msg, err } -func (k KafkaProvider) Close(ctx context.Context) error { +func (k *KafkaProvider) Close(ctx context.Context) error { logger := logging.FromContext(ctx) if err := k.reader.Close(); err != nil { diff --git a/pkg/handler/collector/s3/messaging/message.go b/pkg/handler/collector/s3/messaging/message.go index 43f7fbcda7..719df6dc59 100644 --- a/pkg/handler/collector/s3/messaging/message.go +++ b/pkg/handler/collector/s3/messaging/message.go @@ -53,15 +53,17 @@ type MpBuilder struct { } // GetMessageProvider Returns a MessageProvider with the given config. Defaults to Kafka provider if no MESSAGE_PROVIDER environment variable is found -func (mb MpBuilder) GetMessageProvider(config MessageProviderConfig) (MessageProvider, error) { +func (mb *MpBuilder) GetMessageProvider(config MessageProviderConfig) (MessageProvider, error) { switch config.Provider { case "sqs": - return NewSqsProvider(config) + provider, err := NewSqsProvider(config) + return &provider, err default: - return NewKafkaProvider(config) + provider, err := NewKafkaProvider(config) + return &provider, err } } func GetDefaultMessageProviderBuilder() MessageProviderBuilder { - return MpBuilder{} + return &MpBuilder{} } diff --git a/pkg/handler/collector/s3/messaging/sqs.go b/pkg/handler/collector/s3/messaging/sqs.go index b665250c11..ac3fbec079 100644 --- a/pkg/handler/collector/s3/messaging/sqs.go +++ b/pkg/handler/collector/s3/messaging/sqs.go @@ -55,7 +55,7 @@ type SqsMessage struct { Records []SqsRecord `json:"Records"` } -func (m SqsMessage) GetEvent() (EventName, error) { +func (m *SqsMessage) GetEvent() (EventName, error) { if len(m.Records) == 0 { return "", fmt.Errorf("error getting event from message %s", m) } @@ -67,14 +67,14 @@ func (m SqsMessage) GetEvent() (EventName, error) { return "", nil } -func (m SqsMessage) GetBucket() (string, error) { +func (m *SqsMessage) GetBucket() (string, error) { if len(m.Records) == 0 { return "", fmt.Errorf("error getting bucket from message %s", m) } return m.Records[0].S3.Bucket.Name, nil } -func (m SqsMessage) GetItem() (string, error) { +func (m *SqsMessage) GetItem() (string, error) { if len(m.Records) == 0 { return "", fmt.Errorf("error getting item from message %s", m) } @@ -105,7 +105,7 @@ func NewSqsProvider(mpConfig MessageProviderConfig) (SqsProvider, error) { return sqsProvider, nil } -func (s SqsProvider) ReceiveMessage(ctx context.Context) (Message, error) { +func (s *SqsProvider) ReceiveMessage(ctx context.Context) (Message, error) { logger := logging.FromContext(ctx) addr := fmt.Sprintf("http://%s:%s/000000000000/%s", s.hostname, s.port, s.queue) @@ -120,7 +120,7 @@ func (s SqsProvider) ReceiveMessage(ctx context.Context) (Message, error) { for { select { case <-ctx.Done(): - return SqsMessage{}, nil + return &SqsMessage{}, nil default: receiveOutput, err := s.client.ReceiveMessage(ctx, receiveInput) if err != nil { @@ -137,7 +137,7 @@ func (s SqsProvider) ReceiveMessage(ctx context.Context) (Message, error) { err := json.Unmarshal([]byte(*message.Body), &msg) if err != nil { logger.Errorf("error unmarshalling message: %v", err) - return SqsMessage{}, err + return &SqsMessage{}, err } // Delete the received message from the queue (stardard sqs procedure) @@ -151,12 +151,12 @@ func (s SqsProvider) ReceiveMessage(ctx context.Context) (Message, error) { } logger.Debugf("Message deleted from the queue") - return msg, nil + return &msg, nil } } } } -func (s SqsProvider) Close(ctx context.Context) error { +func (s *SqsProvider) Close(ctx context.Context) error { return nil } diff --git a/pkg/handler/collector/s3/s3.go b/pkg/handler/collector/s3/s3.go index 2326be9b3a..e208ea5980 100644 --- a/pkg/handler/collector/s3/s3.go +++ b/pkg/handler/collector/s3/s3.go @@ -59,12 +59,12 @@ func NewS3Collector(cfg S3CollectorConfig) *S3Collector { return s3collector } -func (s S3Collector) RetrieveArtifacts(ctx context.Context, docChannel chan<- *processor.Document) error { +func (s *S3Collector) RetrieveArtifacts(ctx context.Context, docChannel chan<- *processor.Document) error { if s.config.Poll { - retrieveWithPoll(s, ctx, docChannel) + retrieveWithPoll(*s, ctx, docChannel) } else { - retrieve(s, ctx, docChannel) + retrieve(*s, ctx, docChannel) } return nil @@ -234,6 +234,6 @@ func getDownloader(s S3Collector) bucket.Bucket { return downloader } -func (s S3Collector) Type() string { +func (s *S3Collector) Type() string { return S3CollectorType } diff --git a/pkg/handler/collector/s3/s3_test.go b/pkg/handler/collector/s3/s3_test.go index c98de31d0b..2c26c4ee33 100644 --- a/pkg/handler/collector/s3/s3_test.go +++ b/pkg/handler/collector/s3/s3_test.go @@ -41,15 +41,15 @@ type TestMessage struct { event messaging.EventName } -func (msg TestMessage) GetEvent() (messaging.EventName, error) { +func (msg *TestMessage) GetEvent() (messaging.EventName, error) { return msg.event, nil } -func (msg TestMessage) GetBucket() (string, error) { +func (msg *TestMessage) GetBucket() (string, error) { return msg.bucket, nil } -func (msg TestMessage) GetItem() (string, error) { +func (msg *TestMessage) GetItem() (string, error) { return msg.item, nil } @@ -62,18 +62,18 @@ func NewTestProvider(queue string) TestProvider { return TestProvider{queue} } -func (t TestProvider) ReceiveMessage(context.Context) (messaging.Message, error) { +func (t *TestProvider) ReceiveMessage(context.Context) (messaging.Message, error) { fmt.Printf("returning message for queue %s\n", t.queue) time.Sleep(2 * time.Second) - return TestMessage{ + return &TestMessage{ item: "item", bucket: t.queue, event: messaging.PUT, }, nil } -func (t TestProvider) Close(ctx context.Context) error { +func (t *TestProvider) Close(ctx context.Context) error { return nil } @@ -81,28 +81,29 @@ func (t TestProvider) Close(ctx context.Context) error { type TestMpBuilder struct { } -func (tb TestMpBuilder) GetMessageProvider(config messaging.MessageProviderConfig) (messaging.MessageProvider, error) { - return NewTestProvider(config.Queue), nil +func (tb *TestMpBuilder) GetMessageProvider(config messaging.MessageProviderConfig) (messaging.MessageProvider, error) { + provider := NewTestProvider(config.Queue) + return &provider, nil } // Test Bucket type TestBucket struct { } -func (td TestBucket) DownloadFile(ctx context.Context, bucket string, item string) ([]byte, error) { +func (td *TestBucket) DownloadFile(ctx context.Context, bucket string, item string) ([]byte, error) { fmt.Printf("downloading file with bucket %s and name %s\n", bucket, item) return []byte{1, 2, 3}, nil } -func (td TestBucket) GetEncoding(ctx context.Context, bucket string, item string) (string, error) { +func (td *TestBucket) GetEncoding(ctx context.Context, bucket string, item string) (string, error) { return "", nil } type TestBucketBuilder struct { } -func (td TestBucketBuilder) GetDownloader(hostname string, port string, region string) bucket.Bucket { - return TestBucket{} +func (td *TestBucketBuilder) GetDownloader(hostname string, port string, region string) bucket.Bucket { + return &TestBucket{} } func TestQueuesSplitPolling(t *testing.T) { @@ -111,8 +112,8 @@ func TestQueuesSplitPolling(t *testing.T) { sigChan := make(chan os.Signal, 1) s3Collector := NewS3Collector(S3CollectorConfig{ Queues: "q1,q2", - MpBuilder: TestMpBuilder{}, - BucketBuilder: TestBucketBuilder{}, + MpBuilder: &TestMpBuilder{}, + BucketBuilder: &TestBucketBuilder{}, SigChan: sigChan, Poll: true, }) @@ -183,7 +184,7 @@ func TestNoPolling(t *testing.T) { ctx := context.Background() s3Collector := NewS3Collector(S3CollectorConfig{ - BucketBuilder: TestBucketBuilder{}, + BucketBuilder: &TestBucketBuilder{}, S3Bucket: "no-poll-bucket", S3Item: "no-poll-item", Poll: false, From c8345e02cb69da65faaca342ea062780a07d0abf Mon Sep 17 00:00:00 2001 From: Juan Manuel Leflet Estrada Date: Mon, 2 Oct 2023 10:29:39 +0200 Subject: [PATCH 11/21] Make struct private Signed-off-by: Juan Manuel Leflet Estrada --- pkg/handler/collector/s3/bucket/bucket.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/handler/collector/s3/bucket/bucket.go b/pkg/handler/collector/s3/bucket/bucket.go index b96fcf6ad5..d26aae50b4 100644 --- a/pkg/handler/collector/s3/bucket/bucket.go +++ b/pkg/handler/collector/s3/bucket/bucket.go @@ -34,7 +34,7 @@ type BucketBuilder struct { } func (bd *BucketBuilder) GetBucket(hostname string, port string, region string) Bucket { - return &S3Bucket{ + return &s3Bucket{ hostname, port, region, @@ -46,17 +46,17 @@ type Bucket interface { GetEncoding(ctx context.Context, bucket string, item string) (string, error) } -type S3Bucket struct { +type s3Bucket struct { hostname string port string region string } func GetDefaultBucket(hostname string, port string, region string) Bucket { - return &S3Bucket{hostname, port, region} + return &s3Bucket{hostname, port, region} } -func (d *S3Bucket) DownloadFile(ctx context.Context, bucket string, item string) ([]byte, error) { +func (d *s3Bucket) DownloadFile(ctx context.Context, bucket string, item string) ([]byte, error) { cfg, err := config.LoadDefaultConfig(ctx) if err != nil { return nil, fmt.Errorf("error loading AWS SDK config: %w", err) @@ -90,7 +90,7 @@ func (d *S3Bucket) DownloadFile(ctx context.Context, bucket string, item string) return buf.Bytes(), nil } -func (d *S3Bucket) GetEncoding(ctx context.Context, bucket string, item string) (string, error) { +func (d *s3Bucket) GetEncoding(ctx context.Context, bucket string, item string) (string, error) { cfg, err := config.LoadDefaultConfig(ctx) if err != nil { return "", fmt.Errorf("error loading AWS SDK config: %w", err) From d321bc68c5740907c4cb2b66459b75d54601cd73 Mon Sep 17 00:00:00 2001 From: Juan Manuel Leflet Estrada Date: Mon, 2 Oct 2023 10:33:01 +0200 Subject: [PATCH 12/21] Use processor encodings Signed-off-by: Juan Manuel Leflet Estrada --- pkg/handler/collector/s3/bucket/encoding.go | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/pkg/handler/collector/s3/bucket/encoding.go b/pkg/handler/collector/s3/bucket/encoding.go index efb35b6e20..2e9d0d1097 100644 --- a/pkg/handler/collector/s3/bucket/encoding.go +++ b/pkg/handler/collector/s3/bucket/encoding.go @@ -21,18 +21,12 @@ import ( "github.com/guacsec/guac/pkg/handler/processor" ) -const ( - BZIP2 = "BZIP2" - ZSTD = "ZSTD" - UNKNOWN = "UNKNOWN" -) - func ExtractEncoding(encoding string, filename string) processor.EncodingType { switch encoding { - case BZIP2: - return BZIP2 - case ZSTD: - return ZSTD + case "BZIP2": + return processor.EncodingBzip2 + case "ZSTD": + return processor.EncodingZstd default: return FromFile(filename) } @@ -43,10 +37,10 @@ func FromFile(file string) processor.EncodingType { extension := strs[len(strs)-1] switch extension { case "bz2": - return BZIP2 + return processor.EncodingBzip2 case "zst": - return ZSTD + return processor.EncodingZstd default: - return UNKNOWN + return processor.EncodingUnknown } } From b81c5e5530a44bca7b6322453b26a11eb4c9a371 Mon Sep 17 00:00:00 2001 From: Juan Manuel Leflet Estrada Date: Mon, 2 Oct 2023 10:36:50 +0200 Subject: [PATCH 13/21] Make struct private, remove unnecessary printing Signed-off-by: Juan Manuel Leflet Estrada --- pkg/handler/collector/s3/messaging/kafka.go | 1 - pkg/handler/collector/s3/messaging/message.go | 6 +++--- pkg/handler/collector/s3/s3_test.go | 2 -- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/pkg/handler/collector/s3/messaging/kafka.go b/pkg/handler/collector/s3/messaging/kafka.go index ed1ca38dbb..e0fc5e6629 100644 --- a/pkg/handler/collector/s3/messaging/kafka.go +++ b/pkg/handler/collector/s3/messaging/kafka.go @@ -68,7 +68,6 @@ func NewKafkaProvider(mpConfig MessageProviderConfig) (KafkaProvider, error) { Brokers: []string{fmt.Sprintf("%s:%s", kafkaHostname, kafkaPort)}, Topic: kafkaTopic, Partition: 0, - MaxBytes: 10e6, }) err := kafkaProvider.reader.SetOffset(kafka.LastOffset) if err != nil { diff --git a/pkg/handler/collector/s3/messaging/message.go b/pkg/handler/collector/s3/messaging/message.go index 719df6dc59..6e10f6c2d0 100644 --- a/pkg/handler/collector/s3/messaging/message.go +++ b/pkg/handler/collector/s3/messaging/message.go @@ -49,11 +49,11 @@ type MessageProviderBuilder interface { GetMessageProvider(config MessageProviderConfig) (MessageProvider, error) } -type MpBuilder struct { +type mpBuilder struct { } // GetMessageProvider Returns a MessageProvider with the given config. Defaults to Kafka provider if no MESSAGE_PROVIDER environment variable is found -func (mb *MpBuilder) GetMessageProvider(config MessageProviderConfig) (MessageProvider, error) { +func (mb *mpBuilder) GetMessageProvider(config MessageProviderConfig) (MessageProvider, error) { switch config.Provider { case "sqs": provider, err := NewSqsProvider(config) @@ -65,5 +65,5 @@ func (mb *MpBuilder) GetMessageProvider(config MessageProviderConfig) (MessagePr } func GetDefaultMessageProviderBuilder() MessageProviderBuilder { - return &MpBuilder{} + return &mpBuilder{} } diff --git a/pkg/handler/collector/s3/s3_test.go b/pkg/handler/collector/s3/s3_test.go index 2c26c4ee33..ada5265bea 100644 --- a/pkg/handler/collector/s3/s3_test.go +++ b/pkg/handler/collector/s3/s3_test.go @@ -164,8 +164,6 @@ func TestQueuesSplitPolling(t *testing.T) { os.Stdout = oldStdout // restoring the real stdout out := <-outC - fmt.Println(out) - if !strings.Contains(out, "returning message for queue q1") { t.Errorf("message for q1 not returned") } From 64238f9159f21b8c779cecb7a92ae0cdd625aed9 Mon Sep 17 00:00:00 2001 From: Juan Manuel Leflet Estrada Date: Mon, 2 Oct 2023 10:43:32 +0200 Subject: [PATCH 14/21] Use existing poll option, return error Signed-off-by: Juan Manuel Leflet Estrada --- cmd/guacone/cmd/s3.go | 2 +- pkg/cli/store.go | 1 - pkg/handler/collector/s3/s3.go | 9 +++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/guacone/cmd/s3.go b/cmd/guacone/cmd/s3.go index 2979f49cd2..414a9d4bc6 100644 --- a/cmd/guacone/cmd/s3.go +++ b/cmd/guacone/cmd/s3.go @@ -67,7 +67,7 @@ var s3Cmd = &cobra.Command{ viper.GetString("s3-mp-port"), viper.GetString("s3-queues"), viper.GetString("s3-region"), - viper.GetBool("s3-poll"), + viper.GetBool("poll"), ) if err != nil { logger.Errorf("failed to validate flags: %v", err) diff --git a/pkg/cli/store.go b/pkg/cli/store.go index 18e6f720e5..17fc110c4b 100644 --- a/pkg/cli/store.go +++ b/pkg/cli/store.go @@ -97,7 +97,6 @@ func init() { set.String("s3-mp-port", "", "port for the message provider") set.String("s3-queues", "", "comma-separated list of queue/topic names") set.String("s3-region", "us-east-1", "aws region") - set.Bool("s3-poll", false, "polling or non-polling behaviour") set.VisitAll(func(f *pflag.Flag) { flagStore[f.Name] = f diff --git a/pkg/handler/collector/s3/s3.go b/pkg/handler/collector/s3/s3.go index e208ea5980..9f2b33e4f8 100644 --- a/pkg/handler/collector/s3/s3.go +++ b/pkg/handler/collector/s3/s3.go @@ -64,13 +64,13 @@ func (s *S3Collector) RetrieveArtifacts(ctx context.Context, docChannel chan<- * if s.config.Poll { retrieveWithPoll(*s, ctx, docChannel) } else { - retrieve(*s, ctx, docChannel) + return retrieve(*s, ctx, docChannel) } return nil } -func retrieve(s S3Collector, ctx context.Context, docChannel chan<- *processor.Document) { +func retrieve(s S3Collector, ctx context.Context, docChannel chan<- *processor.Document) error { logger := logging.FromContext(ctx) downloader := getDownloader(s) @@ -80,13 +80,13 @@ func retrieve(s S3Collector, ctx context.Context, docChannel chan<- *processor.D blob, err := downloader.DownloadFile(ctx, bckt, item) if err != nil { logger.Errorf("could not download item %v: %v", item, err) - return + return err } enc, err := downloader.GetEncoding(ctx, bckt, item) if err != nil { logger.Errorf("could not get encoding for item %v: %v", item, err) - return + return err } doc := &processor.Document{ @@ -101,6 +101,7 @@ func retrieve(s S3Collector, ctx context.Context, docChannel chan<- *processor.D } docChannel <- doc + return nil } func retrieveWithPoll(s S3Collector, ctx context.Context, docChannel chan<- *processor.Document) { From 095e9549b3b99750bcf2d96507ca8b93670d2d9d Mon Sep 17 00:00:00 2001 From: Juan Manuel Leflet Estrada Date: Mon, 2 Oct 2023 10:59:35 +0200 Subject: [PATCH 15/21] Use positional arguments for s3 host and port Signed-off-by: Juan Manuel Leflet Estrada --- cmd/guacone/cmd/s3.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/cmd/guacone/cmd/s3.go b/cmd/guacone/cmd/s3.go index 414a9d4bc6..8ff21d1e4b 100644 --- a/cmd/guacone/cmd/s3.go +++ b/cmd/guacone/cmd/s3.go @@ -50,16 +50,15 @@ type s3Options struct { var s3Opts s3Options var s3Cmd = &cobra.Command{ - Use: "s3 [flags]", - Short: "listens to kafka/sqs s3 events to download documents and add them to the GUAC graph", + Use: "s3 [flags] s3hostname s3port", + Short: "listens to kafka/sqs s3 events to download documents and add them to the GUAC graph, or directly downloads from s3", Args: cobra.MinimumNArgs(0), Run: func(cmd *cobra.Command, args []string) { ctx := logging.WithLogger(context.Background()) logger := logging.FromContext(ctx) s3Opts, err := validateS3Opts( - viper.GetString("s3-host"), - viper.GetString("s3-port"), + args, viper.GetString("s3-bucket"), viper.GetString("s3-item"), viper.GetString("s3-mp"), @@ -138,14 +137,21 @@ var s3Cmd = &cobra.Command{ }, } -func validateS3Opts(s3hostname string, s3port string, s3bucket string, s3item string, mp string, mpHostname string, mpPort string, queues string, region string, poll bool) (s3Options, error) { +func validateS3Opts(args []string, s3bucket string, s3item string, mp string, mpHostname string, mpPort string, queues string, region string, poll bool) (s3Options, error) { var opts s3Options + + if len(args) < 2 { + return opts, fmt.Errorf("wrong number of arguments") + } + s3hostname := args[0] + s3port := args[1] if len(s3hostname) == 0 { return opts, fmt.Errorf("expected s3 hostname") } if len(s3port) == 0 { return opts, fmt.Errorf("expected s3 port") } + if poll { if len(mpHostname) == 0 { return opts, fmt.Errorf("expected hostname for message provider") @@ -171,7 +177,7 @@ func validateS3Opts(s3hostname string, s3port string, s3bucket string, s3item st } func init() { - set, err := cli.BuildFlags([]string{"s3-host", "s3-port", "s3-bucket", "s3-item", "s3-mp", "s3-mp-host", "s3-mp-port", "s3-queues", "s3-region", "s3-poll"}) + set, err := cli.BuildFlags([]string{"s3-bucket", "s3-item", "s3-mp", "s3-mp-host", "s3-mp-port", "s3-queues", "s3-region"}) if err != nil { fmt.Fprintf(os.Stderr, "failed to setup flag: %s", err) os.Exit(1) From 32b6c8feb2378b98b5691a15329d3ddbe09101a8 Mon Sep 17 00:00:00 2001 From: Juan Manuel Leflet Estrada Date: Mon, 2 Oct 2023 11:13:51 +0200 Subject: [PATCH 16/21] Remove unneeded loop Signed-off-by: Juan Manuel Leflet Estrada --- cmd/guacone/cmd/s3.go | 2 +- pkg/handler/collector/s3/messaging/sqs.go | 63 +++++++++++------------ 2 files changed, 32 insertions(+), 33 deletions(-) diff --git a/cmd/guacone/cmd/s3.go b/cmd/guacone/cmd/s3.go index 8ff21d1e4b..3da8433172 100644 --- a/cmd/guacone/cmd/s3.go +++ b/cmd/guacone/cmd/s3.go @@ -177,7 +177,7 @@ func validateS3Opts(args []string, s3bucket string, s3item string, mp string, mp } func init() { - set, err := cli.BuildFlags([]string{"s3-bucket", "s3-item", "s3-mp", "s3-mp-host", "s3-mp-port", "s3-queues", "s3-region"}) + set, err := cli.BuildFlags([]string{"s3-bucket", "s3-item", "s3-mp", "s3-mp-host", "s3-mp-port", "s3-queues", "s3-region", "poll"}) if err != nil { fmt.Fprintf(os.Stderr, "failed to setup flag: %s", err) os.Exit(1) diff --git a/pkg/handler/collector/s3/messaging/sqs.go b/pkg/handler/collector/s3/messaging/sqs.go index ac3fbec079..424af25f4c 100644 --- a/pkg/handler/collector/s3/messaging/sqs.go +++ b/pkg/handler/collector/s3/messaging/sqs.go @@ -117,44 +117,43 @@ func (s *SqsProvider) ReceiveMessage(ctx context.Context) (Message, error) { WaitTimeSeconds: 10, } - for { - select { - case <-ctx.Done(): - return &SqsMessage{}, nil - default: - receiveOutput, err := s.client.ReceiveMessage(ctx, receiveInput) + select { + case <-ctx.Done(): + return &SqsMessage{}, ctx.Err() + default: + receiveOutput, err := s.client.ReceiveMessage(ctx, receiveInput) + if err != nil { + fmt.Printf("error receiving message, skipping: %v\n", err) + //continue + } + + messages := receiveOutput.Messages + if len(messages) > 0 { + message := receiveOutput.Messages[0] + logger.Debugf("Received message: %v\n", *message.Body) + + var msg SqsMessage + err := json.Unmarshal([]byte(*message.Body), &msg) if err != nil { - fmt.Printf("error receiving message, skipping: %v\n", err) - continue + logger.Errorf("error unmarshalling message: %v", err) + return &SqsMessage{}, err } - messages := receiveOutput.Messages - if len(messages) > 0 { - message := receiveOutput.Messages[0] - logger.Debugf("Received message: %v\n", *message.Body) - - var msg SqsMessage - err := json.Unmarshal([]byte(*message.Body), &msg) - if err != nil { - logger.Errorf("error unmarshalling message: %v", err) - return &SqsMessage{}, err - } - - // Delete the received message from the queue (stardard sqs procedure) - deleteInput := &sqs.DeleteMessageInput{ - QueueUrl: &addr, - ReceiptHandle: message.ReceiptHandle, - } - _, err = s.client.DeleteMessage(context.TODO(), deleteInput) - if err != nil { - logger.Errorf("error deleting message: %v\n", err) - } - logger.Debugf("Message deleted from the queue") - - return &msg, nil + // Delete the received message from the queue (stardard sqs procedure) + deleteInput := &sqs.DeleteMessageInput{ + QueueUrl: &addr, + ReceiptHandle: message.ReceiptHandle, } + _, err = s.client.DeleteMessage(context.TODO(), deleteInput) + if err != nil { + logger.Errorf("error deleting message: %v\n", err) + } + logger.Debugf("Message deleted from the queue") + + return &msg, nil } } + return &SqsMessage{}, nil } func (s *SqsProvider) Close(ctx context.Context) error { From 41e5c590176bb28b2a6e7e6e78cf5c926f406939 Mon Sep 17 00:00:00 2001 From: Juan Manuel Leflet Estrada Date: Mon, 2 Oct 2023 11:15:50 +0200 Subject: [PATCH 17/21] Defer call to close message provider Signed-off-by: Juan Manuel Leflet Estrada --- pkg/handler/collector/s3/s3.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/handler/collector/s3/s3.go b/pkg/handler/collector/s3/s3.go index 9f2b33e4f8..8f260a61f3 100644 --- a/pkg/handler/collector/s3/s3.go +++ b/pkg/handler/collector/s3/s3.go @@ -113,7 +113,7 @@ func retrieveWithPoll(s S3Collector, ctx context.Context, docChannel chan<- *pro cancelCtx, cancel := context.WithCancel(ctx) defer cancel() - // Send cancellation in case of receiving SIGINT + // Send cancellation in case of receiving SIGINT/SIGTERM go func(cancel context.CancelFunc) { <-sigChan cancel() @@ -137,6 +137,7 @@ func retrieveWithPoll(s S3Collector, ctx context.Context, docChannel chan<- *pro logger.Errorf("error getting message provider for queue %v: %v", queue, err) return } + defer mp.Close(cncCtx) for { select { From af9b22f9f8d33dd1a5e2099b8ab6f85b8742eb04 Mon Sep 17 00:00:00 2001 From: Juan Manuel Leflet Estrada Date: Tue, 3 Oct 2023 09:17:15 +0200 Subject: [PATCH 18/21] Remove unused var Signed-off-by: Juan Manuel Leflet Estrada --- cmd/guacone/cmd/s3.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/cmd/guacone/cmd/s3.go b/cmd/guacone/cmd/s3.go index 3da8433172..3c09e053b3 100644 --- a/cmd/guacone/cmd/s3.go +++ b/cmd/guacone/cmd/s3.go @@ -47,8 +47,6 @@ type s3Options struct { poll bool // polling or non-polling behaviour? (defaults to non-polling) } -var s3Opts s3Options - var s3Cmd = &cobra.Command{ Use: "s3 [flags] s3hostname s3port", Short: "listens to kafka/sqs s3 events to download documents and add them to the GUAC graph, or directly downloads from s3", From 88c8a906f5b8a98812c8ceafcb248dc643681b86 Mon Sep 17 00:00:00 2001 From: Juan Manuel Leflet Estrada Date: Thu, 5 Oct 2023 10:41:21 +0200 Subject: [PATCH 19/21] Return with error if message was not receiving --- pkg/handler/collector/s3/messaging/kafka.go | 2 +- pkg/handler/collector/s3/messaging/sqs.go | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/handler/collector/s3/messaging/kafka.go b/pkg/handler/collector/s3/messaging/kafka.go index e0fc5e6629..4f5637fbb6 100644 --- a/pkg/handler/collector/s3/messaging/kafka.go +++ b/pkg/handler/collector/s3/messaging/kafka.go @@ -82,7 +82,7 @@ func (k *KafkaProvider) ReceiveMessage(ctx context.Context) (Message, error) { m, err := k.reader.ReadMessage(ctx) if err != nil { - fmt.Println(err.Error()) + return &KafkaMessage{}, fmt.Errorf("error receiving message, skipping: %w\n", err) } logger.Debugf("Message at offset %d: %s = %s\n", m.Offset, string(m.Key), string(m.Value)) diff --git a/pkg/handler/collector/s3/messaging/sqs.go b/pkg/handler/collector/s3/messaging/sqs.go index 424af25f4c..fbeb1b1707 100644 --- a/pkg/handler/collector/s3/messaging/sqs.go +++ b/pkg/handler/collector/s3/messaging/sqs.go @@ -123,8 +123,7 @@ func (s *SqsProvider) ReceiveMessage(ctx context.Context) (Message, error) { default: receiveOutput, err := s.client.ReceiveMessage(ctx, receiveInput) if err != nil { - fmt.Printf("error receiving message, skipping: %v\n", err) - //continue + return &SqsMessage{}, fmt.Errorf("error receiving message, skipping: %w\n", err) } messages := receiveOutput.Messages From 6d0dee209f4c54df10d77a3e22eb03df1ec05caf Mon Sep 17 00:00:00 2001 From: Dejan Bosanac Date: Thu, 5 Oct 2023 15:04:12 +0200 Subject: [PATCH 20/21] :open_file_folder: update Red Hat specific files --- .tekton/ts | 0 OWNERS | 0 dockerfiles/Dockerfile.goreleaser | 10 +- dockerfiles/Dockerfile.guac-cont | 13 ++- .../dockerfiles/Dockerfile.goreleaser | 9 ++ .../overlays/dockerfiles/Dockerfile.guac-cont | 11 ++ redhat/release/update-to-head.sh | 108 ++++++++++++++++++ 7 files changed, 138 insertions(+), 13 deletions(-) create mode 100644 .tekton/ts create mode 100644 OWNERS create mode 100644 redhat/overlays/dockerfiles/Dockerfile.goreleaser create mode 100644 redhat/overlays/dockerfiles/Dockerfile.guac-cont create mode 100755 redhat/release/update-to-head.sh diff --git a/.tekton/ts b/.tekton/ts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/OWNERS b/OWNERS new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dockerfiles/Dockerfile.goreleaser b/dockerfiles/Dockerfile.goreleaser index 6c663d8e80..fef91736dc 100644 --- a/dockerfiles/Dockerfile.goreleaser +++ b/dockerfiles/Dockerfile.goreleaser @@ -1,13 +1,9 @@ -FROM --platform=$BUILDPLATFORM alpine:latest as certs -RUN apk --update add ca-certificates - -FROM --platform=$BUILDPLATFORM alpine:latest -COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt -RUN apk --update --no-cache add wget && rm -rf /var/cache/apk/* +FROM registry.access.redhat.com/ubi9/ubi-minimal:latest +RUN microdnf install -y tar gzip WORKDIR /guac COPY guaccollect* /opt/guac/guaccollect COPY guaccsub* /opt/guac/guaccsub COPY guacgql* /opt/guac/guacgql COPY guacingest* /opt/guac/guacingest -COPY guacone* /opt/guac/guacone +COPY guacone* /opt/guac/guacone \ No newline at end of file diff --git a/dockerfiles/Dockerfile.guac-cont b/dockerfiles/Dockerfile.guac-cont index 59912a4896..c169587510 100644 --- a/dockerfiles/Dockerfile.guac-cont +++ b/dockerfiles/Dockerfile.guac-cont @@ -1,10 +1,11 @@ -FROM docker.io/library/golang:1.19 as builder -WORKDIR /go/src/github.com/guacsec/guac/ +FROM registry.access.redhat.com/ubi9/ubi:latest as builder + ADD . /go/src/github.com/guacsec/guac/ -RUN --mount=type=cache,target=/go/pkg/mod make build +WORKDIR /go/src/github.com/guacsec/guac/ +RUN dnf install -y go-toolset jq https://github.com/goreleaser/goreleaser/releases/download/v1.19.2/goreleaser-1.19.2.x86_64.rpm +RUN rm -rf bin/ && make build -FROM docker.io/library/ubuntu:22.04 -RUN apt update -RUN apt install -y ca-certificates +FROM registry.access.redhat.com/ubi9/ubi-minimal:latest +RUN microdnf install -y tar gzip WORKDIR /root COPY --from=builder /go/src/github.com/guacsec/guac/bin/ /opt/guac/ diff --git a/redhat/overlays/dockerfiles/Dockerfile.goreleaser b/redhat/overlays/dockerfiles/Dockerfile.goreleaser new file mode 100644 index 0000000000..fef91736dc --- /dev/null +++ b/redhat/overlays/dockerfiles/Dockerfile.goreleaser @@ -0,0 +1,9 @@ +FROM registry.access.redhat.com/ubi9/ubi-minimal:latest +RUN microdnf install -y tar gzip + +WORKDIR /guac +COPY guaccollect* /opt/guac/guaccollect +COPY guaccsub* /opt/guac/guaccsub +COPY guacgql* /opt/guac/guacgql +COPY guacingest* /opt/guac/guacingest +COPY guacone* /opt/guac/guacone \ No newline at end of file diff --git a/redhat/overlays/dockerfiles/Dockerfile.guac-cont b/redhat/overlays/dockerfiles/Dockerfile.guac-cont new file mode 100644 index 0000000000..c169587510 --- /dev/null +++ b/redhat/overlays/dockerfiles/Dockerfile.guac-cont @@ -0,0 +1,11 @@ +FROM registry.access.redhat.com/ubi9/ubi:latest as builder + +ADD . /go/src/github.com/guacsec/guac/ +WORKDIR /go/src/github.com/guacsec/guac/ +RUN dnf install -y go-toolset jq https://github.com/goreleaser/goreleaser/releases/download/v1.19.2/goreleaser-1.19.2.x86_64.rpm +RUN rm -rf bin/ && make build + +FROM registry.access.redhat.com/ubi9/ubi-minimal:latest +RUN microdnf install -y tar gzip +WORKDIR /root +COPY --from=builder /go/src/github.com/guacsec/guac/bin/ /opt/guac/ diff --git a/redhat/release/update-to-head.sh b/redhat/release/update-to-head.sh new file mode 100755 index 0000000000..db671bd821 --- /dev/null +++ b/redhat/release/update-to-head.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash + +# Copyright 2023 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The local git repo must have a remote "upstream" pointing +# to upstream sigstore/cosign, and a remote "origin" +# pointing to securesign/cosign + +# Synchs the release-next branch to either the upstream `main` branch +# or a provided git-ref (typically an upstream release tag) and then triggers CI. +# +# NOTE: This requires a corresponding midstream branch to exist in the securesign fork +# with the same name as the upstream branch/ref, but prefixed with "midstream-". +# +# Usage: update-to-head.sh [] + +if [ "$#" -ne 1 ]; then + upstream_ref="main" + midstream_ref="redhat-main" + redhat_ref="release-next" +else + upstream_ref=$1 + midstream_ref="midstream-${upstream_ref}" # The overlays and patches for the given version + redhat_ref="redhat-${upstream_ref}" # The midstream repo with overlays and patches applied +fi + +echo "Synchronizing ${redhat_ref} to upstream/${upstream_ref}..." + +set -e +REPO_NAME=$(basename $(git rev-parse --show-toplevel)) + +# Custom files +custom_files=$(cat <. +git fetch upstream $upstream_ref +if [[ "$upstream_ref" == "main" ]]; then + git checkout upstream/main -B ${redhat_ref} +else + git checkout $upstream_ref -B ${redhat_ref} +fi + +#TODO provide proper patches +git pull -r git@github.com:jmle/guac.git s3-collector +#git pull -r git@github.com:mrizzi/guac.git correlation + +# Update redhat's main and take all needed files from there. +git fetch origin $midstream_ref +git checkout origin/$midstream_ref $custom_files + +# Apply midstream patches +if [[ -d redhat/patches ]]; then + git apply redhat/patches/* +fi + +# RHTAP writes its pipeline files to the root of ${redhat_ref} +# Fetch those from origin and apply them to the the release branch +# since we just wiped out our local copy with the upstream ref. +git fetch origin $redhat_ref +git checkout origin/$redhat_ref .tekton + +# Move overlays to root +if [[ -d redhat/overlays ]]; then + #git mv redhat/overlays/* . + cp -R redhat/overlays/* . + git add * +fi + +git add . # Adds applied patches +git add $custom_files # Adds custom files +git commit -m "${redhat_files_msg}" + +# Trigger CI +# TODO: Set up openshift or github CI to run on release-next-ci +git checkout "${redhat_ref}" -B "${redhat_ref}"-ci +date > ci +git add ci +git commit -m "${robot_trigger_msg}" +git push -f origin "${redhat_ref}-ci" + +if hash gh 2>/dev/null; then + # Test if there is already a sync PR in + COUNT=$(gh -R trustification/guac pr list \ + | grep -c "${robot_trigger_msg}") || true + if [ "$COUNT" = "0" ]; then + gh pr -R trustification/guac create -f -l "kind/sync-fork-to-upstream" -B ${redhat_ref} -H ${redhat_ref}-ci -t "${robot_trigger_msg}" + fi +else + echo "gh cli (https://cli.github.com/) is not installed, so you'll need to create a PR manually." +fi \ No newline at end of file From 9ba8454ae8778280cf2b90388eb1d63d697a2c8d Mon Sep 17 00:00:00 2001 From: Dejan Bosanac Date: Thu, 5 Oct 2023 15:04:15 +0200 Subject: [PATCH 21/21] :robot: triggering CI on branch 'release-next' after synching from upstream/main --- ci | 1 + 1 file changed, 1 insertion(+) create mode 100644 ci diff --git a/ci b/ci new file mode 100644 index 0000000000..7a3b11d2ad --- /dev/null +++ b/ci @@ -0,0 +1 @@ +Thu Oct 5 15:04:15 CEST 2023