{Earthfile,Jenkinsfile,test}: update test structure and ci files

Change-Id: I63420597ed3a689d319b54ad6a4b1b1cc5bd44a3
storj · Oct 13, 2023 · 5c2e457 · 5c2e457
1 parent cfb8ef8
commit 5c2e457
Show file tree

Hide file tree

Showing 11 changed files with 164 additions and 202 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -5,14 +5,15 @@ on:
 jobs:
   build:
     runs-on: ubuntu-latest
+    env:
+      FORCE_COLOR: 1
     steps:
       - uses: actions/checkout@v2
       - name: Setup Earthly
-        run: "sudo /bin/sh -c 'wget https://github.com/earthly/earthly/releases/download/v0.6.23/earthly-linux-amd64 -O /usr/local/bin/earthly && chmod +x /usr/local/bin/earthly'"
+        run: "sudo /bin/sh -c 'wget https://github.com/earthly/earthly/releases/download/v0.7.20/earthly-linux-amd64 -O /usr/local/bin/earthly && chmod +x /usr/local/bin/earthly'"
       - name: Test
         run: earthly --ci +test
       - name: Lint
         run: earthly --ci +lint
       - name: Integration
-        run: earthly -P --ci +integration
-
+        run: earthly -P --ci +integration-all
diff --git a/Earthfile b/Earthfile
@@ -2,11 +2,13 @@ VERSION 0.6
 FROM golang:1.19
 WORKDIR /go/storj-up
 
-lint:
-    WORKDIR /go/storj-up
+lint-deps:
     RUN go install github.com/storj/ci/...@5286f52
     RUN go install honnef.co/go/tools/cmd/[email protected]
     RUN go install github.com/golangci/golangci-lint/cmd/[email protected]
+
+lint:
+    FROM +lint-deps
     COPY . /go/storj-up
     RUN staticcheck ./...
     RUN golangci-lint --build-tags mage -j=2 run
@@ -18,7 +20,19 @@ lint:
     RUN check-monkit ./...
     RUN check-mod-tidy
 
-build:
+build-app-deps:
+    # Download deps before copying code.
+    COPY go.mod go.sum ./pkg/config/gen/go.mod ./pkg/config/gen/go.sum .
+    RUN go mod download
+    # Output these back in case go mod download changes them.
+    SAVE ARTIFACT go.mod AS LOCAL go.mod
+    SAVE ARTIFACT go.sum AS LOCAL go.sum
+    SAVE ARTIFACT ./pkg/config/gen/go.mod AS LOCAL ./pkg/config/gen/go.mod
+    SAVE ARTIFACT ./pkg/config/gen/go.sum AS LOCAL ./pkg/config/gen/go.sum
+
+build-app:
+    FROM +build-app-deps
+    # Copy and build code.
     COPY . .
     RUN --mount=type=cache,target=/root/.cache/go-build \
         --mount=type=cache,target=/go/pkg/mod \
@@ -40,22 +54,57 @@ test:
        ./scripts/test.sh
    SAVE ARTIFACT build/tests.json AS LOCAL build/tests.json
 
-integration:
-   FROM earthly/dind:ubuntu
-   RUN apt-get update && apt-get install -y golang-go docker-compose-plugin gcc
-   ENV PATH=$PATH:/root/go/bin
-   COPY . .
-   WORKDIR /test
-   WITH DOCKER --pull img.dev.storj.io/storjup/storj:1.83.2 \
-               --pull img.dev.storj.io/storjup/edge:1.58.1 \
-               --pull cockroachdb/cockroach \
-               --pull redis:6.0.9 \
-               --pull img.dev.storj.io/storjup/storjscan:1.0.3 \
-               --pull ethereum/client-go
-      RUN ./test.sh && \
-          ./test-edge.sh && \
-          ./test-storjscan.sh
-   END
+integration-all:
+   BUILD +integration-uplink
+   BUILD +integration-edge
+   BUILD +integration-storjscan
+
+integration-uplink:
+    FROM earthly/dind:ubuntu
+    WORKDIR /go/storj-up
+    COPY +build-app/storj-up .
+    RUN /go/storj-up/storj-up init minimal,db,uplink
+    COPY ./test/uplink/basic_upload_download.sh basic_upload_download.sh
+    WITH DOCKER --compose docker-compose.yaml
+        RUN docker compose exec -T -u 0 uplink bash -c 'storj-up health --host cockroach -d 120' && \
+            docker compose exec -T -u 0 uplink bash -c 'storj-up credentials -s satellite-api:7777 -c satellite-api:10000 -e >> ~/.bashrc' && \
+            < basic_upload_download.sh docker compose exec -T -u 0 uplink bash -i
+    END
+
+integration-edge:
+    FROM earthly/dind:ubuntu
+    WORKDIR /go/storj-up
+    COPY +build-app/storj-up .
+    RUN /go/storj-up/storj-up init minimal,edge,db,uplink && \
+        /go/storj-up/storj-up image uplink img.dev.storj.io/storjup/storj:1.89.5 && \
+        /go/storj-up/storj-up env set authservice STORJ_ENDPOINT=http://gateway-mt:9999
+    COPY ./test/edge/basic_rclone.sh basic_rclone.sh
+    WITH DOCKER --compose docker-compose.yaml
+        RUN docker compose exec -T -u 0 uplink bash -c 'storj-up health --host cockroach -d 120' && \
+            docker compose exec -T -u 0 uplink bash -c 'storj-up credentials -s satellite-api:7777 -c satellite-api:10000 -a http://authservice:8888 -e --s3 >> ~/.bashrc' && \
+            < basic_rclone.sh docker compose exec -T -u 0 uplink bash -i
+    END
+
+integration-storjscan:
+    FROM earthly/dind:ubuntu
+    WORKDIR /go/storj-up
+    COPY +build-app/storj-up .
+    RUN /go/storj-up/storj-up init minimal,satellite-core,edge,db,billing && \
+        /go/storj-up/storj-up env setenv satellite-core STORJ_PAYMENTS_BILLING_CONFIG_INTERVAL=5s && \
+        /go/storj-up/storj-up env setenv satellite-core STORJ_PAYMENTS_STORJSCAN_INTERVAL=5s && \
+        /go/storj-up/storj-up env setenv satellite-core STORJ_PAYMENTS_STORJSCAN_CONFIRMATIONS=12 && \
+        /go/storj-up/storj-up env setenv storjscan STORJ_TOKEN_PRICE_USE_TEST_PRICES=true && \
+        /go/storj-up/storj-up image storjscan img.dev.storj.io/storjup/storjscan:20231012
+    COPY ./test/storjscan/basic_storjscan.sh basic_storjscan.sh
+    WITH DOCKER --compose docker-compose.yaml
+        RUN docker compose exec -T -u 0 storjscan bash -c 'storj-up health --host cockroach -d 120' && \
+            docker compose exec -T -u 0 storjscan bash -c 'storj-up credentials -s satellite-api:7777 -c satellite-api:10000 -a http://authservice:8888 -e >> ~/.bashrc' && \
+            docker compose cp pk.json storjscan:/var/lib/storj/pk.json && \
+            docker compose cp pass storjscan:/var/lib/storj/pass && \
+            docker compose cp TestToken.abi storjscan:/var/lib/storj/TestToken.abi && \
+            docker compose cp TestToken.bin storjscan:/var/lib/storj/TestToken.bin && \
+            < basic_storjscan.sh docker compose exec -T -u 0 storjscan bash -i
+    END
 
 check-format:
    COPY . .

diff --git a/Jenkinsfile b/Jenkinsfile
@@ -1,6 +1,6 @@
 pipeline {
     agent {
-       label 'node4'
+       label('nomad || node4')
     }
 
     options {
@@ -17,20 +17,34 @@ pipeline {
                checkout scm
             }
         }
-	stage('Lint') {
-		steps {
-			sh "earthly +lint"
-		}
-	}
-	stage('Test') {
-		steps {
-			sh "earthly +test"
-		}
-	}
-	stage('Integration') {
-		steps {
-			sh "earthly -P +integration"
-		}
-	}
+        stage('Lint') {
+            steps {
+                sh "earthly +lint"
+            }
+        }
+        stage('Test') {
+            steps {
+                sh "earthly +test"
+            }
+        }
+        stage('Integration') {
+            parallel {
+                stage('Uplink') {
+                    steps {
+                        sh "earthly -P +integration-uplink"
+                    }
+                }
+                stage('Edge') {
+                    steps {
+                        sh "earthly -P +integration-edge"
+                    }
+                }
+                stage('Storjscan') {
+                    steps {
+                        sh "earthly -P +integration-storjscan"
+                    }
+                }
+            }
+        }
     }
 }
diff --git a/test/test-pay-invoice.sh → test/billing/test-pay-invoice.sh b/test/test-pay-invoice.sh → test/billing/test-pay-invoice.sh
diff --git a/test/edge/basic_rclone.sh b/test/edge/basic_rclone.sh
@@ -0,0 +1,16 @@
+dd if=/dev/random of=data count=10240 bs=1024
+sha256sum data > sha256.sum
+
+curl https://rclone.org/install.sh | bash
+# shellcheck disable=SC2086
+rclone config create storjdevs3 s3 env_auth true provider Minio access_key_id $AWS_ACCESS_KEY_ID secret_access_key "$AWS_SECRET_ACCESS_KEY" endpoint "$STORJ_GATEWAY" chunk_size 64M upload_cutoff 64M
+
+BUCKET=bucket$RANDOM
+rclone mkdir storjdevs3:$BUCKET
+rclone copy data storjdevs3:$BUCKET/data
+sha256sum -c sha256.sum
+
+rm data
+rclone copy storjdevs3:$BUCKET/data download
+mv download/data ./
+sha256sum -c sha256.sum
diff --git a/test/storjscan/basic_storjscan.sh b/test/storjscan/basic_storjscan.sh
@@ -0,0 +1,34 @@
+set -xueo pipefail
+
+export CETH_CHAIN=http://geth:8545
+export CETH_ACCOUNT=2e9a0761ce9815b95b2389634f6af66abe5fec2b1e04b772728442b4c35ea365
+export CETH_CONTRACT=$(cethacea contract deploy --quiet --name TOKEN TestToken.bin --abi TestToken.abi '(uint256)' 1000000000000)
+
+curl -X GET -u "eu1:eu1secret" http://127.0.0.1:12000/api/v0/auth/whoami
+curl -X GET -u "us1:us1secret" http://127.0.0.1:12000/api/v0/auth/whoami
+
+storjscan mnemonic >.mnemonic
+storjscan generate >.wallets
+storjscan import --input-file .wallets --api-key us1 --api-secret us1secret --address http://127.0.0.1:12000
+storjscan mnemonic >.mnemonic
+storjscan generate >.wallets
+storjscan import --input-file .wallets --api-key eu1 --api-secret eu1secret --address http://127.0.0.1:12000
+rm -rf .mnemonic .wallets
+
+curl -X POST 'http://satellite-api:10000/api/v0/payments/wallet' --header "Cookie: _tokenKey=$_tokenKey"
+ADDRESS=$(curl -X GET -s http://satellite-api:10000/api/v0/payments/wallet --header "Cookie: _tokenKey=$_tokenKey" | jq -r '.address')
+
+#ACCOUNT is defined with environment variables above
+for i in {1..15}; do cethacea token transfer 1000 "$ADDRESS" && sleep 1; done
+storj-up health --host cockroach -t billing_transactions -n 6 -d 12
+
+RESPONSE=$(curl -X GET http://satellite-api:10000/api/v0/payments/wallet/payments --header "Cookie: _tokenKey=$_tokenKey")
+STATUS=$(echo "$RESPONSE" | jq -r '.payments[-4].Status')
+STATUS_BONUS=$(echo "$RESPONSE" | jq -r '.payments[-1].Status')
+
+if [ "${STATUS_BONUS}" != 'complete' ] || [ "${STATUS}" != 'confirmed' ]; then
+  echo "Test FAILED. Payment status: ${STATUS} Payment bonus status: ${STATUS_BONUS}"
+  exit 1
+else
+  echo "Test PASSED."
+fi
diff --git a/test/test-reorg.sh → test/storjscan/reorg_storjscan.sh b/test/test-reorg.sh → test/storjscan/reorg_storjscan.sh
@@ -22,14 +22,14 @@ cleanup() {
 }
 trap cleanup EXIT
 
-go install storj.io/storj-up
+go install storj.io/storj-up@main
 
 if [ ! "$(which storjscan )" ]; then
    go install storj.io/storjscan/cmd/storjscan@latest
 fi
 
 if [ ! "$(which cethacea)" ]; then
-   go install github.com/elek/cethacea@latest
+   go install github.com/elek/cethacea@main
 fi
 
 export STORJUP_NO_HISTORY=true

diff --git a/test/test-edge.sh b/test/test-edge.sh
diff --git a/test/test-storjscan.sh b/test/test-storjscan.sh