{Earthfile, Jenkinsfile}: Earthly ci testing

Change-Id: Idc1292cddab20526afa2e3f783dd4b1f51891efd
storj · Oct 11, 2023 · d886afb · d886afb
1 parent d65b60e
commit d886afb
Show file tree

Hide file tree

Showing 11 changed files with 109 additions and 147 deletions.
diff --git a/Earthfile b/Earthfile
@@ -2,11 +2,13 @@ VERSION 0.6
 FROM golang:1.19
 WORKDIR /go/storj-up
 
-lint:
-    WORKDIR /go/storj-up
+lint-deps:
     RUN go install github.com/storj/ci/...@5286f52
     RUN go install honnef.co/go/tools/cmd/[email protected]
     RUN go install github.com/golangci/golangci-lint/cmd/[email protected]
+
+lint:
+    FROM +lint-deps
     COPY . /go/storj-up
     RUN staticcheck ./...
     RUN golangci-lint --build-tags mage -j=2 run
@@ -18,7 +20,19 @@ lint:
     RUN check-monkit ./...
     RUN check-mod-tidy
 
+build-deps:
+    # Download deps before copying code.
+    COPY go.mod go.sum ./pkg/config/gen/go.mod ./pkg/config/gen/go.sum .
+    RUN go mod download
+    # Output these back in case go mod download changes them.
+    SAVE ARTIFACT go.mod AS LOCAL go.mod
+    SAVE ARTIFACT go.sum AS LOCAL go.sum
+    SAVE ARTIFACT ./pkg/config/gen/go.mod AS LOCAL ./pkg/config/gen/go.mod
+    SAVE ARTIFACT ./pkg/config/gen/go.sum AS LOCAL ./pkg/config/gen/go.sum
+
 build:
+    FROM +build-deps
+    # Copy and build code.
     COPY . .
     RUN --mount=type=cache,target=/root/.cache/go-build \
         --mount=type=cache,target=/go/pkg/mod \
@@ -40,23 +54,58 @@ test:
        ./scripts/test.sh
    SAVE ARTIFACT build/tests.json AS LOCAL build/tests.json
 
-integration:
-#   FROM earthly/dind:ubuntu
-#   RUN apt-get update && apt-get install -y golang-go docker-compose-plugin gcc
-#   ENV PATH=$PATH:/root/go/bin
-#   COPY . .
- #  WORKDIR /test
-   LOCALLY
-   WITH DOCKER --pull img.dev.storj.io/storjup/storj:1.83.2 \
-               --pull img.dev.storj.io/storjup/edge:1.58.1 \
-               --pull cockroachdb/cockroach \
-               --pull redis:6.0.9 \
-               --pull img.dev.storj.io/storjup/storjscan:1.0.3 \
-               --pull ethereum/client-go
-      RUN /test/uplink/storj-up.sh && \
-          /test/edge/storj-up.sh && \
-          /test/storjscan/storj-up.sh
-   END
+integration-uplink:
+    FROM earthly/dind:ubuntu
+    WORKDIR /go/storj-up
+    COPY +build/storj-up .
+    RUN /go/storj-up/storj-up init minimal,db,uplink
+    COPY ./test/uplink/basic_upload_download.sh basic_upload_download.sh
+    WITH DOCKER --compose docker-compose.yaml
+        RUN docker compose exec -T -u 0 uplink bash -c 'storj-up health --host cockroach -d 90' && \
+            docker compose exec -T -u 0 uplink bash -c 'storj-up credentials -s satellite-api:7777 -c satellite-api:10000 -e >> ~/.bashrc' && \
+            < basic_upload_download.sh docker compose exec -T -u 0 uplink bash -i
+    END
+
+integration-edge:
+    FROM earthly/dind:ubuntu
+    WORKDIR /go/storj-up
+    COPY +build/storj-up .
+    RUN /go/storj-up/storj-up init minimal,edge,db,uplink && \
+        /go/storj-up/storj-up image uplink img.dev.storj.io/dlm/storj:1 && \
+        /go/storj-up/storj-up env set authservice STORJ_ENDPOINT=http://gateway-mt:9999
+    COPY ./test/edge/basic_rclone.sh basic_rclone.sh
+    WITH DOCKER --compose docker-compose.yaml
+        RUN docker compose exec -T -u 0 uplink bash -c 'storj-up health --host cockroach -d 90' && \
+            docker compose exec -T -u 0 uplink bash -c 'storj-up credentials -s satellite-api:7777 -c satellite-api:10000 -a http://authservice:8888 -e --s3 >> ~/.bashrc' && \
+            < basic_rclone.sh docker compose exec -T -u 0 uplink bash -i
+    END
+
+integration-storjscan:
+    FROM earthly/dind:ubuntu
+    WORKDIR /go/storj-up
+    COPY +build/storj-up .
+    RUN /go/storj-up/storj-up init && \
+        /go/storj-up/storj-up init minimal,satellite-core,edge,db,billing && \
+        /go/storj-up/storj-up env setenv satellite-core STORJ_PAYMENTS_BILLING_CONFIG_INTERVAL=5s && \
+        /go/storj-up/storj-up env setenv satellite-core STORJ_PAYMENTS_STORJSCAN_INTERVAL=5s && \
+        /go/storj-up/storj-up env setenv satellite-core STORJ_PAYMENTS_STORJSCAN_CONFIRMATIONS=12 && \
+        /go/storj-up/storj-up env setenv storjscan STORJ_TOKEN_PRICE_USE_TEST_PRICES=true && \
+        /go/storj-up/storj-up image storjscan img.dev.storj.io/dlm/storjscan:1
+    COPY ./test/storjscan/basic_storjscan.sh basic_storjscan.sh
+    WITH DOCKER --compose docker-compose.yaml
+        RUN docker compose exec -T -u 0 storjscan bash -c 'storj-up health --host cockroach -d 90' && \
+            docker compose exec -T -u 0 storjscan bash -c 'storj-up credentials -s satellite-api:7777 -c satellite-api:10000 -a http://authservice:8888 -e >> ~/.bashrc' && \
+            docker compose cp pk.json storjscan:/var/lib/storj/pk.json && \
+            docker compose cp pass storjscan:/var/lib/storj/pass && \
+            docker compose cp TestToken.abi storjscan:/var/lib/storj/TestToken.abi && \
+            docker compose cp TestToken.bin storjscan:/var/lib/storj/TestToken.bin && \
+            < basic_storjscan.sh docker compose exec -T -u 0 storjscan bash -i
+    END
+
+integration-all:
+   BUILD +integration-uplink
+   BUILD +integration-edge
+   BUILD +integration-storjscan
 
 check-format:
    COPY . .

diff --git a/Jenkinsfile b/Jenkinsfile
@@ -1,6 +1,6 @@
 pipeline {
     agent {
-       label 'node4'
+       label('nomad || node4')
     }
 
     options {
@@ -17,20 +17,43 @@ pipeline {
                checkout scm
             }
         }
-	stage('Lint') {
-		steps {
-			sh "earthly +lint"
-		}
-	}
-	stage('Test') {
-		steps {
-			sh "earthly +test"
-		}
-	}
-	stage('Integration') {
-		steps {
-			sh "earthly -P +integration"
-		}
-	}
+        stage('Lint') {
+            steps {
+                sh "earthly +lint"
+            }
+        }
+        stage('Test') {
+            steps {
+                sh "earthly +test"
+            }
+        }
+        stage('Integration') {
+            parallel {
+                stage('Uplink') {
+                    steps {
+                        withCredentials([usernamePassword(credentialsId: '67d52f80-1802-407a-a21a-ea24d4c28a83', passwordVariable: 'password', usernameVariable: 'username')]) {
+                            sh 'docker login img.dev.storj.io -u $username -p $password'
+                        }
+                        sh "earthly -P +integration-uplink"
+                    }
+                }
+                stage('Edge') {
+                    steps {
+                        withCredentials([usernamePassword(credentialsId: '67d52f80-1802-407a-a21a-ea24d4c28a83', passwordVariable: 'password', usernameVariable: 'username')]) {
+                            sh 'docker login img.dev.storj.io -u $username -p $password'
+                        }
+                        sh "earthly -P +integration-edge"
+                    }
+                }
+                stage('Storjscan') {
+                    steps {
+                        withCredentials([usernamePassword(credentialsId: '67d52f80-1802-407a-a21a-ea24d4c28a83', passwordVariable: 'password', usernameVariable: 'username')]) {
+                            sh 'docker login img.dev.storj.io -u $username -p $password'
+                        }
+                        sh "earthly -P +integration-storjscan"
+                    }
+                }
+            }
+        }
     }
 }