Improve Cloud-Pipeline-Distribution GitHub Actions workflow (#3370)

* Use python 3 for Cloud-Pipeline-Distribution GitHub action * Use python 3.9 for Cloud-Pipeline-Distribution GitHub action * Exclude ntlmaps from pipe macos build * Use python 3 for pipe macos build * Clean pyinstaller resources in pipe cli module clean task * Publish all pipe macos builds * Use prettytable hooks for macos build only * Publish all cloud pipeline distributions * Add temporary debug logs * Add temporary debug logs * Add temporary debug logs * Add temporary debug logs * Add temporary debug logs * Add temporary debug logs * Build pipe for each platform separately * Fix pipe build paths * Fix pipe build paths * Remove temporary debug logs * Revert unnecessary changes * Use separate scripts to build pipe macos * Build client separately * Fix typo * Install node * Build cloud data separately * Build fsbrowser separately * Build data sharing service separately * Add data sharing service to distribution * Build data sharing service beforehand * Skip extra builds * Build gpu stat separately * Build necessary modules only once * Refactor main workflow * Remove node install step from build all job * Build data transfer service separately * Build necessary modules only once * Transfer s3 files more efficiently * Upload gha built distributions to a separate directory in s3
epam · Nov 8, 2023 · 4a28572 · 4a28572
1 parent 4bcd6a9
commit 4a28572
Show file tree

Hide file tree

Showing 17 changed files with 767 additions and 37 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -16,6 +16,42 @@ env:
   CLOUD_PIPELINE_BUILD_NUMBER_SEED: 30000
 
 jobs:
+  CLI_Linux:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-java@v2
+        with:
+          distribution: 'zulu'
+          java-version: '8'
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.9'
+          architecture: 'x64'
+      - uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
+      - run: bash deploy/github_actions/gha_build_pipe_linux.sh
+  CLI_Linux_el6:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-java@v2
+        with:
+          distribution: 'zulu'
+          java-version: '8'
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.9'
+          architecture: 'x64'
+      - uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
+      - run: bash deploy/github_actions/gha_build_pipe_linux_el6.sh
   CLI_MacOS:
     runs-on: macos-11
     steps:
@@ -26,17 +62,34 @@ jobs:
           java-version: '8'
       - uses: actions/setup-python@v2
         with:
-          python-version: '2.7'
+          python-version: '3.9'
           architecture: 'x64'
       - uses: aws-actions/configure-aws-credentials@v1
         with:
           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
-      - run: bash deploy/github_actions/gha_build_macos.sh
-  Build_All:
+      - run: bash deploy/github_actions/gha_build_pipe_macos.sh
+  CLI_Windows:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-java@v2
+        with:
+          distribution: 'zulu'
+          java-version: '8'
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.9'
+          architecture: 'x64'
+      - uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
+      - run: bash deploy/github_actions/gha_build_pipe_windows.sh
+  Build_Client:
     runs-on: ubuntu-20.04
-    needs: CLI_MacOS
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-java@v2
@@ -45,7 +98,7 @@ jobs:
           java-version: '8'
       - uses: actions/setup-python@v2
         with:
-          python-version: '2.7.18'
+          python-version: '3.9'
           architecture: 'x64'
       - uses: actions/setup-node@v3
         with:
@@ -55,4 +108,137 @@ jobs:
           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
+      - run: bash deploy/github_actions/gha_build_client.sh
+  Build_CloudData_Linux:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-java@v2
+        with:
+          distribution: 'zulu'
+          java-version: '8'
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.9'
+          architecture: 'x64'
+      - uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
+      - run: bash deploy/github_actions/gha_build_cloud_data_linux.sh
+  Build_CloudData_Windows:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-java@v2
+        with:
+          distribution: 'zulu'
+          java-version: '8'
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.9'
+          architecture: 'x64'
+      - uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
+      - run: bash deploy/github_actions/gha_build_cloud_data_windows.sh
+  Build_DataSharingService:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-java@v2
+        with:
+          distribution: 'zulu'
+          java-version: '8'
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.9'
+          architecture: 'x64'
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 14
+      - uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
+      - run: bash deploy/github_actions/gha_build_data_sharing_service.sh
+  Build_DataTransferService:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-java@v2
+        with:
+          distribution: 'zulu'
+          java-version: '8'
+      - uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
+      - run: bash deploy/github_actions/gha_build_data_transfer_service.sh
+  Build_FSBrowser:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-java@v2
+        with:
+          distribution: 'zulu'
+          java-version: '8'
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.9'
+          architecture: 'x64'
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 14
+      - uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
+      - run: bash deploy/github_actions/gha_build_fs_browser.sh
+  Build_GpuStat:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-java@v2
+        with:
+          distribution: 'zulu'
+          java-version: '8'
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.9'
+          architecture: 'x64'
+      - uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
+      - run: bash deploy/github_actions/gha_build_gpu_stat.sh
+  Build_All:
+    runs-on: ubuntu-20.04
+    needs: [CLI_Linux, CLI_Linux_el6, CLI_MacOS, CLI_Windows,
+            Build_Client,
+            Build_CloudData_Linux, Build_CloudData_Windows,
+            Build_DataSharingService, Build_DataTransferService,
+            Build_FSBrowser, Build_GpuStat]
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-java@v2
+        with:
+          distribution: 'zulu'
+          java-version: '8'
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.9'
+          architecture: 'x64'
+      - uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
       - run: sudo -E PATH=$PATH bash deploy/github_actions/gha_pack_dist.sh
diff --git a/deploy/github_actions/gha_build_client.sh b/deploy/github_actions/gha_build_client.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+CLOUD_PIPELINE_BUILD_NUMBER=$(($CLOUD_PIPELINE_BUILD_NUMBER_SEED+$GITHUB_RUN_NUMBER))
+
+./gradlew -PbuildNumber=${CLOUD_PIPELINE_BUILD_NUMBER}.${GITHUB_SHA} \
+          -Pprofile=release \
+          client:buildUI \
+          --no-daemon
+
+tar -czf $PWD/client.tar.gz -C client/build .
+
+ls -lh client.tar.gz
+
+if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then
+    aws s3 cp --no-progress client.tar.gz s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/client.tar.gz
+fi
diff --git a/deploy/github_actions/gha_build_cloud_data_linux.sh b/deploy/github_actions/gha_build_cloud_data_linux.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+CLOUD_PIPELINE_BUILD_NUMBER=$(($CLOUD_PIPELINE_BUILD_NUMBER_SEED+$GITHUB_RUN_NUMBER))
+
+./gradlew -PbuildNumber=${CLOUD_PIPELINE_BUILD_NUMBER}.${GITHUB_SHA} \
+          -Pprofile=release \
+          cloud-pipeline-webdav-client:buildLinux \
+          --no-daemon
+
+ls -lh cloud-pipeline-webdav-client/out/cloud-data-linux.tar.gz
+
+if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then
+    aws s3 cp --no-progress cloud-pipeline-webdav-client/out/cloud-data-linux.tar.gz s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/cloud-data-linux.tar.gz
+fi
diff --git a/deploy/github_actions/gha_build_cloud_data_windows.sh b/deploy/github_actions/gha_build_cloud_data_windows.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+CLOUD_PIPELINE_BUILD_NUMBER=$(($CLOUD_PIPELINE_BUILD_NUMBER_SEED+$GITHUB_RUN_NUMBER))
+
+./gradlew -PbuildNumber=${CLOUD_PIPELINE_BUILD_NUMBER}.${GITHUB_SHA} \
+          -Pprofile=release \
+          cloud-pipeline-webdav-client:buildWin \
+          --no-daemon
+
+ls -lh cloud-pipeline-webdav-client/out/cloud-data-win64.zip
+
+if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then
+    aws s3 cp --no-progress cloud-pipeline-webdav-client/out/cloud-data-win64.zip s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/cloud-data-win64.zip
+fi
diff --git a/deploy/github_actions/gha_build_data_sharing_service.sh b/deploy/github_actions/gha_build_data_sharing_service.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+CLOUD_PIPELINE_BUILD_NUMBER=$(($CLOUD_PIPELINE_BUILD_NUMBER_SEED+$GITHUB_RUN_NUMBER))
+
+./gradlew -PbuildNumber=${CLOUD_PIPELINE_BUILD_NUMBER}.${GITHUB_SHA} \
+          -Pprofile=release \
+          data-sharing-service:buildFast \
+          --no-daemon
+
+ls -lh data-sharing-service/api/build/libs/data-sharing-service.jar
+
+if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then
+    aws s3 cp --no-progress data-sharing-service/api/build/libs/data-sharing-service.jar s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/data-sharing-service.jar
+fi
diff --git a/deploy/github_actions/gha_build_data_transfer_service.sh b/deploy/github_actions/gha_build_data_transfer_service.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# Copyright 2017-2023 EPAM Systems, Inc. (https://www.epam.com/)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+CLOUD_PIPELINE_BUILD_NUMBER=$(($CLOUD_PIPELINE_BUILD_NUMBER_SEED+$GITHUB_RUN_NUMBER))
+
+./gradlew -PbuildNumber=${CLOUD_PIPELINE_BUILD_NUMBER}.${GITHUB_SHA} \
+          -Pprofile=release \
+          data-transfer-service:build \
+          data-transfer-service:bundleWindows \
+          data-transfer-service:bundleLinux \
+          --no-daemon
+
+ls -lh data-transfer-service/build/libs/data-transfer-service.jar \
+       data-transfer-service/build/distributions/data-transfer-service-windows.zip \
+       data-transfer-service/build/distributions/data-transfer-service-linux.zip
+
+if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then
+    aws s3 cp --no-progress data-transfer-service/build/libs/data-transfer-service.jar s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/data-transfer-service.jar
+    aws s3 cp --no-progress data-transfer-service/build/distributions/data-transfer-service-windows.zip s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/data-transfer-service-windows.zip
+    aws s3 cp --no-progress data-transfer-service/build/distributions/data-transfer-service-linux.zip s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/data-transfer-service-linux.zip
+fi