From 3a125c4169cbb55f6f05e50e3bd722609faf3bb2 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 4 Jul 2024 12:41:21 +0300 Subject: [PATCH] [fix][ci] Fix OWASP Dependency Check download by using NVD API key (#22999) (cherry picked from commit 8b7754f11f113af9d341a460795d0c7b8095f594) # Conflicts: # .github/workflows/ci-owasp-dependency-check.yaml # pom.xml --- .../workflows/ci-owasp-dependency-check.yaml | 80 ++++++++++++++----- .github/workflows/pulsar-ci.yaml | 7 +- distribution/io/pom.xml | 1 - pom.xml | 14 +++- pulsar-io/docs/pom.xml | 1 - pulsar-io/flume/pom.xml | 1 - pulsar-io/hbase/pom.xml | 1 - pulsar-io/hdfs2/pom.xml | 7 +- pulsar-io/hdfs3/pom.xml | 9 +-- tiered-storage/file-system/pom.xml | 1 - 10 files changed, 79 insertions(+), 43 deletions(-) diff --git a/.github/workflows/ci-owasp-dependency-check.yaml b/.github/workflows/ci-owasp-dependency-check.yaml index 0ee1275bdfefc..a70f4a82ff1af 100644 --- a/.github/workflows/ci-owasp-dependency-check.yaml +++ b/.github/workflows/ci-owasp-dependency-check.yaml @@ -24,7 +24,9 @@ on: workflow_dispatch: env: - MAVEN_OPTS: -Xss1500k -Xmx1024m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 + MAVEN_OPTS: -Xss1500k -Xmx1500m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 + JDK_DISTRIBUTION: corretto + NIST_NVD_API_KEY: ${{ secrets.NIST_NVD_API_KEY }} jobs: run-owasp-dependency-check: @@ -34,62 +36,96 @@ jobs: JOB_NAME: Check ${{ matrix.branch }} GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} runs-on: ubuntu-22.04 - timeout-minutes: 45 + timeout-minutes: 75 strategy: fail-fast: false + max-parallel: 1 matrix: include: - branch: master - - branch: branch-3.1 + - branch: branch-3.3 + - branch: branch-3.2 - branch: branch-3.0 - - branch: branch-2.11 - - branch: branch-2.10 - jdk: 11 - - branch: branch-2.9 - jdk: 11 - - branch: branch-2.8 - jdk: 11 steps: - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ matrix.branch }} - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - - name: Cache local Maven repository - uses: actions/cache@v3 + - name: Restore Maven repository cache + uses: actions/cache/restore@v4 timeout-minutes: 5 with: path: | ~/.m2/repository/*/*/* !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-owasp-${{ hashFiles('**/pom.xml') }} + key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} restore-keys: | - ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} ${{ runner.os }}-m2-dependencies-core-modules- - name: Set up JDK ${{ matrix.jdk || '17' }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: - distribution: 'temurin' + distribution: ${{ env.JDK_DISTRIBUTION }} java-version: ${{ matrix.jdk || '17' }} - name: run install by skip tests - run: mvn -B -ntp clean install -DskipTests -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true -DskipDocker=true + run: mvn -B -ntp clean install -DskipTests -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true -DskipDocker=true -DnarPluginPhase=none -pl '!distribution/io,!distribution/offloaders' + + - name: OWASP cache key weeknum + id: get-weeknum + run: | + echo "weeknum=$(date -u +"%Y-%U")" >> $GITHUB_OUTPUT + shell: bash + + - name: Restore OWASP Dependency Check data + id: restore-owasp-dependency-check-data + uses: actions/cache/restore@v4 + timeout-minutes: 5 + with: + path: ~/.m2/repository/org/owasp/dependency-check-data + key: owasp-dependency-check-data-${{ steps.get-weeknum.outputs.weeknum }} + enableCrossOsArchive: true + restore-keys: | + owasp-dependency-check-data- + + - name: Update OWASP Dependency Check data + id: update-owasp-dependency-check-data + if: ${{ matrix.branch == 'master' && (steps.restore-owasp-dependency-check-data.outputs.cache-hit != 'true' || steps.restore-owasp-dependency-check-data.outputs.cache-matched-key != steps.restore-owasp-dependency-check-data.outputs.cache-primary-key) }} + run: mvn -B -ntp -Powasp-dependency-check initialize -pl . dependency-check:update-only + + - name: Save OWASP Dependency Check data + if: ${{ steps.update-owasp-dependency-check-data.outcome == 'success' }} + uses: actions/cache/save@v4 + timeout-minutes: 5 + with: + path: ~/.m2/repository/org/owasp/dependency-check-data + key: ${{ steps.restore-owasp-dependency-check-data.outputs.cache-primary-key }} + enableCrossOsArchive: true - name: run OWASP Dependency Check for distribution/server (-DfailBuildOnAnyVulnerability=true) run: mvn -B -ntp -Pmain,skip-all,skipDocker,owasp-dependency-check initialize verify -pl distribution/server -DfailBuildOnAnyVulnerability=true + - name: run OWASP Dependency Check for offloaders/tiered-storage and pulsar-io connectors (-DfailOnError=false) + if: ${{ !cancelled() }} + run: | + mvnprojects=$(mvn -B -ntp -Dscan=false initialize \ + | grep -- "-< .* >-" \ + | sed -E 's/.*-< (.*) >-.*/\1/' \ + | grep -E 'pulsar-io-|tiered-storage-|offloader' \ + | tr '\n' ',' | sed 's/,$/\n/' ) + set -xe + mvn --fail-at-end -B -ntp -Pmain,skip-all,skipDocker,owasp-dependency-check initialize verify -DfailOnError=false -pl "${mvnprojects}" + - name: Upload OWASP Dependency Check reports - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: name: owasp-dependency-check-reports-${{ matrix.branch }} path: | - distribution/server/target/dependency-check-report.html - distribution/offloaders/target/dependency-check-report.html - distribution/io/target/dependency-check-report.html + **/target/dependency-check-report.html \ No newline at end of file diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index 7cee6c3db3d13..2a9a58c08d09a 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -1338,6 +1338,7 @@ jobs: env: GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} + NIST_NVD_API_KEY: ${{ secrets.NIST_NVD_API_KEY }} steps: - name: checkout uses: actions/checkout@v4 @@ -1353,16 +1354,14 @@ jobs: with: limit-access-to-actor: true - - name: Cache Maven dependencies - uses: actions/cache@v4 + - name: Restore Maven repository cache + uses: actions/cache/restore@v4 timeout-minutes: 5 with: path: | ~/.m2/repository/*/*/* !~/.m2/repository/org/apache/pulsar - !~/.m2/repository/org/owasp/dependency-check-data key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - lookup-only: true restore-keys: | ${{ runner.os }}-m2-dependencies-core-modules- - name: Set up JDK ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }} diff --git a/distribution/io/pom.xml b/distribution/io/pom.xml index bab6df3441ca3..61abe5670bc89 100644 --- a/distribution/io/pom.xml +++ b/distribution/io/pom.xml @@ -137,7 +137,6 @@ org.owasp dependency-check-maven - ${dependency-check-maven.version} diff --git a/pom.xml b/pom.xml index a8fa4bd6d0bda..55fd073ddb50e 100644 --- a/pom.xml +++ b/pom.xml @@ -305,7 +305,7 @@ flexible messaging model and an intuitive client API. 0.1.4 1.3 0.4 - 9.1.0 + 10.0.1 0.9.44 1.6.1 6.4.0 @@ -2090,6 +2090,16 @@ flexible messaging model and an intuitive client API. build-helper-maven-plugin ${build-helper-maven-plugin.version} + + org.owasp + dependency-check-maven + ${dependency-check-maven.version} + + NIST_NVD_API_KEY + + + + @@ -2526,7 +2536,6 @@ flexible messaging model and an intuitive client API. org.owasp dependency-check-maven - ${dependency-check-maven.version} ${pulsar.basedir}/src/owasp-dependency-check-false-positives.xml @@ -2561,7 +2570,6 @@ flexible messaging model and an intuitive client API. org.owasp dependency-check-maven - ${dependency-check-maven.version} diff --git a/pulsar-io/docs/pom.xml b/pulsar-io/docs/pom.xml index ff23e0905954c..4100b08a2724e 100644 --- a/pulsar-io/docs/pom.xml +++ b/pulsar-io/docs/pom.xml @@ -253,7 +253,6 @@ org.owasp dependency-check-maven - ${dependency-check-maven.version} diff --git a/pulsar-io/flume/pom.xml b/pulsar-io/flume/pom.xml index f2a5e712c00b4..bcd62b412c1f3 100644 --- a/pulsar-io/flume/pom.xml +++ b/pulsar-io/flume/pom.xml @@ -141,7 +141,6 @@ org.owasp dependency-check-maven - ${dependency-check-maven.version} diff --git a/pulsar-io/hbase/pom.xml b/pulsar-io/hbase/pom.xml index 39986667bc191..88a7d22dfaa35 100644 --- a/pulsar-io/hbase/pom.xml +++ b/pulsar-io/hbase/pom.xml @@ -108,7 +108,6 @@ org.owasp dependency-check-maven - ${dependency-check-maven.version} diff --git a/pulsar-io/hdfs2/pom.xml b/pulsar-io/hdfs2/pom.xml index b613f7032051b..5c7cfb9d3eb59 100644 --- a/pulsar-io/hdfs2/pom.xml +++ b/pulsar-io/hdfs2/pom.xml @@ -27,14 +27,14 @@ pulsar-io-hdfs2 Pulsar IO :: Hdfs2 - + ${project.groupId} pulsar-io-core ${project.version} - + com.fasterxml.jackson.core jackson-databind @@ -74,7 +74,7 @@ commons-lang3 - + @@ -113,7 +113,6 @@ org.owasp dependency-check-maven - ${dependency-check-maven.version} diff --git a/pulsar-io/hdfs3/pom.xml b/pulsar-io/hdfs3/pom.xml index 00703154a61db..547e29e98643c 100644 --- a/pulsar-io/hdfs3/pom.xml +++ b/pulsar-io/hdfs3/pom.xml @@ -27,14 +27,14 @@ pulsar-io-hdfs3 Pulsar IO :: Hdfs3 - + ${project.groupId} pulsar-io-core ${project.version} - + com.fasterxml.jackson.core jackson-databind @@ -49,7 +49,7 @@ org.apache.commons commons-collections4 - + org.apache.hadoop hadoop-client @@ -80,7 +80,7 @@ - + @@ -119,7 +119,6 @@ org.owasp dependency-check-maven - ${dependency-check-maven.version} diff --git a/tiered-storage/file-system/pom.xml b/tiered-storage/file-system/pom.xml index ed637860e1d96..5b0d6ed34ef72 100644 --- a/tiered-storage/file-system/pom.xml +++ b/tiered-storage/file-system/pom.xml @@ -205,7 +205,6 @@ org.owasp dependency-check-maven - ${dependency-check-maven.version}