From 848c525c71165661851544ccbf804e4fc0078170 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 3 Jul 2024 17:41:07 +0300 Subject: [PATCH 01/11] Use pluginManagement to manage plugin version --- distribution/io/pom.xml | 1 - pom.xml | 7 +++++-- pulsar-io/docs/pom.xml | 1 - pulsar-io/flume/pom.xml | 1 - pulsar-io/hbase/pom.xml | 1 - pulsar-io/hdfs2/pom.xml | 7 +++---- pulsar-io/hdfs3/pom.xml | 9 ++++----- tiered-storage/file-system/pom.xml | 1 - 8 files changed, 12 insertions(+), 16 deletions(-) diff --git a/distribution/io/pom.xml b/distribution/io/pom.xml index bd65d5a81232b..96dd8b071106b 100644 --- a/distribution/io/pom.xml +++ b/distribution/io/pom.xml @@ -136,7 +136,6 @@ org.owasp dependency-check-maven - ${dependency-check-maven.version} diff --git a/pom.xml b/pom.xml index 7c556fa127786..f4305d57eaad1 100644 --- a/pom.xml +++ b/pom.xml @@ -2192,6 +2192,11 @@ flexible messaging model and an intuitive client API. build-helper-maven-plugin ${build-helper-maven-plugin.version} + + org.owasp + dependency-check-maven + ${dependency-check-maven.version} + @@ -2639,7 +2644,6 @@ flexible messaging model and an intuitive client API. org.owasp dependency-check-maven - ${dependency-check-maven.version} ${pulsar.basedir}/src/owasp-dependency-check-false-positives.xml @@ -2674,7 +2678,6 @@ flexible messaging model and an intuitive client API. org.owasp dependency-check-maven - ${dependency-check-maven.version} diff --git a/pulsar-io/docs/pom.xml b/pulsar-io/docs/pom.xml index 82c8f0bb6f96a..1e21656305b6c 100644 --- a/pulsar-io/docs/pom.xml +++ b/pulsar-io/docs/pom.xml @@ -258,7 +258,6 @@ org.owasp dependency-check-maven - ${dependency-check-maven.version} diff --git a/pulsar-io/flume/pom.xml b/pulsar-io/flume/pom.xml index 9b2839970ab79..86cec763cbe4a 100644 --- a/pulsar-io/flume/pom.xml +++ b/pulsar-io/flume/pom.xml @@ -141,7 +141,6 @@ org.owasp dependency-check-maven - ${dependency-check-maven.version} diff --git a/pulsar-io/hbase/pom.xml b/pulsar-io/hbase/pom.xml index 0c38d4f06d029..9fb98069a8ceb 100644 --- a/pulsar-io/hbase/pom.xml +++ b/pulsar-io/hbase/pom.xml @@ -108,7 +108,6 @@ org.owasp dependency-check-maven - ${dependency-check-maven.version} diff --git a/pulsar-io/hdfs2/pom.xml b/pulsar-io/hdfs2/pom.xml index 81b67f8e095fa..3b73adae46caa 100644 --- a/pulsar-io/hdfs2/pom.xml +++ b/pulsar-io/hdfs2/pom.xml @@ -27,14 +27,14 @@ pulsar-io-hdfs2 Pulsar IO :: Hdfs2 - + ${project.groupId} pulsar-io-core ${project.version} - + com.fasterxml.jackson.core jackson-databind @@ -74,7 +74,7 @@ commons-lang3 - + @@ -113,7 +113,6 @@ org.owasp dependency-check-maven - ${dependency-check-maven.version} diff --git a/pulsar-io/hdfs3/pom.xml b/pulsar-io/hdfs3/pom.xml index 3d9f185e37582..29a1c248c756f 100644 --- a/pulsar-io/hdfs3/pom.xml +++ b/pulsar-io/hdfs3/pom.xml @@ -27,14 +27,14 @@ pulsar-io-hdfs3 Pulsar IO :: Hdfs3 - + ${project.groupId} pulsar-io-core ${project.version} - + com.fasterxml.jackson.core jackson-databind @@ -49,7 +49,7 @@ org.apache.commons commons-collections4 - + org.apache.hadoop hadoop-client @@ -80,7 +80,7 @@ - + @@ -119,7 +119,6 @@ org.owasp dependency-check-maven - ${dependency-check-maven.version} diff --git a/tiered-storage/file-system/pom.xml b/tiered-storage/file-system/pom.xml index d20b92692fc58..03dc5371ef7f6 100644 --- a/tiered-storage/file-system/pom.xml +++ b/tiered-storage/file-system/pom.xml @@ -208,7 +208,6 @@ org.owasp dependency-check-maven - ${dependency-check-maven.version} From ef1381881b0fe6d8fb9103372b30c2487cd121de Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 3 Jul 2024 17:43:51 +0300 Subject: [PATCH 02/11] Use NIST_NVD_API_KEY as nvdApiKeyEnvironmentVariable --- .github/workflows/ci-owasp-dependency-check.yaml | 1 + .github/workflows/pulsar-ci.yaml | 1 + pom.xml | 3 +++ 3 files changed, 5 insertions(+) diff --git a/.github/workflows/ci-owasp-dependency-check.yaml b/.github/workflows/ci-owasp-dependency-check.yaml index a273e902c88d2..7f86d7fc4be37 100644 --- a/.github/workflows/ci-owasp-dependency-check.yaml +++ b/.github/workflows/ci-owasp-dependency-check.yaml @@ -26,6 +26,7 @@ on: env: MAVEN_OPTS: -Xss1500k -Xmx1024m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 JDK_DISTRIBUTION: corretto + NIST_NVD_API_KEY: ${{ secrets.NIST_NVD_API_KEY }} jobs: run-owasp-dependency-check: diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index 8decde1c999ca..bc78a8381b4fd 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -1427,6 +1427,7 @@ jobs: env: GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} + NIST_NVD_API_KEY: ${{ secrets.NIST_NVD_API_KEY }} steps: - name: checkout uses: actions/checkout@v4 diff --git a/pom.xml b/pom.xml index f4305d57eaad1..b3eae4e7718f5 100644 --- a/pom.xml +++ b/pom.xml @@ -2196,6 +2196,9 @@ flexible messaging model and an intuitive client API. org.owasp dependency-check-maven ${dependency-check-maven.version} + + NIST_NVD_API_KEY + From 6fd64976473e0b68e727458ce17c3163650c16b5 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 3 Jul 2024 18:07:25 +0300 Subject: [PATCH 03/11] Upgrade plugin version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b3eae4e7718f5..f6c79a7f6b755 100644 --- a/pom.xml +++ b/pom.xml @@ -316,7 +316,7 @@ flexible messaging model and an intuitive client API. 0.1.21 1.3 0.4 - 9.1.0 + 10.0.1 1.0.6 1.6.1 6.4.0 From 9377871b5ef3d560e54be3ef4fdc0330ab0f5dc3 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 3 Jul 2024 18:21:52 +0300 Subject: [PATCH 04/11] Increase max heap size for maven --- .github/workflows/ci-owasp-dependency-check.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-owasp-dependency-check.yaml b/.github/workflows/ci-owasp-dependency-check.yaml index 7f86d7fc4be37..e4f6d6173d0f2 100644 --- a/.github/workflows/ci-owasp-dependency-check.yaml +++ b/.github/workflows/ci-owasp-dependency-check.yaml @@ -24,7 +24,7 @@ on: workflow_dispatch: env: - MAVEN_OPTS: -Xss1500k -Xmx1024m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 + MAVEN_OPTS: -Xss1500k -Xmx1500m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 JDK_DISTRIBUTION: corretto NIST_NVD_API_KEY: ${{ secrets.NIST_NVD_API_KEY }} From b4c5aeab1396854af58c9c308e2b8cbe61c6c23e Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 3 Jul 2024 18:23:27 +0300 Subject: [PATCH 05/11] Fix cache lookup --- .github/workflows/ci-owasp-dependency-check.yaml | 6 ++---- .github/workflows/pulsar-ci.yaml | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci-owasp-dependency-check.yaml b/.github/workflows/ci-owasp-dependency-check.yaml index e4f6d6173d0f2..97b1f6e18c1e4 100644 --- a/.github/workflows/ci-owasp-dependency-check.yaml +++ b/.github/workflows/ci-owasp-dependency-check.yaml @@ -59,16 +59,14 @@ jobs: - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - - name: Cache local Maven repository - uses: actions/cache@v4 + - name: Restore Maven repository cache + uses: actions/cache/restore@v4 timeout-minutes: 5 with: path: | ~/.m2/repository/*/*/* !~/.m2/repository/org/apache/pulsar - !~/.m2/repository/org/owasp/dependency-check-data key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} - lookup-only: true restore-keys: | ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} ${{ runner.os }}-m2-dependencies-core-modules- diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index bc78a8381b4fd..d7dec94a84323 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -1443,16 +1443,14 @@ jobs: with: limit-access-to-actor: true - - name: Cache Maven dependencies - uses: actions/cache@v4 + - name: Restore Maven repository cache + uses: actions/cache/restore@v4 timeout-minutes: 5 with: path: | ~/.m2/repository/*/*/* !~/.m2/repository/org/apache/pulsar - !~/.m2/repository/org/owasp/dependency-check-data key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - lookup-only: true restore-keys: | ${{ runner.os }}-m2-dependencies-core-modules- From 755ee86cfab3286a79aba464098824ef1c354680 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 3 Jul 2024 19:04:49 +0300 Subject: [PATCH 06/11] Add -DnarPluginPhase=none to skip nar file building --- .github/workflows/ci-owasp-dependency-check.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-owasp-dependency-check.yaml b/.github/workflows/ci-owasp-dependency-check.yaml index 97b1f6e18c1e4..8ee24f7c1452d 100644 --- a/.github/workflows/ci-owasp-dependency-check.yaml +++ b/.github/workflows/ci-owasp-dependency-check.yaml @@ -78,7 +78,7 @@ jobs: java-version: ${{ matrix.jdk || '17' }} - name: run install by skip tests - run: mvn -B -ntp clean install -DskipTests -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true -DskipDocker=true + run: mvn -B -ntp clean install -DskipTests -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true -DskipDocker=true -DnarPluginPhase=none - name: OWASP cache key weeknum id: get-weeknum From 45d0b11ed3da288f6c0c5390f327fff0b7e9709b Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 3 Jul 2024 19:20:21 +0300 Subject: [PATCH 07/11] Fix building without building nar files --- .github/workflows/ci-owasp-dependency-check.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-owasp-dependency-check.yaml b/.github/workflows/ci-owasp-dependency-check.yaml index 8ee24f7c1452d..842e3d281853c 100644 --- a/.github/workflows/ci-owasp-dependency-check.yaml +++ b/.github/workflows/ci-owasp-dependency-check.yaml @@ -78,7 +78,7 @@ jobs: java-version: ${{ matrix.jdk || '17' }} - name: run install by skip tests - run: mvn -B -ntp clean install -DskipTests -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true -DskipDocker=true -DnarPluginPhase=none + run: mvn -B -ntp clean install -DskipTests -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true -DskipDocker=true -DnarPluginPhase=none -pl '!distribution/io,!distribution/offloaders' - name: OWASP cache key weeknum id: get-weeknum From 53da04e61516f4093cb488c636b78e22bbd66932 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 3 Jul 2024 19:24:18 +0300 Subject: [PATCH 08/11] Scan only maintained branches --- .github/workflows/ci-owasp-dependency-check.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/ci-owasp-dependency-check.yaml b/.github/workflows/ci-owasp-dependency-check.yaml index 842e3d281853c..95ced006e643a 100644 --- a/.github/workflows/ci-owasp-dependency-check.yaml +++ b/.github/workflows/ci-owasp-dependency-check.yaml @@ -43,12 +43,9 @@ jobs: matrix: include: - branch: master + - branch: branch-3.3 - branch: branch-3.2 - - branch: branch-3.1 - branch: branch-3.0 - - branch: branch-2.11 - - branch: branch-2.10 - jdk: 11 steps: - name: checkout From 53200b413cc3bd6de694dcb9910469637656db42 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 3 Jul 2024 19:56:20 +0300 Subject: [PATCH 09/11] Add nvdDatafeedUrl to speed up database download --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index f6c79a7f6b755..54fda3fb0f5f7 100644 --- a/pom.xml +++ b/pom.xml @@ -2198,6 +2198,7 @@ flexible messaging model and an intuitive client API. ${dependency-check-maven.version} NIST_NVD_API_KEY + https://jeremylong.github.io/DependencyCheck/hb_nvd/ From ddafa6ac1ef7c83c7b9fd6e9c2dc6063cdeead83 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 3 Jul 2024 20:40:10 +0300 Subject: [PATCH 10/11] Upgrade to v4 version --- .github/workflows/ci-owasp-dependency-check.yaml | 4 ++-- .github/workflows/pulsar-ci.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-owasp-dependency-check.yaml b/.github/workflows/ci-owasp-dependency-check.yaml index 95ced006e643a..a70f4a82ff1af 100644 --- a/.github/workflows/ci-owasp-dependency-check.yaml +++ b/.github/workflows/ci-owasp-dependency-check.yaml @@ -85,7 +85,7 @@ jobs: - name: Restore OWASP Dependency Check data id: restore-owasp-dependency-check-data - uses: actions/cache/restore@v3 + uses: actions/cache/restore@v4 timeout-minutes: 5 with: path: ~/.m2/repository/org/owasp/dependency-check-data @@ -101,7 +101,7 @@ jobs: - name: Save OWASP Dependency Check data if: ${{ steps.update-owasp-dependency-check-data.outcome == 'success' }} - uses: actions/cache/save@v3 + uses: actions/cache/save@v4 timeout-minutes: 5 with: path: ~/.m2/repository/org/owasp/dependency-check-data diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index d7dec94a84323..828f876f13194 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -1479,7 +1479,7 @@ jobs: - name: Restore OWASP Dependency Check data id: restore-owasp-dependency-check-data - uses: actions/cache/restore@v3 + uses: actions/cache/restore@v4 timeout-minutes: 5 with: path: ~/.m2/repository/org/owasp/dependency-check-data From 585912e9ba8ec4e9de899d17e176bf52d6d3b30a Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 3 Jul 2024 20:43:39 +0300 Subject: [PATCH 11/11] Comment out nvdDatafeedUrl --- pom.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 54fda3fb0f5f7..d42eac2d5af59 100644 --- a/pom.xml +++ b/pom.xml @@ -2198,7 +2198,8 @@ flexible messaging model and an intuitive client API. ${dependency-check-maven.version} NIST_NVD_API_KEY - https://jeremylong.github.io/DependencyCheck/hb_nvd/ + +