From 62cc62ef722fbe47277704e45166aefaee8d1ac4 Mon Sep 17 00:00:00 2001 From: Qingsheng Ren Date: Fri, 10 May 2024 12:20:33 +0800 Subject: [PATCH] [FLINK-35309][cdc] Introduce CI checks for licenses and NOTICE Co-authored-by: GOODBOY008 --- .github/workflows/flink_cdc.yml | 35 +- pom.xml | 28 + tools/ci/flink-cdc-ci-tools/pom.xml | 82 +++ .../tools/ci/licensecheck/JarFileChecker.java | 321 +++++++++++ .../tools/ci/licensecheck/LicenseChecker.java | 53 ++ .../ci/licensecheck/NoticeFileChecker.java | 392 +++++++++++++ .../ci/optional/ShadeOptionalChecker.java | 266 +++++++++ .../ci/suffixcheck/ScalaSuffixChecker.java | 269 +++++++++ .../ci/utils/dependency/DependencyParser.java | 228 ++++++++ .../tools/ci/utils/deploy/DeployParser.java | 73 +++ .../tools/ci/utils/notice/NoticeContents.java | 42 ++ .../tools/ci/utils/notice/NoticeParser.java | 94 ++++ .../cdc/tools/ci/utils/shade/ShadeParser.java | 112 ++++ .../cdc/tools/ci/utils/shared/Dependency.java | 134 +++++ .../tools/ci/utils/shared/DependencyTree.java | 137 +++++ .../tools/ci/utils/shared/ParserUtils.java | 73 +++ .../src/main/resources/log4j2.properties | 25 + ...es-defining-excess-dependencies.modulelist | 20 + .../ci/licensecheck/JarFileCheckerTest.java | 525 ++++++++++++++++++ .../licensecheck/NoticeFileCheckerTest.java | 209 +++++++ .../ci/optional/ShadeOptionalCheckerTest.java | 181 ++++++ .../dependency/DependencyParserCopyTest.java | 121 ++++ .../dependency/DependencyParserTreeTest.java | 148 +++++ .../ci/utils/deploy/DeployParserTest.java | 57 ++ .../ci/utils/notice/NoticeParserTest.java | 93 ++++ .../tools/ci/utils/shade/ShadeParserTest.java | 95 ++++ .../ci/utils/shared/DependencyTreeTest.java | 96 ++++ .../org.junit.jupiter.api.extension.Extension | 16 + tools/ci/license_check.rb | 135 ----- 29 files changed, 3906 insertions(+), 154 deletions(-) create mode 100644 tools/ci/flink-cdc-ci-tools/pom.xml create mode 100644 tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/licensecheck/JarFileChecker.java create mode 100644 tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/licensecheck/LicenseChecker.java create mode 100644 tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/licensecheck/NoticeFileChecker.java create mode 100644 tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/optional/ShadeOptionalChecker.java create mode 100644 tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/suffixcheck/ScalaSuffixChecker.java create mode 100644 tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/utils/dependency/DependencyParser.java create mode 100644 tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/utils/deploy/DeployParser.java create mode 100644 tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/utils/notice/NoticeContents.java create mode 100644 tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/utils/notice/NoticeParser.java create mode 100644 tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/utils/shade/ShadeParser.java create mode 100644 tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/utils/shared/Dependency.java create mode 100644 tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/utils/shared/DependencyTree.java create mode 100644 tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/utils/shared/ParserUtils.java create mode 100644 tools/ci/flink-cdc-ci-tools/src/main/resources/log4j2.properties create mode 100644 tools/ci/flink-cdc-ci-tools/src/main/resources/modules-defining-excess-dependencies.modulelist create mode 100644 tools/ci/flink-cdc-ci-tools/src/test/java/org/apache/flink/tools/ci/licensecheck/JarFileCheckerTest.java create mode 100644 tools/ci/flink-cdc-ci-tools/src/test/java/org/apache/flink/tools/ci/licensecheck/NoticeFileCheckerTest.java create mode 100644 tools/ci/flink-cdc-ci-tools/src/test/java/org/apache/flink/tools/ci/optional/ShadeOptionalCheckerTest.java create mode 100644 tools/ci/flink-cdc-ci-tools/src/test/java/org/apache/flink/tools/ci/utils/dependency/DependencyParserCopyTest.java create mode 100644 tools/ci/flink-cdc-ci-tools/src/test/java/org/apache/flink/tools/ci/utils/dependency/DependencyParserTreeTest.java create mode 100644 tools/ci/flink-cdc-ci-tools/src/test/java/org/apache/flink/tools/ci/utils/deploy/DeployParserTest.java create mode 100644 tools/ci/flink-cdc-ci-tools/src/test/java/org/apache/flink/tools/ci/utils/notice/NoticeParserTest.java create mode 100644 tools/ci/flink-cdc-ci-tools/src/test/java/org/apache/flink/tools/ci/utils/shade/ShadeParserTest.java create mode 100644 tools/ci/flink-cdc-ci-tools/src/test/java/org/apache/flink/tools/ci/utils/shared/DependencyTreeTest.java create mode 100644 tools/ci/flink-cdc-ci-tools/src/test/resources/META-INF/services/org.junit.jupiter.api.extension.Extension delete mode 100755 tools/ci/license_check.rb diff --git a/.github/workflows/flink_cdc.yml b/.github/workflows/flink_cdc.yml index 85ad4b3b15e..61e05301c0a 100644 --- a/.github/workflows/flink_cdc.yml +++ b/.github/workflows/flink_cdc.yml @@ -100,23 +100,29 @@ env: jobs: license_check: runs-on: ubuntu-latest + env: + MVN_COMMON_OPTIONS: -U -B --no-transfer-progress + MVN_BUILD_OUTPUT_FILE: "/tmp/mvn_build_output.out" + MVN_VALIDATION_DIR: "/tmp/flink-validation-deployment" steps: - name: Check out repository code uses: actions/checkout@v4 with: submodules: true - - name: Set up Ruby environment - uses: ruby/setup-ruby@v1 - with: - ruby-version: '3.3' - - name: Compiling jar packages - run: mvn --no-snapshot-updates -B package -DskipTests + - name: Build + run: | + set -o pipefail + + mvn clean deploy ${{ env.MVN_COMMON_OPTIONS }} -DskipTests \ + -DaltDeploymentRepository=validation_repository::default::file:${{ env.MVN_VALIDATION_DIR }} \ + | tee ${{ env.MVN_BUILD_OUTPUT_FILE }} + - name: Run license check - run: gem install rubyzip -v 2.3.0 && ./tools/ci/license_check.rb + run: | + mvn ${{ env.MVN_COMMON_OPTIONS }} exec:java@check-license -N \ + -Dexec.args="${{ env.MVN_BUILD_OUTPUT_FILE }} $(pwd) ${{ env.MVN_VALIDATION_DIR }}" compile_and_test: - # Only run the CI pipeline for the flink-cdc-connectors repository -# if: github.repository == 'apache/flink-cdc-connectors' runs-on: ubuntu-latest strategy: matrix: @@ -135,15 +141,6 @@ jobs: "e2e" ] timeout-minutes: 120 - env: - MVN_COMMON_OPTIONS: -Dmaven.wagon.http.pool=false \ - -Dorg.slf4j.simpleLogger.showDateTime=true \ - -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss.SSS \ - -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \ - --no-snapshot-updates -B \ - --settings /home/vsts/work/1/s/tools/ci/google-mirror-settings.xml \ - -Dfast -Dlog.dir=/home/vsts/work/_temp/debug_files \ - -Dlog4j.configurationFile=file:///home/vsts/work/1/s/tools/ci/log4j.properties steps: - run: echo "Running CI pipeline for JDK version ${{ matrix.jdk }}" @@ -249,4 +246,4 @@ jobs: jcmd $java_pid GC.heap_info || true done fi - exit 0 + exit 0 \ No newline at end of file diff --git a/pom.xml b/pom.xml index b15b82da3c7..4a9bebbb1bd 100644 --- a/pom.xml +++ b/pom.xml @@ -40,6 +40,7 @@ limitations under the License. flink-cdc-connect flink-cdc-runtime flink-cdc-e2e-tests + tools/ci/flink-cdc-ci-tools @@ -675,6 +676,33 @@ limitations under the License. + + org.codehaus.mojo + exec-maven-plugin + 3.1.0 + + + check-license + + none + + java + + + org.apache.flink.cdc.tools.ci.licensecheck.LicenseChecker + true + false + + + + + + org.apache.flink + flink-cdc-ci-tools + ${revision} + + + diff --git a/tools/ci/flink-cdc-ci-tools/pom.xml b/tools/ci/flink-cdc-ci-tools/pom.xml new file mode 100644 index 00000000000..73701e9f72c --- /dev/null +++ b/tools/ci/flink-cdc-ci-tools/pom.xml @@ -0,0 +1,82 @@ + + + + + 4.0.0 + + + org.apache.flink + flink-cdc-parent + ${revision} + ../../.. + + + flink-cdc-ci-tools + ${revision} + + + true + + + + + org.apache.flink + flink-annotations + ${flink.version} + + + org.apache.flink + flink-test-utils-junit + ${flink.version} + test + + + org.apache.logging.log4j + log4j-slf4j-impl + ${log4j.version} + compile + + + org.apache.logging.log4j + ${log4j.version} + log4j-api + compile + + + org.apache.logging.log4j + ${log4j.version} + log4j-core + compile + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + + + + diff --git a/tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/licensecheck/JarFileChecker.java b/tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/licensecheck/JarFileChecker.java new file mode 100644 index 00000000000..884fd885dec --- /dev/null +++ b/tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/licensecheck/JarFileChecker.java @@ -0,0 +1,321 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.tools.ci.licensecheck; + +import org.apache.flink.annotation.VisibleForTesting; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.URI; +import java.nio.charset.MalformedInputException; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** Checks the Jar files created by the build process. */ +public class JarFileChecker { + private static final Logger LOG = LoggerFactory.getLogger(JarFileChecker.class); + + public static int checkPath(Path path) throws Exception { + List files = getBuildJars(path); + LOG.info("Checking directory {} with a total of {} jar files.", path, files.size()); + + int severeIssues = 0; + for (Path file : files) { + severeIssues += checkJar(file); + } + + return severeIssues; + } + + private static List getBuildJars(Path path) throws IOException { + return Files.walk(path) + .filter(file -> file.toString().endsWith(".jar")) + .collect(Collectors.toList()); + } + + @VisibleForTesting + static int checkJar(Path file) throws Exception { + final URI uri = file.toUri(); + + int numSevereIssues = 0; + try (final FileSystem fileSystem = + FileSystems.newFileSystem( + new URI("jar:file", uri.getHost(), uri.getPath(), uri.getFragment()), + Collections.emptyMap())) { + if (isTestJarAndEmpty(file, fileSystem.getPath("/"))) { + return 0; + } + if (!noticeFileExistsAndIsValid(fileSystem.getPath("META-INF", "NOTICE"), file)) { + numSevereIssues++; + } + if (!licenseFileExistsAndIsValid(fileSystem.getPath("META-INF", "LICENSE"), file)) { + numSevereIssues++; + } + + numSevereIssues += + getNumLicenseFilesOutsideMetaInfDirectory(file, fileSystem.getPath("/")); + + numSevereIssues += getFilesWithIncompatibleLicenses(file, fileSystem.getPath("/")); + } + return numSevereIssues; + } + + private static boolean isTestJarAndEmpty(Path jar, Path jarRoot) throws IOException { + if (jar.getFileName().toString().endsWith("-tests.jar")) { + try (Stream files = Files.walk(jarRoot)) { + long numClassFiles = + files.filter(path -> !path.equals(jarRoot)) + .filter(path -> path.getFileName().toString().endsWith(".class")) + .count(); + if (numClassFiles == 0) { + return true; + } + } + } + + return false; + } + + private static boolean noticeFileExistsAndIsValid(Path noticeFile, Path jar) + throws IOException { + if (!Files.exists(noticeFile)) { + LOG.error("Missing META-INF/NOTICE in {}", jar); + return false; + } + + final String noticeFileContents = readFile(noticeFile); + if (!noticeFileContents.toLowerCase().contains("flink") + || !noticeFileContents.contains("The Apache Software Foundation")) { + LOG.error("The notice file in {} does not contain the expected entries.", jar); + return false; + } + + return true; + } + + private static boolean licenseFileExistsAndIsValid(Path licenseFile, Path jar) + throws IOException { + if (!Files.exists(licenseFile)) { + LOG.error("Missing META-INF/LICENSE in {}", jar); + return false; + } + + final String licenseFileContents = readFile(licenseFile); + if (!licenseFileContents.contains("Apache License") + || !licenseFileContents.contains("Version 2.0, January 2004")) { + LOG.error("The license file in {} does not contain the expected entries.", jar); + return false; + } + + return true; + } + + private static int getFilesWithIncompatibleLicenses(Path jar, Path jarRoot) throws IOException { + // patterns are based on https://www.apache.org/legal/resolved.html#category-x + return findNonBinaryFilesContainingText( + jar, + jarRoot, + asPatterns( + "Binary Code License", + "Intel Simplified Software License", + "JSR 275", + "Microsoft Limited Public License", + "Amazon Software License", + // Java SDK for Satori RTM license + "as necessary for your use of Satori services", + "REDIS SOURCE AVAILABLE LICENSE", + "Booz Allen Public License", + "Confluent Community License Agreement Version 1.0", + // “Commons Clause” License Condition v1.0 + "the License does not grant to you, the right to Sell the Software.", + "Sun Community Source License Version 3.0", + "GNU General Public License", + "GNU Affero General Public License", + "GNU Lesser General Public License", + "Q Public License", + "Sleepycat License", + "Server Side Public License", + "Code Project Open License", + // BSD 4-Clause + " All advertising materials mentioning features or use of this software must display the following acknowledgement", + // Facebook Patent clause v1 + "The license granted hereunder will terminate, automatically and without notice, for anyone that makes any claim", + // Facebook Patent clause v2 + "The license granted hereunder will terminate, automatically and without notice, if you (or any of your subsidiaries, corporate affiliates or agents) initiate directly or indirectly, or take a direct financial interest in, any Patent Assertion: (i) against Facebook", + "Netscape Public License", + "SOLIPSISTIC ECLIPSE PUBLIC LICENSE", + // DON'T BE A DICK PUBLIC LICENSE + "Do whatever you like with the original work, just don't be a dick.", + // JSON License + "The Software shall be used for Good, not Evil.", + // can sometimes be found in "funny" licenses + "Don’t be evil")); + } + + private static Collection asPatterns(String... texts) { + return Stream.of(texts) + .map(JarFileChecker::asPatternWithPotentialLineBreaks) + .collect(Collectors.toList()); + } + + private static Pattern asPatternWithPotentialLineBreaks(String text) { + // allows word sequences to be separated by whitespace, line-breaks and comments(//, #) + return Pattern.compile(text.toLowerCase(Locale.ROOT).replaceAll(" ", " ?\\\\R?[\\\\s/#]*")); + } + + private static int findNonBinaryFilesContainingText( + Path jar, Path jarRoot, Collection forbidden) throws IOException { + try (Stream files = Files.walk(jarRoot)) { + return files.filter(path -> !path.equals(jarRoot)) + .filter(path -> !Files.isDirectory(path)) + .filter(JarFileChecker::isNoClassFile) + // frequent false-positives due to dual-licensing; generated by maven + .filter(path -> !getFileName(path).equals("dependencies")) + // false-positives due to dual-licensing; use startsWith to cover .txt/.md files + .filter(path -> !getFileName(path).startsWith("license")) + // false-positives due to optional components; startsWith covers .txt/.md files + .filter(path -> !getFileName(path).startsWith("notice")) + // dual-licensed under GPL 2 and CDDL 1.1 + .filter(path -> !pathStartsWith(path, "/javax/annotation")) + .filter(path -> !pathStartsWith(path, "/javax/servlet")) + .filter(path -> !pathStartsWith(path, "/javax/xml/bind")) + .filter(path -> !isJavaxManifest(jar, path)) + // dual-licensed under GPL 2 and EPL 2.0 + .filter(path -> !pathStartsWith(path, "/org/glassfish/jersey")) + .map( + path -> { + try { + final String fileContents; + try { + fileContents = readFile(path).toLowerCase(Locale.ROOT); + } catch (MalformedInputException mie) { + // binary file + return 0; + } + + int violations = 0; + for (Pattern text : forbidden) { + if (text.matcher(fileContents).find()) { + // do not count individual violations because it can be + // confusing when checking with aliases for the same + // license + violations = 1; + LOG.error( + "File '{}' in jar '{}' contains match with forbidden regex '{}'.", + path, + jar, + text); + } + } + return violations; + } catch (IOException e) { + throw new RuntimeException( + String.format( + "Could not read contents of file '%s' in jar '%s'.", + path, jar), + e); + } + }) + .reduce(Integer::sum) + .orElse(0); + } + } + + private static int getNumLicenseFilesOutsideMetaInfDirectory(Path jar, Path jarRoot) + throws IOException { + try (Stream files = Files.walk(jarRoot)) { + /* + * LICENSE or NOTICE files found outside of the META-INF directory are most likely shading mistakes + * (we are including the files from other dependencies, thus providing an invalid LICENSE file) + * + *

In such a case, we recommend updating the shading exclusions, and adding the license file + * to META-INF/licenses. + */ + final List filesWithIssues = + files.filter(path -> !path.equals(jarRoot)) + .filter( + path -> + getFileName(path).contains("license") + || getFileName(path).contains("notice")) + // ignore directories, e.g. "license/" + .filter(path -> !Files.isDirectory(path)) + // some class files contain LICENSE in their name + .filter(JarFileChecker::isNoClassFile) + // false-positive in Doris and Starrocks + .filter(path -> !getFileName(path).endsWith(".ftl")) + .map(Path::toString) + // license files in META-INF are expected + .filter(path -> !path.contains("META-INF")) + // false-positive for Microsoft SQL Server + .filter(path -> !path.startsWith("/container-license-acceptance")) + .collect(Collectors.toList()); + for (String fileWithIssue : filesWithIssues) { + LOG.error( + "Jar file {} contains a LICENSE file in an unexpected location: {}", + jar, + fileWithIssue); + } + return filesWithIssues.size(); + } + } + + private static String getFileName(Path path) { + return path.getFileName().toString().toLowerCase(); + } + + private static boolean pathStartsWith(Path file, String path) { + return file.startsWith(file.getFileSystem().getPath(path)); + } + + private static boolean equals(Path file, String path) { + return file.equals(file.getFileSystem().getPath(path)); + } + + private static boolean isNoClassFile(Path file) { + return !getFileName(file).endsWith(".class"); + } + + private static boolean isJavaxManifest(Path jar, Path potentialManifestFile) { + try { + return equals(potentialManifestFile, "/META-INF/versions/11/META-INF/MANIFEST.MF") + && readFile(potentialManifestFile).contains("Specification-Title: jaxb-api"); + } catch (IOException e) { + throw new RuntimeException( + String.format( + "Error while reading file %s from jar %s.", potentialManifestFile, jar), + e); + } + } + + private static String readFile(Path file) throws IOException { + return new String(Files.readAllBytes(file), StandardCharsets.UTF_8); + } +} diff --git a/tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/licensecheck/LicenseChecker.java b/tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/licensecheck/LicenseChecker.java new file mode 100644 index 00000000000..428c18048b3 --- /dev/null +++ b/tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/licensecheck/LicenseChecker.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.tools.ci.licensecheck; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.nio.file.Paths; + +/** Utility for checking all things related to License and Notice files. */ +public class LicenseChecker { + // ---------------------------------------- Launcher ---------------------------------------- // + + private static final Logger LOG = LoggerFactory.getLogger(LicenseChecker.class); + + public static void main(String[] args) throws Exception { + if (args.length < 2) { + System.out.println( + "Usage: LicenseChecker "); + System.exit(1); + } + LOG.warn( + "THIS UTILITY IS ONLY CHECKING FOR COMMON LICENSING MISTAKES. A MANUAL CHECK OF THE NOTICE FILES, DEPLOYED ARTIFACTS, ETC. IS STILL NEEDED!"); + + int severeIssueCount = NoticeFileChecker.run(new File(args[0]), Paths.get(args[1])); + + severeIssueCount += JarFileChecker.checkPath(Paths.get(args[2])); + + if (severeIssueCount > 0) { + LOG.warn("Found a total of {} severe license issues", severeIssueCount); + + System.exit(1); + } + LOG.info("License check completed without severe issues."); + } +} diff --git a/tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/licensecheck/NoticeFileChecker.java b/tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/licensecheck/NoticeFileChecker.java new file mode 100644 index 00000000000..252e8ebf5ac --- /dev/null +++ b/tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/licensecheck/NoticeFileChecker.java @@ -0,0 +1,392 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.tools.ci.licensecheck; + +import org.apache.flink.annotation.VisibleForTesting; +import org.apache.flink.cdc.tools.ci.utils.dependency.DependencyParser; +import org.apache.flink.cdc.tools.ci.utils.deploy.DeployParser; +import org.apache.flink.cdc.tools.ci.utils.notice.NoticeContents; +import org.apache.flink.cdc.tools.ci.utils.notice.NoticeParser; +import org.apache.flink.cdc.tools.ci.utils.shade.ShadeParser; +import org.apache.flink.cdc.tools.ci.utils.shared.Dependency; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nullable; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** Utility class checking for proper NOTICE files based on the maven build output. */ +public class NoticeFileChecker { + + private static final Logger LOG = LoggerFactory.getLogger(NoticeFileChecker.class); + + private static final List MODULES_DEFINING_EXCESS_DEPENDENCIES = + loadFromResources("modules-defining-excess-dependencies.modulelist"); + + // Examples: + // "- org.apache.htrace:htrace-core:3.1.0-incubating" + // or + // "This project bundles "net.jcip:jcip-annotations:1.0". + private static final Pattern NOTICE_DEPENDENCY_PATTERN = + Pattern.compile( + "- ([^ :]+):([^:]+):([^ ]+)($| )|.*bundles \"([^:]+):([^:]+):([^\"]+)\".*"); + + static int run(File buildResult, Path root) throws IOException { + // parse included dependencies from build output + final Map> modulesWithBundledDependencies = + combineAndFilterFlinkDependencies( + ShadeParser.parseShadeOutput(buildResult.toPath()), + DependencyParser.parseDependencyCopyOutput(buildResult.toPath())); + + final Set deployedModules = DeployParser.parseDeployOutput(buildResult); + + LOG.info( + "Extracted " + + deployedModules.size() + + " modules that were deployed and " + + modulesWithBundledDependencies.keySet().size() + + " modules which bundle dependencies with a total of " + + modulesWithBundledDependencies.values().size() + + " dependencies"); + + // find modules producing a shaded-jar + List noticeFiles = findNoticeFiles(root); + LOG.info("Found {} NOTICE files to check", noticeFiles.size()); + + final Map> moduleToNotice = + noticeFiles.stream() + .collect( + Collectors.toMap( + NoticeFileChecker::getModuleFromNoticeFile, + noticeFile -> { + try { + return NoticeParser.parseNoticeFile(noticeFile); + } catch (IOException e) { + // some machine issue + throw new RuntimeException(e); + } + })); + + return run(modulesWithBundledDependencies, deployedModules, moduleToNotice); + } + + @VisibleForTesting + static int run( + Map> modulesWithBundledDependencies, + Set deployedModules, + Map> noticeFiles) + throws IOException { + int severeIssueCount = 0; + + final Set modulesSkippingDeployment = + new HashSet<>(modulesWithBundledDependencies.keySet()); + modulesSkippingDeployment.removeAll(deployedModules); + + LOG.debug( + "The following {} modules are skipping deployment: {}", + modulesSkippingDeployment.size(), + modulesSkippingDeployment.stream() + .sorted() + .collect(Collectors.joining("\n\t", "\n\t", ""))); + + for (String moduleSkippingDeployment : modulesSkippingDeployment) { + // TODO: this doesn't work for modules requiring a NOTICE that are bundled indirectly + // TODO: via another non-deployed module + boolean bundledByDeployedModule = + modulesWithBundledDependencies.entrySet().stream() + .filter( + entry -> + entry.getValue().stream() + .map(Dependency::getArtifactId) + .anyMatch( + artifactId -> + artifactId.equals( + moduleSkippingDeployment))) + .anyMatch(entry -> !modulesSkippingDeployment.contains(entry.getKey())); + + if (!bundledByDeployedModule) { + modulesWithBundledDependencies.remove(moduleSkippingDeployment); + } else { + LOG.debug( + "Including module {} in license checks, despite not being deployed, because it is bundled by another deployed module.", + moduleSkippingDeployment); + } + } + + // check that all required NOTICE files exists + severeIssueCount += + ensureRequiredNoticeFiles(modulesWithBundledDependencies, noticeFiles.keySet()); + + // check each NOTICE file + for (Map.Entry> noticeFile : noticeFiles.entrySet()) { + severeIssueCount += + checkNoticeFileAndLogProblems( + modulesWithBundledDependencies, + noticeFile.getKey(), + noticeFile.getValue().orElse(null)); + } + + return severeIssueCount; + } + + private static Map> combineAndFilterFlinkDependencies( + Map> modulesWithBundledDependencies, + Map> modulesWithCopiedDependencies) { + + final Map> combinedAndFiltered = new LinkedHashMap<>(); + + Stream.concat( + modulesWithBundledDependencies.entrySet().stream(), + modulesWithCopiedDependencies.entrySet().stream()) + .forEach( + (entry) -> { + final Set dependencies = + combinedAndFiltered.computeIfAbsent( + entry.getKey(), ignored -> new LinkedHashSet<>()); + + for (Dependency dependency : entry.getValue()) { + if (!dependency.getGroupId().contains("org.apache.flink")) { + dependencies.add(dependency); + } + } + }); + + return combinedAndFiltered; + } + + private static int ensureRequiredNoticeFiles( + Map> modulesWithShadedDependencies, + Collection modulesWithNoticeFile) { + int severeIssueCount = 0; + Set shadingModules = new HashSet<>(modulesWithShadedDependencies.keySet()); + shadingModules.removeAll(modulesWithNoticeFile); + for (String moduleWithoutNotice : shadingModules) { + if (modulesWithShadedDependencies.get(moduleWithoutNotice).stream() + .anyMatch(dependency -> !dependency.getGroupId().equals("org.apache.flink"))) { + LOG.error( + "Module {} is missing a NOTICE file. It has shaded dependencies: {}", + moduleWithoutNotice, + modulesWithShadedDependencies.get(moduleWithoutNotice).stream() + .map(Dependency::toString) + .collect(Collectors.joining("\n\t", "\n\t", ""))); + severeIssueCount++; + } + } + return severeIssueCount; + } + + private static String getModuleFromNoticeFile(Path noticeFile) { + Path moduleDirectory = + noticeFile + .getParent() // META-INF + .getParent() // resources + .getParent() // main + .getParent() // src + .getParent(); // <-- module name + return moduleDirectory.getFileName().toString(); + } + + private static int checkNoticeFileAndLogProblems( + Map> modulesWithShadedDependencies, + String moduleName, + @Nullable NoticeContents noticeContents) + throws IOException { + + final Map> problemsBySeverity = + checkNoticeFile(modulesWithShadedDependencies, moduleName, noticeContents); + + final List severeProblems = + problemsBySeverity.getOrDefault(Severity.CRITICAL, Collections.emptyList()); + + if (!problemsBySeverity.isEmpty()) { + final List toleratedProblems = + problemsBySeverity.getOrDefault(Severity.TOLERATED, Collections.emptyList()); + final List expectedProblems = + problemsBySeverity.getOrDefault(Severity.SUPPRESSED, Collections.emptyList()); + + LOG.info( + "Problems were detected for a NOTICE file.\n" + "\t{}:\n" + "{}{}{}", + moduleName, + convertProblemsToIndentedString( + severeProblems, + "These issue are legally problematic and MUST be fixed:"), + convertProblemsToIndentedString( + toleratedProblems, + "These issues are mistakes that aren't legally problematic. They SHOULD be fixed at some point, but we don't have to:"), + convertProblemsToIndentedString( + expectedProblems, "These issues are assumed to be false-positives:")); + } + + return severeProblems.size(); + } + + @VisibleForTesting + static Map> checkNoticeFile( + Map> modulesWithShadedDependencies, + String moduleName, + @Nullable NoticeContents noticeContents) { + + final Map> problemsBySeverity = new HashMap<>(); + + if (noticeContents == null) { + addProblem(problemsBySeverity, Severity.CRITICAL, "The NOTICE file was empty."); + } else { + // first line must be the module name. + if (!noticeContents.getNoticeModuleName().equals(moduleName)) { + addProblem( + problemsBySeverity, + Severity.TOLERATED, + String.format( + "First line does not start with module name. firstLine=%s", + noticeContents.getNoticeModuleName())); + } + + // collect all declared dependencies from NOTICE file + Set declaredDependencies = new HashSet<>(); + for (Dependency declaredDependency : noticeContents.getDeclaredDependencies()) { + if (!declaredDependencies.add(declaredDependency)) { + addProblem( + problemsBySeverity, + Severity.CRITICAL, + String.format("Dependency %s is declared twice.", declaredDependency)); + } + } + + // find all dependencies missing from NOTICE file + Collection expectedDependencies = + modulesWithShadedDependencies.getOrDefault(moduleName, Collections.emptySet()) + .stream() + .filter( + dependency -> + !dependency.getGroupId().equals("org.apache.flink")) + .collect(Collectors.toList()); + + for (Dependency expectedDependency : expectedDependencies) { + if (!declaredDependencies.contains(expectedDependency)) { + addProblem( + problemsBySeverity, + Severity.CRITICAL, + String.format("Dependency %s is not listed.", expectedDependency)); + } + } + + boolean moduleDefinesExcessDependencies = + MODULES_DEFINING_EXCESS_DEPENDENCIES.contains(moduleName); + + // find all dependencies defined in NOTICE file, which were not expected + for (Dependency declaredDependency : declaredDependencies) { + if (!expectedDependencies.contains(declaredDependency)) { + final Severity severity = + moduleDefinesExcessDependencies + ? Severity.SUPPRESSED + : Severity.TOLERATED; + addProblem( + problemsBySeverity, + severity, + String.format( + "Dependency %s is not bundled, but listed.", + declaredDependency)); + } + } + } + + return problemsBySeverity; + } + + private static void addProblem( + Map> problemsBySeverity, Severity severity, String problem) { + problemsBySeverity.computeIfAbsent(severity, ignored -> new ArrayList<>()).add(problem); + } + + @VisibleForTesting + enum Severity { + /** Issues that a legally problematic which must be fixed. */ + CRITICAL, + /** Issues that affect the correctness but aren't legally problematic. */ + TOLERATED, + /** Issues where we intentionally break the rules. */ + SUPPRESSED + } + + private static String convertProblemsToIndentedString(List problems, String header) { + return problems.isEmpty() + ? "" + : problems.stream() + .map(s -> "\t\t\t" + s) + .collect(Collectors.joining("\n", "\t\t " + header + " \n", "\n")); + } + + private static List findNoticeFiles(Path root) throws IOException { + return Files.walk(root) + .filter( + file -> { + int nameCount = file.getNameCount(); + return nameCount >= 3 + && file.getName(nameCount - 3).toString().equals("resources") + && file.getName(nameCount - 2).toString().equals("META-INF") + && file.getName(nameCount - 1).toString().equals("NOTICE"); + }) + .collect(Collectors.toList()); + } + + private static List loadFromResources(String fileName) { + try { + try (BufferedReader bufferedReader = + new BufferedReader( + new InputStreamReader( + Objects.requireNonNull( + NoticeFileChecker.class.getResourceAsStream( + "/" + fileName))))) { + + List result = + bufferedReader + .lines() + .filter(line -> !line.startsWith("#") && !line.isEmpty()) + .collect(Collectors.toList()); + LOG.debug("Loaded {} items from resource {}", result.size(), fileName); + return result; + } + } catch (Throwable e) { + // wrap anything in a RuntimeException to be callable from the static initializer + throw new RuntimeException("Error while loading resource", e); + } + } +} diff --git a/tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/optional/ShadeOptionalChecker.java b/tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/optional/ShadeOptionalChecker.java new file mode 100644 index 00000000000..906d32bbdbb --- /dev/null +++ b/tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/optional/ShadeOptionalChecker.java @@ -0,0 +1,266 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.tools.ci.optional; + +import org.apache.flink.annotation.VisibleForTesting; +import org.apache.flink.cdc.tools.ci.utils.dependency.DependencyParser; +import org.apache.flink.cdc.tools.ci.utils.shade.ShadeParser; +import org.apache.flink.cdc.tools.ci.utils.shared.Dependency; +import org.apache.flink.cdc.tools.ci.utils.shared.DependencyTree; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Verifies that all dependencies bundled with the shade-plugin are marked as optional in the pom. + * This ensures compatibility with later maven versions and in general simplifies dependency + * management as transitivity is no longer dependent on the shade-plugin. + * + *

In Maven 3.3 the dependency tree was made immutable at runtime, and thus can no longer be + * changed by the shade plugin. The plugin would usually remove a dependency from the tree when it + * is being bundled (known as dependency reduction). While dependency reduction still works for the + * published poms (== what users consume) since it can still change the content of the final pom, + * while developing Flink it no longer works. This breaks plenty of things, since suddenly a bunch + * of dependencies are still visible to downstream modules that weren't before. + * + *

To workaround this we mark all dependencies that we bundle as optional; this makes them + * non-transitive. To a downstream module, behavior-wise a non-transitive dependency is identical to + * a removed dependency. + * + *

This checker analyzes the bundled dependencies (based on the shade-plugin output) and the set + * of dependencies (based on the dependency plugin) to detect cases where a dependency is not marked + * as optional as it should. + * + *

The enforced rule is rather simple: Any dependency that is bundled, or any of its parents, + * must show up as optional in the dependency tree. The parent clause is required to cover cases + * where a module has 2 paths to a bundled dependency. If a module depends on A1/A2, each depending + * on B, with A1 and B being bundled, then even if A1 is marked as optional B is still shown as a + * non-optional dependency (because the non-optional A2 still needs it!). + */ +public class ShadeOptionalChecker { + private static final Logger LOG = LoggerFactory.getLogger(ShadeOptionalChecker.class); + + public static void main(String[] args) throws IOException { + if (args.length < 2) { + System.out.println( + "Usage: ShadeOptionalChecker "); + System.exit(1); + } + + final Path shadeOutputPath = Paths.get(args[0]); + final Path dependencyOutputPath = Paths.get(args[1]); + + final Map> bundledDependenciesByModule = + ShadeParser.parseShadeOutput(shadeOutputPath); + final Map dependenciesByModule = + DependencyParser.parseDependencyTreeOutput(dependencyOutputPath); + + final Map> violations = + checkOptionalFlags(bundledDependenciesByModule, dependenciesByModule); + + if (!violations.isEmpty()) { + LOG.error( + "{} modules bundle in total {} dependencies without them being marked as optional in the pom.", + violations.keySet().size(), + violations.values().stream().mapToInt(Collection::size).sum()); + LOG.error( + "\tIn order for shading to properly work within Flink we require all bundled dependencies to be marked as optional in the pom."); + LOG.error( + "\tFor verification purposes we require the dependency tree from the dependency-plugin to show the dependency as either:"); + LOG.error("\t\ta) an optional dependency,"); + LOG.error("\t\tb) a transitive dependency of another optional dependency."); + LOG.error( + "\tIn most cases adding '${flink.markBundledAsOptional}' to the bundled dependency is sufficient."); + LOG.error( + "\tThere are some edge cases where a transitive dependency might be associated with the \"wrong\" dependency in the tree, for example if a test dependency also requires it."); + LOG.error( + "\tIn such cases you need to adjust the poms so that the dependency shows up in the right spot. This may require adding an explicit dependency (Management) entry, excluding dependencies, or at times even reordering dependencies in the pom."); + LOG.error( + "\tSee the Dependencies page in the wiki for details: https://cwiki.apache.org/confluence/display/FLINK/Dependencies"); + + for (String moduleWithViolations : violations.keySet()) { + final Collection dependencyViolations = + violations.get(moduleWithViolations); + LOG.error( + "\tModule {} ({} violation{}):", + moduleWithViolations, + dependencyViolations.size(), + dependencyViolations.size() == 1 ? "" : "s"); + for (Dependency dependencyViolation : dependencyViolations) { + LOG.error("\t\t{}", dependencyViolation); + } + } + + System.exit(1); + } + } + + private static Map> checkOptionalFlags( + Map> bundledDependenciesByModule, + Map dependenciesByModule) { + + final Map> allViolations = new HashMap<>(); + + for (String module : bundledDependenciesByModule.keySet()) { + LOG.debug("Checking module '{}'.", module); + if (!dependenciesByModule.containsKey(module)) { + throw new IllegalStateException( + String.format( + "Module %s listed by shade-plugin, but not dependency-plugin.", + module)); + } + + final Collection bundledDependencies = + bundledDependenciesByModule.get(module); + final DependencyTree dependencyTree = dependenciesByModule.get(module); + + final Set violations = + checkOptionalFlags(module, bundledDependencies, dependencyTree); + + if (violations.isEmpty()) { + LOG.info("OK: {}", module); + } else { + allViolations.put(module, violations); + } + } + + return allViolations; + } + + @VisibleForTesting + static Set checkOptionalFlags( + String module, + Collection bundledDependencies, + DependencyTree dependencyTree) { + + bundledDependencies = + bundledDependencies.stream() + // force-shading isn't relevant for this check but breaks some shortcuts + .filter( + dependency -> + !dependency + .getArtifactId() + .equals("flink-shaded-force-shading")) + .collect(Collectors.toSet()); + + final Set violations = new HashSet<>(); + + if (bundledDependencies.isEmpty()) { + LOG.debug("\tModule is not bundling any dependencies."); + return violations; + } + + // The set of dependencies that the module directly depends on and which downstream modules + // would pull in transitively. + // + // If this set is empty we do not need to check anything. + // This allows us to avoid some edge-cases: + // + // Assume module M has the following (full) dependency tree, bundling dependency 1 and 2: + // + // +- dependency1 (compile/optional)", + // | \- dependency2 (compile) (implicitly optional because dependency1 is optional) + // \- dependency3 (test) + // \- dependency2 (compile) + // + // However, in the dependency plugin output a dependency can only show up once, so Maven may + // return this: + // + // +- dependency1 (compile/optional)", + // \- dependency3 (test) + // \- dependency2 (compile) + // + // Given this tree, and knowing that dependency2 is bundled, we would draw the conclusion + // that dependency2 is missing the optional flag. + // + // However, because dependency 1/3 are optional/test dependencies they are not transitive. + // Without any direct transitive dependency nothing can leak through to downstream modules, + // removing the need to check dependency 2 at all (and in turn, saving us from having to + // resolve this problem). + final List directTransitiveDependencies = + dependencyTree.getDirectDependencies().stream() + .filter( + dependency -> + !(isOptional(dependency) + || hasProvidedScope(dependency) + || hasTestScope(dependency) + || isCommonCompileDependency(dependency))) + .collect(Collectors.toList()); + + // if nothing would be exposed to downstream modules we exit early to reduce noise on CI + if (directTransitiveDependencies.isEmpty()) { + LOG.debug( + "Skipping deep-check of module {} because all direct dependencies are not transitive.", + module); + return violations; + } + LOG.debug( + "Running deep-check of module {} because there are direct dependencies that are transitive: {}", + module, + directTransitiveDependencies); + + for (Dependency bundledDependency : bundledDependencies) { + LOG.debug("\tChecking dependency '{}'.", bundledDependency); + + final List dependencyPath = dependencyTree.getPathTo(bundledDependency); + + final boolean isOptional = + dependencyPath.stream().anyMatch(parent -> parent.isOptional().orElse(false)); + + if (!isOptional) { + violations.add(bundledDependency); + } + } + + return violations; + } + + private static boolean isOptional(Dependency dependency) { + return dependency.isOptional().orElse(false); + } + + private static boolean hasProvidedScope(Dependency dependency) { + return "provided".equals(dependency.getScope().orElse(null)); + } + + private static boolean hasTestScope(Dependency dependency) { + return "test".equals(dependency.getScope().orElse(null)); + } + + /** + * These are compile dependencies that are set up in the root pom. We do not require modules to + * mark these as optional because all modules depend on them anyway; whether they leak through + * or not is therefore irrelevant. + */ + private static boolean isCommonCompileDependency(Dependency dependency) { + return "flink-shaded-force-shading".equals(dependency.getArtifactId()) + || "jsr305".equals(dependency.getArtifactId()) + || "slf4j-api".equals(dependency.getArtifactId()); + } +} diff --git a/tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/suffixcheck/ScalaSuffixChecker.java b/tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/suffixcheck/ScalaSuffixChecker.java new file mode 100644 index 00000000000..378ebf850a5 --- /dev/null +++ b/tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/suffixcheck/ScalaSuffixChecker.java @@ -0,0 +1,269 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.tools.ci.suffixcheck; + +import org.apache.flink.cdc.tools.ci.utils.dependency.DependencyParser; +import org.apache.flink.cdc.tools.ci.utils.shared.Dependency; +import org.apache.flink.cdc.tools.ci.utils.shared.DependencyTree; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** Utility for checking the presence/absence of scala-suffixes. */ +public class ScalaSuffixChecker { + private static final Logger LOG = LoggerFactory.getLogger(ScalaSuffixChecker.class); + + // [INFO] --- maven-dependency-plugin:3.1.1:tree (default-cli) @ flink-annotations --- + private static final Pattern moduleNamePattern = + Pattern.compile(".* --- maven-dependency-plugin.* @ (.*) ---.*"); + + // [INFO] +- junit:junit:jar:4.13.2:test + // [INFO] | \- org.hamcrest:hamcrest-core:jar:1.3:test + // [INFO] \- org.apache.logging.log4j:log4j-1.2-api:jar:2.14.1:test + private static final Pattern blockPattern = Pattern.compile(".* [+|\\\\].*"); + + // [INFO] +- org.scala-lang:scala-reflect:jar:2.11.12:test + private static final Pattern scalaSuffixPattern = Pattern.compile("_2.1[0-9]"); + + private static final Set EXCLUDED_MODULES = + new HashSet<>( + Arrays.asList( + // we ignore flink-rpc-akka because it is loaded through a separate + // class loader + "flink-rpc-akka", + // we ignore flink-table-planner-loader because it loads the planner + // through a different classpath + "flink-table-planner-loader")); + + public static void main(String[] args) throws IOException { + if (args.length < 2) { + System.out.println("Usage: ScalaSuffixChecker "); + System.exit(1); + } + + final Path mavenOutputPath = Paths.get(args[0]); + final Path flinkRootPath = Paths.get(args[1]); + + final ParseResult parseResult = parseMavenOutput(mavenOutputPath); + if (parseResult.getCleanModules().isEmpty()) { + LOG.error("Parsing found 0 scala-free modules; the parsing is likely broken."); + System.exit(1); + } + if (parseResult.getInfectedModules().isEmpty()) { + LOG.error("Parsing found 0 scala-dependent modules; the parsing is likely broken."); + System.exit(1); + } + + final Collection violations = checkScalaSuffixes(parseResult, flinkRootPath); + + if (!violations.isEmpty()) { + LOG.error( + "Violations found:{}", + violations.stream().collect(Collectors.joining("\n\t", "\n\t", ""))); + System.exit(1); + } + } + + private static ParseResult parseMavenOutput(final Path path) throws IOException { + final Set cleanModules = new HashSet<>(); + final Set infectedModules = new HashSet<>(); + + final Map dependenciesByModule = + DependencyParser.parseDependencyTreeOutput(path); + + for (String module : dependenciesByModule.keySet()) { + final String moduleName = stripScalaSuffix(module); + if (isExcluded(moduleName)) { + continue; + } + LOG.trace("Processing module '{}'.", moduleName); + + final List dependencies = + dependenciesByModule.get(module).flatten().collect(Collectors.toList()); + + boolean infected = false; + for (Dependency dependency : dependencies) { + final boolean dependsOnScala = dependsOnScala(dependency); + final boolean isTestDependency = dependency.getScope().get().equals("test"); + // we ignored flink-rpc-akka because it is loaded through a separate class loader + final boolean isExcluded = isExcluded(dependency.getArtifactId()); + LOG.trace("\tdependency:{}", dependency); + LOG.trace("\t\tdepends-on-scala:{}", dependsOnScala); + LOG.trace("\t\tis-test-dependency:{}", isTestDependency); + LOG.trace("\t\tis-excluded:{}", isExcluded); + if (dependsOnScala && !isTestDependency && !isExcluded) { + LOG.trace("\t\tOutbreak detected at {}!", moduleName); + infected = true; + } + } + + if (infected) { + infectedModules.add(moduleName); + } else { + cleanModules.add(moduleName); + } + } + + return new ParseResult(cleanModules, infectedModules); + } + + private static String stripScalaSuffix(final String moduleName) { + final int i = moduleName.indexOf("_2."); + return i > 0 ? moduleName.substring(0, i) : moduleName; + } + + private static boolean dependsOnScala(final Dependency dependency) { + return dependency.getGroupId().contains("org.scala-lang") + || scalaSuffixPattern.matcher(dependency.getArtifactId()).find(); + } + + private static Collection checkScalaSuffixes( + final ParseResult parseResult, Path flinkRootPath) throws IOException { + final Collection violations = new ArrayList<>(); + + // exclude e2e modules and flink-docs for convenience as they + // a) are not deployed during a release + // b) exist only for dev purposes + // c) no-one should depend on them + final Collection excludedModules = new ArrayList<>(); + excludedModules.add("flink-docs"); + excludedModules.addAll(getEndToEndTestModules(flinkRootPath)); + + for (String excludedModule : excludedModules) { + parseResult.getCleanModules().remove(excludedModule); + parseResult.getInfectedModules().remove(excludedModule); + } + + violations.addAll(checkCleanModules(parseResult.getCleanModules(), flinkRootPath)); + violations.addAll(checkInfectedModules(parseResult.getInfectedModules(), flinkRootPath)); + + return violations; + } + + private static Collection getEndToEndTestModules(Path flinkRootPath) + throws IOException { + try (Stream pathStream = + Files.walk(flinkRootPath.resolve("flink-end-to-end-tests"), 5)) { + return pathStream + .filter(path -> path.getFileName().toString().equals("pom.xml")) + .map(path -> path.getParent().getFileName().toString()) + .collect(Collectors.toList()); + } + } + + private static Collection checkCleanModules( + Collection modules, Path flinkRootPath) throws IOException { + return checkModules( + modules, + flinkRootPath, + "_${scala.binary.version}", + "Scala-free module '%s' is referenced with scala suffix in '%s'."); + } + + private static Collection checkInfectedModules( + Collection modules, Path flinkRootPath) throws IOException { + return checkModules( + modules, + flinkRootPath, + "", + "Scala-dependent module '%s' is referenced without scala suffix in '%s'."); + } + + private static Collection checkModules( + Collection modules, + Path flinkRootPath, + String moduleSuffix, + String violationTemplate) + throws IOException { + + final ArrayList sortedModules = new ArrayList<>(modules); + sortedModules.sort(String::compareTo); + + final Collection violations = new ArrayList<>(); + for (String module : sortedModules) { + int numPreviousViolations = violations.size(); + try (Stream pathStream = Files.walk(flinkRootPath, 3)) { + final List pomFiles = + pathStream + .filter(path -> path.getFileName().toString().equals("pom.xml")) + .collect(Collectors.toList()); + + for (Path pomFile : pomFiles) { + try (Stream lines = Files.lines(pomFile, StandardCharsets.UTF_8)) { + final boolean existsCleanReference = + lines.anyMatch( + line -> + line.contains( + module + moduleSuffix + "")); + + if (existsCleanReference) { + violations.add( + String.format( + violationTemplate, + module, + flinkRootPath.relativize(pomFile))); + } + } + } + } + if (numPreviousViolations == violations.size()) { + LOG.info("OK {}", module); + } + } + return violations; + } + + private static boolean isExcluded(String line) { + return EXCLUDED_MODULES.stream().anyMatch(line::contains); + } + + private static class ParseResult { + + private final Set cleanModules; + private final Set infectedModules; + + private ParseResult(Set cleanModules, Set infectedModules) { + this.cleanModules = cleanModules; + this.infectedModules = infectedModules; + } + + public Set getCleanModules() { + return cleanModules; + } + + public Set getInfectedModules() { + return infectedModules; + } + } +} diff --git a/tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/utils/dependency/DependencyParser.java b/tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/utils/dependency/DependencyParser.java new file mode 100644 index 00000000000..4b2c8104272 --- /dev/null +++ b/tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/utils/dependency/DependencyParser.java @@ -0,0 +1,228 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.tools.ci.utils.dependency; + +import org.apache.flink.annotation.VisibleForTesting; +import org.apache.flink.cdc.tools.ci.utils.shared.Dependency; +import org.apache.flink.cdc.tools.ci.utils.shared.DependencyTree; +import org.apache.flink.cdc.tools.ci.utils.shared.ParserUtils; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.Stack; +import java.util.function.Function; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +/** Parsing utils for the Maven dependency plugin. */ +public class DependencyParser { + + private static final Pattern DEPENDENCY_COPY_NEXT_MODULE_PATTERN = + Pattern.compile( + ".*maven-dependency-plugin:[^:]+:copy .* @ (?[^ _]+)(?:_[0-9.]+)? --.*"); + + private static final Pattern DEPENDENCY_TREE_NEXT_MODULE_PATTERN = + Pattern.compile( + ".*maven-dependency-plugin:[^:]+:tree .* @ (?[^ _]+)(?:_[0-9.]+)? --.*"); + + /** See {@link DependencyParserTreeTest} for examples. */ + private static final Pattern DEPENDENCY_TREE_ITEM_PATTERN = + Pattern.compile( + ".* +" + + "(?.*?):" + + "(?.*?):" + + "(?.*?):" + + "(?:(?.*?):)?" + + "(?.*?):" + + "(?[^ ]*)" + + "(? \\(optional\\))?"); + + /** See {@link DependencyParserCopyTest} for examples. */ + private static final Pattern DEPENDENCY_COPY_ITEM_PATTERN = + Pattern.compile( + ".* Configured Artifact: +" + + "(?.*?):" + + "(?.*?):" + + "(?:(?.*?):)?" + + "(?.*?):" + + "(?:\\?:)?" // unknown cause; e.g.: javax.xml.bind:jaxb-api:?:jar + + "(?.*)"); + + /** + * Parses the output of a Maven build where {@code dependency:copy} was used, and returns a set + * of copied dependencies for each module. + * + *

The returned dependencies will NEVER contain the scope or optional flag. + */ + public static Map> parseDependencyCopyOutput(Path buildOutput) + throws IOException { + return processLines(buildOutput, DependencyParser::parseDependencyCopyOutput); + } + + /** + * Parses the output of a Maven build where {@code dependency:tree} was used, and returns a set + * of dependencies for each module. + */ + public static Map parseDependencyTreeOutput(Path buildOutput) + throws IOException { + return processLines(buildOutput, DependencyParser::parseDependencyTreeOutput); + } + + private static X processLines(Path buildOutput, Function, X> processor) + throws IOException { + try (Stream lines = Files.lines(buildOutput)) { + return processor.apply(lines.filter(line -> line.contains("[INFO]"))); + } + } + + @VisibleForTesting + static Map> parseDependencyCopyOutput(Stream lines) { + return ParserUtils.parsePluginOutput( + lines, + DEPENDENCY_COPY_NEXT_MODULE_PATTERN, + DependencyParser::parseCopyDependencyBlock); + } + + @VisibleForTesting + static Map parseDependencyTreeOutput(Stream lines) { + return ParserUtils.parsePluginOutput( + lines, + DEPENDENCY_TREE_NEXT_MODULE_PATTERN, + DependencyParser::parseTreeDependencyBlock); + } + + private static Set parseCopyDependencyBlock(Iterator block) { + final Set dependencies = new LinkedHashSet<>(); + + Optional parsedDependency = parseCopyDependency(block.next()); + while (parsedDependency.isPresent()) { + dependencies.add(parsedDependency.get()); + + if (block.hasNext()) { + parsedDependency = parseCopyDependency(block.next()); + } else { + parsedDependency = Optional.empty(); + } + } + + return dependencies; + } + + private static DependencyTree parseTreeDependencyBlock(Iterator block) { + // discard one line, which only contains the current module name + block.next(); + + if (!block.hasNext()) { + throw new IllegalStateException("Expected more output from the dependency-plugin."); + } + + final DependencyTree dependencies = new DependencyTree(); + + final Stack parentStack = new Stack<>(); + final Stack treeDepthStack = new Stack<>(); + String line = block.next(); + Optional parsedDependency = parseTreeDependency(line); + while (parsedDependency.isPresent()) { + int treeDepth = getDepth(line); + + while (!treeDepthStack.isEmpty() && treeDepth <= treeDepthStack.peek()) { + parentStack.pop(); + treeDepthStack.pop(); + } + + final Dependency dependency = parsedDependency.get(); + + if (parentStack.isEmpty()) { + dependencies.addDirectDependency(dependency); + } else { + dependencies.addTransitiveDependencyTo(dependency, parentStack.peek()); + } + + if (treeDepthStack.isEmpty() || treeDepth > treeDepthStack.peek()) { + treeDepthStack.push(treeDepth); + parentStack.push(dependency); + } + + if (block.hasNext()) { + line = block.next(); + parsedDependency = parseTreeDependency(line); + } else { + parsedDependency = Optional.empty(); + } + } + + return dependencies; + } + + @VisibleForTesting + static Optional parseCopyDependency(String line) { + Matcher dependencyMatcher = DEPENDENCY_COPY_ITEM_PATTERN.matcher(line); + if (!dependencyMatcher.find()) { + return Optional.empty(); + } + + return Optional.of( + Dependency.create( + dependencyMatcher.group("groupId"), + dependencyMatcher.group("artifactId"), + dependencyMatcher.group("version"), + dependencyMatcher.group("classifier"))); + } + + @VisibleForTesting + static Optional parseTreeDependency(String line) { + Matcher dependencyMatcher = DEPENDENCY_TREE_ITEM_PATTERN.matcher(line); + if (!dependencyMatcher.find()) { + return Optional.empty(); + } + + return Optional.of( + Dependency.create( + dependencyMatcher.group("groupId"), + dependencyMatcher.group("artifactId"), + dependencyMatcher.group("version"), + dependencyMatcher.group("classifier"), + dependencyMatcher.group("scope"), + dependencyMatcher.group("optional") != null)); + } + + /** + * The depths returned by this method do NOT return a continuous sequence. + * + * 

+     * +- org.apache.flink:...
+     * |  +- org.apache.flink:...
+     * |  |  \- org.apache.flink:...
+     * ...
+     *
+ */ + private static int getDepth(String line) { + final int level = line.indexOf('+'); + if (level != -1) { + return level; + } + return line.indexOf('\\'); + } +} diff --git a/tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/utils/deploy/DeployParser.java b/tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/utils/deploy/DeployParser.java new file mode 100644 index 00000000000..ddb0616cc8a --- /dev/null +++ b/tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/utils/deploy/DeployParser.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.tools.ci.utils.deploy; + +import org.apache.flink.annotation.VisibleForTesting; +import org.apache.flink.cdc.tools.ci.utils.shared.ParserUtils; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** Parsing utils for the Maven deploy plugin. */ +public class DeployParser { + + // Examples: + // + // Deployment on CI with alternative repo + // [INFO] --- maven-deploy-plugin:2.8.2:deploy (default-deploy) @ flink-parent --- + // [INFO] Using alternate deployment repository.../tmp/flink-validation-deployment + // + // Skipped deployment: + // [INFO] --- maven-deploy-plugin:2.8.2:deploy (default-deploy) @ flink-parent --- + // [INFO] Skipping artifact deployment + private static final Pattern DEPLOY_MODULE_PATTERN = + Pattern.compile( + ".maven-deploy-plugin:.*:deploy .* @ (?[^ _]+)(?:_[0-9.]+)? --.*"); + + /** + * Parses the output of a Maven build where {@code deploy:deploy} was used, and returns a set of + * deployed modules. + */ + public static Set parseDeployOutput(File buildResult) throws IOException { + try (Stream linesStream = Files.lines(buildResult.toPath())) { + return parseDeployOutput(linesStream); + } + } + + @VisibleForTesting + static Set parseDeployOutput(Stream lines) { + return ParserUtils.parsePluginOutput( + lines, DEPLOY_MODULE_PATTERN, DeployParser::parseDeployBlock) + .entrySet().stream() + .filter(Map.Entry::getValue) + .map(Map.Entry::getKey) + .collect(Collectors.toSet()); + } + + private static boolean parseDeployBlock(Iterator block) { + return block.hasNext() && !block.next().contains("Skipping artifact deployment"); + } +} diff --git a/tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/utils/notice/NoticeContents.java b/tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/utils/notice/NoticeContents.java new file mode 100644 index 00000000000..3d4c3c32283 --- /dev/null +++ b/tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/utils/notice/NoticeContents.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.tools.ci.utils.notice; + +import org.apache.flink.cdc.tools.ci.utils.shared.Dependency; + +import java.util.Collection; + +/** Represents the parsed contents of a NOTICE file. */ +public class NoticeContents { + private final String noticeModuleName; + private final Collection declaredDependencies; + + public NoticeContents(String noticeModuleName, Collection declaredDependencies) { + this.noticeModuleName = noticeModuleName; + this.declaredDependencies = declaredDependencies; + } + + public String getNoticeModuleName() { + return noticeModuleName; + } + + public Collection getDeclaredDependencies() { + return declaredDependencies; + } +} diff --git a/tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/utils/notice/NoticeParser.java b/tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/utils/notice/NoticeParser.java new file mode 100644 index 00000000000..bce9f8bd1ca --- /dev/null +++ b/tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/utils/notice/NoticeParser.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.tools.ci.utils.notice; + +import org.apache.flink.annotation.VisibleForTesting; +import org.apache.flink.cdc.tools.ci.utils.shared.Dependency; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Optional; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** Parsing utils for NOTICE files. */ +public class NoticeParser { + + // "- org.apache.htrace:htrace-core:3.1.0-incubating" + private static final Pattern NOTICE_DEPENDENCY_PATTERN = + Pattern.compile( + "- " + + "(?[^ ]*?):" + + "(?[^ ]*?):" + + "(?:(?[^ ]*?):)?" + + "(?[^ ]*?)" + + "($| )"); + // "This project bundles "net.jcip:jcip-annotations:1.0". + private static final Pattern NOTICE_BUNDLES_DEPENDENCY_PATTERN = + Pattern.compile( + ".*bundles \"" + + "(?[^ ]*?):" + + "(?[^ ]*?):" + + "(?:(?[^ ]*?):)?" + + "(?[^ ]*?)" + + "\".*"); + + public static Optional parseNoticeFile(Path noticeFile) throws IOException { + // 1st line contains module name + final List noticeContents = Files.readAllLines(noticeFile); + + return parseNoticeFile(noticeContents); + } + + @VisibleForTesting + static Optional parseNoticeFile(List noticeContents) { + if (noticeContents.isEmpty()) { + return Optional.empty(); + } + + final String noticeModuleName = noticeContents.get(0); + + Collection declaredDependencies = new ArrayList<>(); + for (String line : noticeContents) { + Optional dependency = tryParsing(NOTICE_DEPENDENCY_PATTERN, line); + if (!dependency.isPresent()) { + dependency = tryParsing(NOTICE_BUNDLES_DEPENDENCY_PATTERN, line); + } + dependency.ifPresent(declaredDependencies::add); + } + + return Optional.of(new NoticeContents(noticeModuleName, declaredDependencies)); + } + + private static Optional tryParsing(Pattern pattern, String line) { + Matcher matcher = pattern.matcher(line); + if (matcher.find()) { + String groupId = matcher.group("groupId"); + String artifactId = matcher.group("artifactId"); + String version = matcher.group("version"); + String classifier = matcher.group("classifier"); + return Optional.of(Dependency.create(groupId, artifactId, version, classifier)); + } + return Optional.empty(); + } +} diff --git a/tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/utils/shade/ShadeParser.java b/tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/utils/shade/ShadeParser.java new file mode 100644 index 00000000000..c6fe49cf950 --- /dev/null +++ b/tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/utils/shade/ShadeParser.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.tools.ci.utils.shade; + +import org.apache.flink.annotation.VisibleForTesting; +import org.apache.flink.cdc.tools.ci.utils.shared.Dependency; +import org.apache.flink.cdc.tools.ci.utils.shared.ParserUtils; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +/** Utils for parsing the shade-plugin output. */ +public final class ShadeParser { + + private static final Pattern SHADE_NEXT_MODULE_PATTERN = + Pattern.compile( + ".*:shade \\((?:shade-flink|shade-dist|default)\\) @ (?[^ _]+)(?:_[0-9.]+)? --.*"); + + private static final Pattern SHADE_INCLUDE_MODULE_PATTERN = + Pattern.compile( + ".* " + + "(?.*?):" + + "(?.*?):" + + "(?.*?):" + + "(?:(?.*?):)?" + + "(?.*?)" + + " in the shaded jar"); + + /** + * Parses the output of a Maven build where {@code shade:shade} was used, and returns a set of + * bundled dependencies for each module. + * + *

The returned dependencies will NEVER contain the scope or optional flag. + * + *

This method only considers the {@code shade-flink} and {@code shade-dist} executions, + * because all artifacts we produce that are either published or referenced are created by these + * executions. In other words, all artifacts from other executions are only used internally by + * the module that created them. + */ + public static Map> parseShadeOutput(Path buildOutput) + throws IOException { + try (Stream lines = Files.lines(buildOutput)) { + return parseShadeOutput(lines); + } + } + + @VisibleForTesting + static Map> parseShadeOutput(Stream lines) { + return ParserUtils.parsePluginOutput( + lines.filter(line -> !line.contains(" Excluding ")), + SHADE_NEXT_MODULE_PATTERN, + ShadeParser::parseBlock); + } + + private static Set parseBlock(Iterator block) { + final Set dependencies = new LinkedHashSet<>(); + + Optional parsedDependency = parseDependency(block.next()); + while (parsedDependency.isPresent()) { + dependencies.add(parsedDependency.get()); + + if (block.hasNext()) { + parsedDependency = parseDependency(block.next()); + } else { + parsedDependency = Optional.empty(); + } + } + + return dependencies; + } + + @VisibleForTesting + static Optional parseDependency(String line) { + Matcher dependencyMatcher = SHADE_INCLUDE_MODULE_PATTERN.matcher(line); + if (!dependencyMatcher.find()) { + return Optional.empty(); + } + + return Optional.of( + Dependency.create( + dependencyMatcher.group("groupId"), + dependencyMatcher.group("artifactId"), + dependencyMatcher.group("version"), + dependencyMatcher.group("classifier"))); + } + + private ShadeParser() {} +} diff --git a/tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/utils/shared/Dependency.java b/tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/utils/shared/Dependency.java new file mode 100644 index 00000000000..84f297ca0c3 --- /dev/null +++ b/tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/utils/shared/Dependency.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.tools.ci.utils.shared; + +import javax.annotation.Nullable; + +import java.util.Objects; +import java.util.Optional; + +/** + * Represents a dependency. + * + *

For some properties we return an {@link Optional}, for those that, depending on the plugin + * goal, may not be determinable. For example, {@code dependency:copy} never prints the scope or + * optional flag. + */ +public final class Dependency { + + private final String groupId; + private final String artifactId; + private final String version; + @Nullable private final String classifier; + @Nullable private final String scope; + @Nullable private final Boolean isOptional; + + private Dependency( + String groupId, + String artifactId, + String version, + @Nullable String classifier, + @Nullable String scope, + @Nullable Boolean isOptional) { + this.groupId = Objects.requireNonNull(groupId); + this.artifactId = Objects.requireNonNull(artifactId); + this.version = Objects.requireNonNull(version); + this.classifier = classifier; + this.scope = scope; + this.isOptional = isOptional; + } + + public static Dependency create( + String groupId, + String artifactId, + String version, + String classifier, + String scope, + boolean isOptional) { + return new Dependency( + groupId, + artifactId, + version, + classifier, + Objects.requireNonNull(scope), + isOptional); + } + + public static Dependency create( + String groupId, String artifactId, String version, String classifier) { + return new Dependency(groupId, artifactId, version, classifier, null, null); + } + + public String getGroupId() { + return groupId; + } + + public String getArtifactId() { + return artifactId; + } + + public String getVersion() { + return version; + } + + public Optional getClassifier() { + return Optional.ofNullable(classifier); + } + + public Optional getScope() { + return Optional.ofNullable(scope); + } + + public Optional isOptional() { + return Optional.ofNullable(isOptional); + } + + @Override + public String toString() { + return groupId + + ":" + + artifactId + + ":" + + version + + (classifier != null ? ":" + classifier : "") + + (scope != null ? ":" + scope : "") + + (isOptional != null && isOptional ? " (optional)" : ""); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Dependency that = (Dependency) o; + return Objects.equals(groupId, that.groupId) + && Objects.equals(artifactId, that.artifactId) + && Objects.equals(version, that.version) + && Objects.equals(classifier, that.classifier) + && Objects.equals(scope, that.scope) + && Objects.equals(isOptional, that.isOptional); + } + + @Override + public int hashCode() { + return Objects.hash(groupId, artifactId, version, classifier, scope, isOptional); + } +} diff --git a/tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/utils/shared/DependencyTree.java b/tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/utils/shared/DependencyTree.java new file mode 100644 index 00000000000..78cd72243ef --- /dev/null +++ b/tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/utils/shared/DependencyTree.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.tools.ci.utils.shared; + +import org.apache.flink.annotation.VisibleForTesting; + +import org.apache.flink.shaded.guava31.com.google.common.graph.Traverser; + +import javax.annotation.Nullable; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +/** + * Represents a dependency tree. + * + *

Every dependency can only occur exactly once. + */ +public class DependencyTree { + + private final Map lookup = new LinkedHashMap<>(); + private final List directDependencies = new ArrayList<>(); + + public DependencyTree addDirectDependency(Dependency dependency) { + final String key = getKey(dependency); + if (lookup.containsKey(key)) { + return this; + } + final Node node = new Node(dependency, null); + + lookup.put(key, node); + directDependencies.add(node); + + return this; + } + + public DependencyTree addTransitiveDependencyTo( + Dependency transitiveDependency, Dependency parent) { + final String key = getKey(transitiveDependency); + if (lookup.containsKey(key)) { + return this; + } + final Node node = lookup.get(getKey(parent)).addTransitiveDependency(transitiveDependency); + + lookup.put(key, node); + + return this; + } + + private static final class Node { + private final Dependency dependency; + @Nullable private final Node parent; + private final List children = new ArrayList<>(); + + private Node(Dependency dependency, @Nullable Node parent) { + this.dependency = dependency; + this.parent = parent; + } + + public Node addTransitiveDependency(Dependency dependency) { + final Node node = new Node(dependency, this); + this.children.add(node); + return node; + } + + private boolean isRoot() { + return parent == null; + } + } + + public List getDirectDependencies() { + return directDependencies.stream() + .map(node -> node.dependency) + .collect(Collectors.toList()); + } + + public List getPathTo(Dependency dependency) { + final LinkedList path = new LinkedList<>(); + + Node node = lookup.get(getKey(dependency)); + path.addFirst(node.dependency); + while (!node.isRoot()) { + node = node.parent; + path.addFirst(node.dependency); + } + + return path; + } + + public Stream flatten() { + return StreamSupport.stream( + Traverser.forTree(node -> node.children) + .depthFirstPreOrder(directDependencies) + .spliterator(), + false) + .map(node -> node.dependency); + } + + /** + * We don't use the {@link Dependency} as a key because we don't want lookups to be dependent on + * scope or the optional flag. + * + * @param dependency + * @return + */ + @VisibleForTesting + static String getKey(Dependency dependency) { + return dependency.getGroupId() + + ":" + + dependency.getArtifactId() + + ":" + + dependency.getVersion() + + ":" + + dependency.getClassifier().orElse("(no-classifier)"); + } +} diff --git a/tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/utils/shared/ParserUtils.java b/tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/utils/shared/ParserUtils.java new file mode 100644 index 00000000000..ab1ddf2fa3d --- /dev/null +++ b/tools/ci/flink-cdc-ci-tools/src/main/java/org/apache/flink/cdc/tools/ci/utils/shared/ParserUtils.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.tools.ci.utils.shared; + +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.function.Function; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +/** Parsing utils. */ +public class ParserUtils { + + /** + * Iterates over the given lines, identifying plugin execution blocks with the given pattern and + * parses the plugin output with the given parser. + * + *

This method assumes that the given pattern matches at most once for each module. + * + *

The given pattern must include a {@code module} group that captures the module that the + * plugin runs on (without the scala suffix!). + * + * @param lines maven output lines + * @param executionLinePattern pattern that matches plugin executions + * @param blockParser parser for the plugin block + * @return map containing the parser result for each module + * @param block parser output + */ + public static Map parsePluginOutput( + Stream lines, + Pattern executionLinePattern, + Function, D> blockParser) { + final Map result = new LinkedHashMap<>(); + + final Iterator iterator = lines.iterator(); + + while (iterator.hasNext()) { + Matcher moduleMatcher = executionLinePattern.matcher(iterator.next()); + while (!moduleMatcher.find()) { + if (iterator.hasNext()) { + moduleMatcher = executionLinePattern.matcher(iterator.next()); + } else { + return result; + } + } + final String currentModule = moduleMatcher.group("module"); + + if (!iterator.hasNext()) { + throw new IllegalStateException("Expected more output from the plugin."); + } + + result.put(currentModule, blockParser.apply(iterator)); + } + return result; + } +} diff --git a/tools/ci/flink-cdc-ci-tools/src/main/resources/log4j2.properties b/tools/ci/flink-cdc-ci-tools/src/main/resources/log4j2.properties new file mode 100644 index 00000000000..b0b222b577f --- /dev/null +++ b/tools/ci/flink-cdc-ci-tools/src/main/resources/log4j2.properties @@ -0,0 +1,25 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +rootLogger.level = DEBUG +rootLogger.appenderRef.console.ref = ConsoleAppender + +appender.console.name = ConsoleAppender +appender.console.type = CONSOLE +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n diff --git a/tools/ci/flink-cdc-ci-tools/src/main/resources/modules-defining-excess-dependencies.modulelist b/tools/ci/flink-cdc-ci-tools/src/main/resources/modules-defining-excess-dependencies.modulelist new file mode 100644 index 00000000000..c496e6db583 --- /dev/null +++ b/tools/ci/flink-cdc-ci-tools/src/main/resources/modules-defining-excess-dependencies.modulelist @@ -0,0 +1,20 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# This file lists modules which define additional dependencies, not shown by the maven shade plugin output. +# These are usually undeclared shaded (or otherwise included) dependencies from transitive dependencies. diff --git a/tools/ci/flink-cdc-ci-tools/src/test/java/org/apache/flink/tools/ci/licensecheck/JarFileCheckerTest.java b/tools/ci/flink-cdc-ci-tools/src/test/java/org/apache/flink/tools/ci/licensecheck/JarFileCheckerTest.java new file mode 100644 index 00000000000..141b8932300 --- /dev/null +++ b/tools/ci/flink-cdc-ci-tools/src/test/java/org/apache/flink/tools/ci/licensecheck/JarFileCheckerTest.java @@ -0,0 +1,525 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.tools.ci.licensecheck; + +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.net.URI; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Tests for the {@link JarFileChecker}. + * + *

For dev purposes, generate a deploy-directory with: mvn clean deploy + * -DaltDeploymentRepository=snapshot-repo::default::file:/tmp/flink-deployment -DskipTests + * -Drat.skip and add a test checking that directory. + */ +class JarFileCheckerTest { + + private static final List VALID_NOTICE_PATH = Arrays.asList("META-INF", "NOTICE"); + private static final List VALID_LICENSE_PATH = Arrays.asList("META-INF", "LICENSE"); + + @Test + void testValidJarAccepted(@TempDir Path tempDir) throws Exception { + assertThat( + JarFileChecker.checkJar( + createJar( + tempDir, + Entry.fileEntry(VALID_NOTICE_CONTENTS, VALID_NOTICE_PATH), + Entry.fileEntry( + VALID_LICENSE_CONTENTS, VALID_LICENSE_PATH)))) + .isEqualTo(0); + } + + @Test + void testRejectedOnMissingNoticeFile(@TempDir Path tempDir) throws Exception { + assertThat( + JarFileChecker.checkJar( + createJar( + tempDir, + Entry.fileEntry( + VALID_LICENSE_CONTENTS, VALID_LICENSE_PATH)))) + .isEqualTo(1); + } + + @Test + void testRejectedOnInvalidNoticeFile(@TempDir Path tempDir) throws Exception { + assertThat( + JarFileChecker.checkJar( + createJar( + tempDir, + Entry.fileEntry(INVALID_NOTICE_CONTENTS, VALID_NOTICE_PATH), + Entry.fileEntry( + VALID_LICENSE_CONTENTS, VALID_LICENSE_PATH)))) + .isEqualTo(1); + } + + @Test + void testRejectedOnNoticeFileInRoot(@TempDir Path tempDir) throws Exception { + assertThat( + JarFileChecker.checkJar( + createJar( + tempDir, + Entry.fileEntry(VALID_NOTICE_CONTENTS, VALID_NOTICE_PATH), + Entry.fileEntry(VALID_LICENSE_CONTENTS, VALID_LICENSE_PATH), + Entry.fileEntry( + VALID_NOTICE_CONTENTS, + Arrays.asList("some_custom_notice"))))) + .isEqualTo(1); + } + + @Test + void testRejectedOnMissingLicenseFile(@TempDir Path tempDir) throws Exception { + assertThat( + JarFileChecker.checkJar( + createJar( + tempDir, + Entry.fileEntry(VALID_NOTICE_CONTENTS, VALID_NOTICE_PATH)))) + .isEqualTo(1); + } + + @Test + void testRejectedOnInvalidLicenseFile(@TempDir Path tempDir) throws Exception { + assertThat( + JarFileChecker.checkJar( + createJar( + tempDir, + Entry.fileEntry(VALID_NOTICE_CONTENTS, VALID_NOTICE_PATH), + Entry.fileEntry( + INVALID_LICENSE_CONTENTS, VALID_LICENSE_PATH)))) + .isEqualTo(1); + } + + @Test + void testRejectedOnLicenseFileInRoot(@TempDir Path tempDir) throws Exception { + assertThat( + JarFileChecker.checkJar( + createJar( + tempDir, + Entry.fileEntry(VALID_NOTICE_CONTENTS, VALID_NOTICE_PATH), + Entry.fileEntry(VALID_LICENSE_CONTENTS, VALID_LICENSE_PATH), + Entry.fileEntry( + VALID_LICENSE_CONTENTS, + Arrays.asList("some_custom_license"))))) + .isEqualTo(1); + } + + @Test + void testRejectedOnLicenseFileInSomeDirectory(@TempDir Path tempDir) throws Exception { + assertThat( + JarFileChecker.checkJar( + createJar( + tempDir, + Entry.fileEntry(VALID_NOTICE_CONTENTS, VALID_NOTICE_PATH), + Entry.fileEntry(VALID_LICENSE_CONTENTS, VALID_LICENSE_PATH), + Entry.fileEntry( + VALID_LICENSE_CONTENTS, + Arrays.asList( + "some", + "directory", + "some_custom_license"))))) + .isEqualTo(1); + } + + @Disabled( + "Currently not checked, but we may want to enforce this in the future to reduce ambiguity.") + void testRejectedOnAdditionalLicenseFileInMetaInf(@TempDir Path tempDir) throws Exception { + assertThat( + JarFileChecker.checkJar( + createJar( + tempDir, + Entry.fileEntry(VALID_NOTICE_CONTENTS, VALID_NOTICE_PATH), + Entry.fileEntry(VALID_LICENSE_CONTENTS, VALID_LICENSE_PATH), + Entry.fileEntry( + VALID_LICENSE_CONTENTS, + Arrays.asList("META-INF", "LICENSE.txt"))))) + .isEqualTo(1); + } + + @Test + void testIgnoreLicenseDirectories(@TempDir Path tempDir) throws Exception { + assertThat( + JarFileChecker.checkJar( + createJar( + tempDir, + Entry.fileEntry(VALID_NOTICE_CONTENTS, VALID_NOTICE_PATH), + Entry.fileEntry(VALID_LICENSE_CONTENTS, VALID_LICENSE_PATH), + Entry.directoryEntry( + Arrays.asList("some", "license", "directory"))))) + .isEqualTo(0); + } + + @Test + void testIgnoreClassFiles(@TempDir Path tempDir) throws Exception { + assertThat( + JarFileChecker.checkJar( + createJar( + tempDir, + Entry.fileEntry(VALID_NOTICE_CONTENTS, VALID_NOTICE_PATH), + Entry.fileEntry(VALID_LICENSE_CONTENTS, VALID_LICENSE_PATH), + Entry.fileEntry( + "content", + Arrays.asList("SomeLicenseClass.class"))))) + .isEqualTo(0); + } + + @Test + void testIgnoreFtlFiles(@TempDir Path tempDir) throws Exception { + assertThat( + JarFileChecker.checkJar( + createJar( + tempDir, + Entry.fileEntry(VALID_NOTICE_CONTENTS, VALID_NOTICE_PATH), + Entry.fileEntry(VALID_LICENSE_CONTENTS, VALID_LICENSE_PATH), + Entry.fileEntry( + "content", Arrays.asList("SomeLicenseFile.ftl"))))) + .isEqualTo(0); + } + + @Test + void testIgnoreWebThirdPartyLicenses(@TempDir Path tempDir) throws Exception { + assertThat( + JarFileChecker.checkJar( + createJar( + tempDir, + Entry.fileEntry(VALID_NOTICE_CONTENTS, VALID_NOTICE_PATH), + Entry.fileEntry(VALID_LICENSE_CONTENTS, VALID_LICENSE_PATH), + Entry.fileEntry( + "class contents", + Arrays.asList("web", "3rdpartylicenses.txt"))))) + .isEqualTo(0); + } + + @Test + void testForbiddenLGPLongTextDetected(@TempDir Path tempDir) throws Exception { + assertThat( + JarFileChecker.checkJar( + createJar( + tempDir, + Entry.fileEntry(VALID_NOTICE_CONTENTS, VALID_NOTICE_PATH), + Entry.fileEntry(VALID_LICENSE_CONTENTS, VALID_LICENSE_PATH), + Entry.fileEntry( + "some GNU Lesser General public License text", + Collections.singletonList("some_file.txt"))))) + .isEqualTo(1); + } + + @Test + void testForbiddenLGPMultiLineLongTextWithCommentAndLeadingWhitespaceDetected( + @TempDir Path tempDir) throws Exception { + assertThat( + JarFileChecker.checkJar( + createJar( + tempDir, + Entry.fileEntry(VALID_NOTICE_CONTENTS, VALID_NOTICE_PATH), + Entry.fileEntry(VALID_LICENSE_CONTENTS, VALID_LICENSE_PATH), + Entry.fileEntry( + "some GNU Lesser General public \n\t\t//#License text", + Collections.singletonList("some_file.txt"))))) + .isEqualTo(1); + } + + private static class Entry { + final String contents; + final List path; + final boolean isDirectory; + + public static Entry directoryEntry(List path) { + return new Entry("", path, true); + } + + public static Entry fileEntry(String contents, List path) { + return new Entry(contents, path, false); + } + + private Entry(String contents, List path, boolean isDirectory) { + this.contents = contents; + this.path = path; + this.isDirectory = isDirectory; + } + } + + private static Path createJar(Path tempDir, Entry... entries) throws Exception { + final Path path = tempDir.resolve(UUID.randomUUID().toString() + ".jar"); + + final URI uriWithoutScheme = path.toUri(); + + final URI uri = + new URI( + "jar:file", + uriWithoutScheme.getHost(), + uriWithoutScheme.getPath(), + uriWithoutScheme.getFragment()); + + // causes FileSystems#newFileSystem to automatically create a valid zip file + // this is easier than manually creating a valid empty zip file manually + final Map env = new HashMap<>(); + env.put("create", "true"); + + try (FileSystem zip = FileSystems.newFileSystem(uri, env)) { + // shortcut to getting the single root + final Path root = zip.getPath("").toAbsolutePath(); + for (Entry entry : entries) { + final Path zipPath = + root.resolve( + zip.getPath( + entry.path.get(0), + entry.path + .subList(1, entry.path.size()) + .toArray(new String[] {}))); + if (entry.isDirectory) { + Files.createDirectories(zipPath); + } else { + Files.createDirectories(zipPath.getParent()); + Files.write(zipPath, entry.contents.getBytes()); + } + } + } + return path; + } + + private static final String INVALID_NOTICE_CONTENTS = "" + "invalid"; + + private static final String VALID_NOTICE_CONTENTS = + "" + + "Flink : SomeModule\n" + + "Copyright 2014-2020 The Apache Software Foundation\n" + + "\n" + + "This product includes software developed at\n" + + "The Apache Software Foundation (http://www.apache.org/)."; + + private static final String INVALID_LICENSE_CONTENTS = "" + "invalid"; + + private static final String VALID_LICENSE_CONTENTS = + "" + + "\n" + + " Apache License\n" + + " Version 2.0, January 2004\n" + + " http://www.apache.org/licenses/\n" + + "\n" + + " TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n" + + "\n" + + " 1. Definitions.\n" + + "\n" + + " \"License\" shall mean the terms and conditions for use, reproduction,\n" + + " and distribution as defined by Sections 1 through 9 of this document.\n" + + "\n" + + " \"Licensor\" shall mean the copyright owner or entity authorized by\n" + + " the copyright owner that is granting the License.\n" + + "\n" + + " \"Legal Entity\" shall mean the union of the acting entity and all\n" + + " other entities that control, are controlled by, or are under common\n" + + " control with that entity. For the purposes of this definition,\n" + + " \"control\" means (i) the power, direct or indirect, to cause the\n" + + " direction or management of such entity, whether by contract or\n" + + " otherwise, or (ii) ownership of fifty percent (50%) or more of the\n" + + " outstanding shares, or (iii) beneficial ownership of such entity.\n" + + "\n" + + " \"You\" (or \"Your\") shall mean an individual or Legal Entity\n" + + " exercising permissions granted by this License.\n" + + "\n" + + " \"Source\" form shall mean the preferred form for making modifications,\n" + + " including but not limited to software source code, documentation\n" + + " source, and configuration files.\n" + + "\n" + + " \"Object\" form shall mean any form resulting from mechanical\n" + + " transformation or translation of a Source form, including but\n" + + " not limited to compiled object code, generated documentation,\n" + + " and conversions to other media types.\n" + + "\n" + + " \"Work\" shall mean the work of authorship, whether in Source or\n" + + " Object form, made available under the License, as indicated by a\n" + + " copyright notice that is included in or attached to the work\n" + + " (an example is provided in the Appendix below).\n" + + "\n" + + " \"Derivative Works\" shall mean any work, whether in Source or Object\n" + + " form, that is based on (or derived from) the Work and for which the\n" + + " editorial revisions, annotations, elaborations, or other modifications\n" + + " represent, as a whole, an original work of authorship. For the purposes\n" + + " of this License, Derivative Works shall not include works that remain\n" + + " separable from, or merely link (or bind by name) to the interfaces of,\n" + + " the Work and Derivative Works thereof.\n" + + "\n" + + " \"Contribution\" shall mean any work of authorship, including\n" + + " the original version of the Work and any modifications or additions\n" + + " to that Work or Derivative Works thereof, that is intentionally\n" + + " submitted to Licensor for inclusion in the Work by the copyright owner\n" + + " or by an individual or Legal Entity authorized to submit on behalf of\n" + + " the copyright owner. For the purposes of this definition, \"submitted\"\n" + + " means any form of electronic, verbal, or written communication sent\n" + + " to the Licensor or its representatives, including but not limited to\n" + + " communication on electronic mailing lists, source code control systems,\n" + + " and issue tracking systems that are managed by, or on behalf of, the\n" + + " Licensor for the purpose of discussing and improving the Work, but\n" + + " excluding communication that is conspicuously marked or otherwise\n" + + " designated in writing by the copyright owner as \"Not a Contribution.\"\n" + + "\n" + + " \"Contributor\" shall mean Licensor and any individual or Legal Entity\n" + + " on behalf of whom a Contribution has been received by Licensor and\n" + + " subsequently incorporated within the Work.\n" + + "\n" + + " 2. Grant of Copyright License. Subject to the terms and conditions of\n" + + " this License, each Contributor hereby grants to You a perpetual,\n" + + " worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n" + + " copyright license to reproduce, prepare Derivative Works of,\n" + + " publicly display, publicly perform, sublicense, and distribute the\n" + + " Work and such Derivative Works in Source or Object form.\n" + + "\n" + + " 3. Grant of Patent License. Subject to the terms and conditions of\n" + + " this License, each Contributor hereby grants to You a perpetual,\n" + + " worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n" + + " (except as stated in this section) patent license to make, have made,\n" + + " use, offer to sell, sell, import, and otherwise transfer the Work,\n" + + " where such license applies only to those patent claims licensable\n" + + " by such Contributor that are necessarily infringed by their\n" + + " Contribution(s) alone or by combination of their Contribution(s)\n" + + " with the Work to which such Contribution(s) was submitted. If You\n" + + " institute patent litigation against any entity (including a\n" + + " cross-claim or counterclaim in a lawsuit) alleging that the Work\n" + + " or a Contribution incorporated within the Work constitutes direct\n" + + " or contributory patent infringement, then any patent licenses\n" + + " granted to You under this License for that Work shall terminate\n" + + " as of the date such litigation is filed.\n" + + "\n" + + " 4. Redistribution. You may reproduce and distribute copies of the\n" + + " Work or Derivative Works thereof in any medium, with or without\n" + + " modifications, and in Source or Object form, provided that You\n" + + " meet the following conditions:\n" + + "\n" + + " (a) You must give any other recipients of the Work or\n" + + " Derivative Works a copy of this License; and\n" + + "\n" + + " (b) You must cause any modified files to carry prominent notices\n" + + " stating that You changed the files; and\n" + + "\n" + + " (c) You must retain, in the Source form of any Derivative Works\n" + + " that You distribute, all copyright, patent, trademark, and\n" + + " attribution notices from the Source form of the Work,\n" + + " excluding those notices that do not pertain to any part of\n" + + " the Derivative Works; and\n" + + "\n" + + " (d) If the Work includes a \"NOTICE\" text file as part of its\n" + + " distribution, then any Derivative Works that You distribute must\n" + + " include a readable copy of the attribution notices contained\n" + + " within such NOTICE file, excluding those notices that do not\n" + + " pertain to any part of the Derivative Works, in at least one\n" + + " of the following places: within a NOTICE text file distributed\n" + + " as part of the Derivative Works; within the Source form or\n" + + " documentation, if provided along with the Derivative Works; or,\n" + + " within a display generated by the Derivative Works, if and\n" + + " wherever such third-party notices normally appear. The contents\n" + + " of the NOTICE file are for informational purposes only and\n" + + " do not modify the License. You may add Your own attribution\n" + + " notices within Derivative Works that You distribute, alongside\n" + + " or as an addendum to the NOTICE text from the Work, provided\n" + + " that such additional attribution notices cannot be construed\n" + + " as modifying the License.\n" + + "\n" + + " You may add Your own copyright statement to Your modifications and\n" + + " may provide additional or different license terms and conditions\n" + + " for use, reproduction, or distribution of Your modifications, or\n" + + " for any such Derivative Works as a whole, provided Your use,\n" + + " reproduction, and distribution of the Work otherwise complies with\n" + + " the conditions stated in this License.\n" + + "\n" + + " 5. Submission of Contributions. Unless You explicitly state otherwise,\n" + + " any Contribution intentionally submitted for inclusion in the Work\n" + + " by You to the Licensor shall be under the terms and conditions of\n" + + " this License, without any additional terms or conditions.\n" + + " Notwithstanding the above, nothing herein shall supersede or modify\n" + + " the terms of any separate license agreement you may have executed\n" + + " with Licensor regarding such Contributions.\n" + + "\n" + + " 6. Trademarks. This License does not grant permission to use the trade\n" + + " names, trademarks, service marks, or product names of the Licensor,\n" + + " except as required for reasonable and customary use in describing the\n" + + " origin of the Work and reproducing the content of the NOTICE file.\n" + + "\n" + + " 7. Disclaimer of Warranty. Unless required by applicable law or\n" + + " agreed to in writing, Licensor provides the Work (and each\n" + + " Contributor provides its Contributions) on an \"AS IS\" BASIS,\n" + + " WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n" + + " implied, including, without limitation, any warranties or conditions\n" + + " of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n" + + " PARTICULAR PURPOSE. You are solely responsible for determining the\n" + + " appropriateness of using or redistributing the Work and assume any\n" + + " risks associated with Your exercise of permissions under this License.\n" + + "\n" + + " 8. Limitation of Liability. In no event and under no legal theory,\n" + + " whether in tort (including negligence), contract, or otherwise,\n" + + " unless required by applicable law (such as deliberate and grossly\n" + + " negligent acts) or agreed to in writing, shall any Contributor be\n" + + " liable to You for damages, including any direct, indirect, special,\n" + + " incidental, or consequential damages of any character arising as a\n" + + " result of this License or out of the use or inability to use the\n" + + " Work (including but not limited to damages for loss of goodwill,\n" + + " work stoppage, computer failure or malfunction, or any and all\n" + + " other commercial damages or losses), even if such Contributor\n" + + " has been advised of the possibility of such damages.\n" + + "\n" + + " 9. Accepting Warranty or Additional Liability. While redistributing\n" + + " the Work or Derivative Works thereof, You may choose to offer,\n" + + " and charge a fee for, acceptance of support, warranty, indemnity,\n" + + " or other liability obligations and/or rights consistent with this\n" + + " License. However, in accepting such obligations, You may act only\n" + + " on Your own behalf and on Your sole responsibility, not on behalf\n" + + " of any other Contributor, and only if You agree to indemnify,\n" + + " defend, and hold each Contributor harmless for any liability\n" + + " incurred by, or claims asserted against, such Contributor by reason\n" + + " of your accepting any such warranty or additional liability.\n" + + "\n" + + " END OF TERMS AND CONDITIONS\n" + + "\n" + + " APPENDIX: How to apply the Apache License to your work.\n" + + "\n" + + " To apply the Apache License to your work, attach the following\n" + + " boilerplate notice, with the fields enclosed by brackets \"[]\"\n" + + " replaced with your own identifying information. (Don't include\n" + + " the brackets!) The text should be enclosed in the appropriate\n" + + " comment syntax for the file format. We also recommend that a\n" + + " file or class name and description of purpose be included on the\n" + + " same \"printed page\" as the copyright notice for easier\n" + + " identification within third-party archives.\n" + + "\n" + + " Copyright [yyyy] [name of copyright owner]\n" + + "\n" + + " Licensed under the Apache License, Version 2.0 (the \"License\");\n" + + " you may not use this file except in compliance with the License.\n" + + " You may obtain a copy of the License at\n" + + "\n" + + " http://www.apache.org/licenses/LICENSE-2.0\n" + + "\n" + + " Unless required by applicable law or agreed to in writing, software\n" + + " distributed under the License is distributed on an \"AS IS\" BASIS,\n" + + " WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n" + + " See the License for the specific language governing permissions and\n" + + " limitations under the License.\n"; +} diff --git a/tools/ci/flink-cdc-ci-tools/src/test/java/org/apache/flink/tools/ci/licensecheck/NoticeFileCheckerTest.java b/tools/ci/flink-cdc-ci-tools/src/test/java/org/apache/flink/tools/ci/licensecheck/NoticeFileCheckerTest.java new file mode 100644 index 00000000000..9ffdd371ef2 --- /dev/null +++ b/tools/ci/flink-cdc-ci-tools/src/test/java/org/apache/flink/tools/ci/licensecheck/NoticeFileCheckerTest.java @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.tools.ci.licensecheck; + +import org.apache.flink.cdc.tools.ci.utils.notice.NoticeContents; +import org.apache.flink.cdc.tools.ci.utils.shared.Dependency; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static org.assertj.core.api.Assertions.assertThat; + +class NoticeFileCheckerTest { + @Test + void testRunHappyPath() throws IOException { + final String moduleName = "test"; + final Dependency bundledDependency = Dependency.create("a", "b", "c", null); + final Map> bundleDependencies = new HashMap<>(); + bundleDependencies.put(moduleName, Collections.singleton(bundledDependency)); + final Set deployedModules = Collections.singleton(moduleName); + final Optional noticeContents = + Optional.of( + new NoticeContents( + moduleName, Collections.singletonList(bundledDependency))); + + assertThat( + NoticeFileChecker.run( + bundleDependencies, + deployedModules, + Collections.singletonMap(moduleName, noticeContents))) + .isEqualTo(0); + } + + @Test + void testRunRejectsMissingNotice() throws IOException { + final String moduleName = "test"; + final Dependency bundledDependency = Dependency.create("a", "b", "c", null); + final Map> bundleDependencies = new HashMap<>(); + bundleDependencies.put(moduleName, Collections.singleton(bundledDependency)); + final Set deployedModules = Collections.singleton(moduleName); + final Optional missingNotice = Optional.empty(); + + assertThat( + NoticeFileChecker.run( + bundleDependencies, + deployedModules, + Collections.singletonMap(moduleName, missingNotice))) + .isEqualTo(1); + } + + @Test + void testRunRejectsIncorrectNotice() throws IOException { + final String moduleName = "test"; + final Dependency bundledDependency = Dependency.create("a", "b", "c", null); + final Map> bundleDependencies = new HashMap<>(); + bundleDependencies.put(moduleName, Collections.singleton(bundledDependency)); + final Set deployedModules = Collections.singleton(moduleName); + final Optional emptyNotice = + Optional.of(new NoticeContents(moduleName, Collections.emptyList())); + + assertThat( + NoticeFileChecker.run( + bundleDependencies, + deployedModules, + Collections.singletonMap(moduleName, emptyNotice))) + .isEqualTo(1); + } + + @Test + void testRunSkipsNonDeployedModules() throws IOException { + final String moduleName = "test"; + final Dependency bundledDependency = Dependency.create("a", "b", "c", null); + final Map> bundleDependencies = new HashMap<>(); + bundleDependencies.put(moduleName, Collections.singleton(bundledDependency)); + final Set deployedModules = Collections.emptySet(); + // this would usually be a problem, but since the module is not deployed it's OK! + final Optional emptyNotice = + Optional.of(new NoticeContents(moduleName, Collections.emptyList())); + + assertThat( + NoticeFileChecker.run( + bundleDependencies, + deployedModules, + Collections.singletonMap(moduleName, emptyNotice))) + .isEqualTo(0); + } + + @Test + void testRunIncludesBundledNonDeployedModules() throws IOException { + final Map> bundledDependencies = new HashMap<>(); + final Map> notices = new HashMap<>(); + + // a module that is not deployed but bundles another dependency with an empty NOTICE + final String nonDeployedModuleName = "nonDeployed"; + final Dependency nonDeployedDependency = + Dependency.create("a", nonDeployedModuleName, "c", null); + final Dependency bundledDependency = Dependency.create("a", "b", "c", null); + bundledDependencies.put(nonDeployedModuleName, Collections.singleton(bundledDependency)); + // this would usually not be a problem, but since the module is not bundled it's not OK! + final Optional emptyNotice = + Optional.of(new NoticeContents(nonDeployedModuleName, Collections.emptyList())); + notices.put(nonDeployedModuleName, emptyNotice); + + // a module that is deploys and bundles the above + final String bundlingModule = "bundling"; + bundledDependencies.put(bundlingModule, Collections.singleton(nonDeployedDependency)); + final Optional correctNotice = + Optional.of( + new NoticeContents( + bundlingModule, Collections.singletonList(nonDeployedDependency))); + notices.put(bundlingModule, correctNotice); + + final Set deployedModules = Collections.singleton(bundlingModule); + + assertThat(NoticeFileChecker.run(bundledDependencies, deployedModules, notices)) + .isEqualTo(1); + } + + @Test + void testCheckNoticeFileHappyPath() { + final String moduleName = "test"; + final Dependency bundledDependency = Dependency.create("a", "b", "c", null); + final Map> bundleDependencies = new HashMap<>(); + bundleDependencies.put(moduleName, Collections.singleton(bundledDependency)); + + assertThat( + NoticeFileChecker.checkNoticeFile( + bundleDependencies, + moduleName, + new NoticeContents( + moduleName, Collections.singletonList(bundledDependency)))) + .isEmpty(); + } + + @Test + void testCheckNoticeFileRejectsEmptyFile() { + assertThat(NoticeFileChecker.checkNoticeFile(Collections.emptyMap(), "test", null)) + .containsOnlyKeys(NoticeFileChecker.Severity.CRITICAL); + } + + @Test + void testCheckNoticeFileToleratesModuleNameMismatch() { + final String moduleName = "test"; + + assertThat( + NoticeFileChecker.checkNoticeFile( + Collections.emptyMap(), + moduleName, + new NoticeContents(moduleName + "2", Collections.emptyList()))) + .containsOnlyKeys(NoticeFileChecker.Severity.TOLERATED); + } + + @Test + void testCheckNoticeFileRejectsDuplicateLine() { + final String moduleName = "test"; + final Map> bundleDependencies = new HashMap<>(); + bundleDependencies.put( + moduleName, Collections.singleton(Dependency.create("a", "b", "c", null))); + + assertThat( + NoticeFileChecker.checkNoticeFile( + bundleDependencies, + moduleName, + new NoticeContents( + moduleName, + Arrays.asList( + Dependency.create("a", "b", "c", null), + Dependency.create("a", "b", "c", null))))) + .containsOnlyKeys(NoticeFileChecker.Severity.CRITICAL); + } + + @Test + void testCheckNoticeFileRejectsMissingDependency() { + final String moduleName = "test"; + final Map> bundleDependencies = new HashMap<>(); + bundleDependencies.put( + moduleName, Collections.singleton(Dependency.create("a", "b", "c", null))); + + assertThat( + NoticeFileChecker.checkNoticeFile( + bundleDependencies, + moduleName, + new NoticeContents(moduleName, Collections.emptyList()))) + .containsOnlyKeys(NoticeFileChecker.Severity.CRITICAL); + } +} diff --git a/tools/ci/flink-cdc-ci-tools/src/test/java/org/apache/flink/tools/ci/optional/ShadeOptionalCheckerTest.java b/tools/ci/flink-cdc-ci-tools/src/test/java/org/apache/flink/tools/ci/optional/ShadeOptionalCheckerTest.java new file mode 100644 index 00000000000..1cf4d49c54c --- /dev/null +++ b/tools/ci/flink-cdc-ci-tools/src/test/java/org/apache/flink/tools/ci/optional/ShadeOptionalCheckerTest.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.tools.ci.optional; + +import org.apache.flink.cdc.tools.ci.utils.shared.Dependency; +import org.apache.flink.cdc.tools.ci.utils.shared.DependencyTree; + +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.Set; + +import static org.assertj.core.api.Assertions.assertThat; + +class ShadeOptionalCheckerTest { + private static final String MODULE = "module"; + + @Test + void testNonBundledDependencyIsIgnored() { + final Dependency dependency = createMandatoryDependency("a"); + final Set bundled = Collections.emptySet(); + final DependencyTree dependencyTree = new DependencyTree().addDirectDependency(dependency); + + final Set violations = + ShadeOptionalChecker.checkOptionalFlags(MODULE, bundled, dependencyTree); + + assertThat(violations).isEmpty(); + } + + @Test + void testNonBundledDependencyIsIgnoredEvenIfOthersAreBundled() { + final Dependency dependencyA = createMandatoryDependency("a"); + final Dependency dependencyB = createMandatoryDependency("B"); + final Set bundled = Collections.singleton(dependencyB); + final DependencyTree dependencyTree = + new DependencyTree() + .addDirectDependency(dependencyA) + .addDirectDependency(dependencyB); + + final Set violations = + ShadeOptionalChecker.checkOptionalFlags(MODULE, bundled, dependencyTree); + + assertThat(violations).containsExactly(dependencyB); + } + + @Test + void testDirectBundledOptionalDependencyIsAccepted() { + final Dependency dependency = createOptionalDependency("a"); + final Set bundled = Collections.singleton(dependency); + final DependencyTree dependencyTree = new DependencyTree().addDirectDependency(dependency); + + final Set violations = + ShadeOptionalChecker.checkOptionalFlags(MODULE, bundled, dependencyTree); + + assertThat(violations).isEmpty(); + } + + @Test + void testDirectBundledDependencyMustBeOptional() { + final Dependency dependency = createMandatoryDependency("a"); + final Set bundled = Collections.singleton(dependency); + final DependencyTree dependencyTree = new DependencyTree().addDirectDependency(dependency); + + final Set violations = + ShadeOptionalChecker.checkOptionalFlags(MODULE, bundled, dependencyTree); + + assertThat(violations).containsExactly(dependency); + } + + @Test + void testTransitiveBundledOptionalDependencyIsAccepted() { + final Dependency dependencyA = createMandatoryDependency("a"); + final Dependency dependencyB = createOptionalDependency("b"); + final Set bundled = Collections.singleton(dependencyB); + final DependencyTree dependencyTree = + new DependencyTree() + .addDirectDependency(dependencyA) + .addTransitiveDependencyTo(dependencyB, dependencyA); + + final Set violations = + ShadeOptionalChecker.checkOptionalFlags(MODULE, bundled, dependencyTree); + + assertThat(violations).isEmpty(); + } + + @Test + void testTransitiveBundledDependencyMustBeOptional() { + final Dependency dependencyA = createMandatoryDependency("a"); + final Dependency dependencyB = createMandatoryDependency("b"); + final Set bundled = Collections.singleton(dependencyB); + final DependencyTree dependencyTree = + new DependencyTree() + .addDirectDependency(dependencyA) + .addTransitiveDependencyTo(dependencyB, dependencyA); + + final Set violations = + ShadeOptionalChecker.checkOptionalFlags(MODULE, bundled, dependencyTree); + + assertThat(violations).containsExactly(dependencyB); + } + + @Test + void testTransitiveBundledDependencyMayNotBeOptionalIfParentIsOptional() { + final Dependency dependencyA = createOptionalDependency("a"); + final Dependency dependencyB = createMandatoryDependency("b"); + final Set bundled = Collections.singleton(dependencyB); + final DependencyTree dependencyTree = + new DependencyTree() + .addDirectDependency(dependencyA) + .addTransitiveDependencyTo(dependencyB, dependencyA); + + final Set violations = + ShadeOptionalChecker.checkOptionalFlags(MODULE, bundled, dependencyTree); + + assertThat(violations).isEmpty(); + } + + @Test + void testTransitiveBundledDependencyMayNotBeOptionalIfParentHasTestScope() { + final Dependency dependencyA = createTestDependency("a"); + final Dependency dependencyB = createMandatoryDependency("b"); + final Set bundled = Collections.singleton(dependencyB); + final DependencyTree dependencyTree = + new DependencyTree() + .addDirectDependency(dependencyA) + .addTransitiveDependencyTo(dependencyB, dependencyA); + + final Set violations = + ShadeOptionalChecker.checkOptionalFlags(MODULE, bundled, dependencyTree); + + assertThat(violations).isEmpty(); + } + + @Test + void testTransitiveBundledDependencyMayNotBeOptionalIfParentHasProvidedScope() { + final Dependency dependencyA = createProvidedDependency("a"); + final Dependency dependencyB = createMandatoryDependency("b"); + final Set bundled = Collections.singleton(dependencyB); + final DependencyTree dependencyTree = + new DependencyTree() + .addDirectDependency(dependencyA) + .addTransitiveDependencyTo(dependencyB, dependencyA); + + final Set violations = + ShadeOptionalChecker.checkOptionalFlags(MODULE, bundled, dependencyTree); + + assertThat(violations).isEmpty(); + } + + private static Dependency createMandatoryDependency(String artifactId) { + return Dependency.create("groupId", artifactId, "version", null); + } + + private static Dependency createOptionalDependency(String artifactId) { + return Dependency.create("groupId", artifactId, "version", null, "compile", true); + } + + private static Dependency createProvidedDependency(String artifactId) { + return Dependency.create("groupId", artifactId, "version", null, "provided", false); + } + + private static Dependency createTestDependency(String artifactId) { + return Dependency.create("groupId", artifactId, "version", null, "test", false); + } +} diff --git a/tools/ci/flink-cdc-ci-tools/src/test/java/org/apache/flink/tools/ci/utils/dependency/DependencyParserCopyTest.java b/tools/ci/flink-cdc-ci-tools/src/test/java/org/apache/flink/tools/ci/utils/dependency/DependencyParserCopyTest.java new file mode 100644 index 00000000000..fb76592f34f --- /dev/null +++ b/tools/ci/flink-cdc-ci-tools/src/test/java/org/apache/flink/tools/ci/utils/dependency/DependencyParserCopyTest.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.tools.ci.utils.dependency; + +import org.apache.flink.cdc.tools.ci.utils.shared.Dependency; + +import org.junit.jupiter.api.Test; + +import java.util.Map; +import java.util.Set; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests the parsing of {@code dependency:copy}. */ +class DependencyParserCopyTest { + + private static Stream getTestDependencyCopy() { + return Stream.of( + "[INFO] --- maven-dependency-plugin:3.2.0:copy (copy) @ m1 ---", + "[INFO] Configured Artifact: external:dependency1:2.1:jar", + "[INFO] Configured Artifact: external:dependency4:classifier:2.4:jar", + "[INFO] Copying dependency1-2.1.jar to /some/path/dependency1-2.1.jar", + "[INFO] Copying dependency4-2.4.jar to /some/path/dependency4-2.4.jar", + "[INFO]", + "[INFO] --- maven-dependency-plugin:3.2.0:copy (copy) @ m2 ---", + "[INFO] Configured Artifact: internal:m1:1.1:jar", + "[INFO] Copying internal-1.1.jar to /some/path/m1-1.1.jar"); + } + + @Test + void testCopyParsing() { + final Map> dependenciesByModule = + DependencyParser.parseDependencyCopyOutput(getTestDependencyCopy()); + + assertThat(dependenciesByModule).containsOnlyKeys("m1", "m2"); + assertThat(dependenciesByModule.get("m1")) + .containsExactlyInAnyOrder( + Dependency.create("external", "dependency1", "2.1", null), + Dependency.create("external", "dependency4", "2.4", "classifier")); + assertThat(dependenciesByModule.get("m2")) + .containsExactlyInAnyOrder(Dependency.create("internal", "m1", "1.1", null)); + } + + @Test + void testCopyLineParsingGroupId() { + assertThat( + DependencyParser.parseCopyDependency( + "[INFO] Configured Artifact: external:dependency1:1.0:jar")) + .hasValueSatisfying( + dependency -> assertThat(dependency.getGroupId()).isEqualTo("external")); + } + + @Test + void testCopyLineParsingArtifactId() { + assertThat( + DependencyParser.parseCopyDependency( + "[INFO] Configured Artifact: external:dependency1:1.0:jar")) + .hasValueSatisfying( + dependency -> + assertThat(dependency.getArtifactId()).isEqualTo("dependency1")); + } + + @Test + void testCopyLineParsingVersion() { + assertThat( + DependencyParser.parseCopyDependency( + "[INFO] Configured Artifact: external:dependency1:1.0:jar")) + .hasValueSatisfying( + dependency -> assertThat(dependency.getVersion()).isEqualTo("1.0")); + } + + @Test + void testCopyLineParsingScope() { + assertThat( + DependencyParser.parseCopyDependency( + "[INFO] Configured Artifact: external:dependency1:1.0:jar")) + .hasValueSatisfying(dependency -> assertThat(dependency.getScope()).isEmpty()); + } + + @Test + void testCopyLineParsingOptional() { + assertThat( + DependencyParser.parseCopyDependency( + "[INFO] Configured Artifact: external:dependency1:1.0:jar")) + .hasValueSatisfying(dependency -> assertThat(dependency.isOptional()).isEmpty()); + } + + @Test + void testCopyLineParsingWithNonJarType() { + assertThat( + DependencyParser.parseCopyDependency( + "[INFO] Configured Artifact: external:dependency1:1.0:pom")) + .hasValue(Dependency.create("external", "dependency1", "1.0", null)); + } + + @Test + void testCopyLineParsingClassifier() { + assertThat( + DependencyParser.parseCopyDependency( + "[INFO] Configured Artifact: external:dependency1:some_classifier:1.0:jar")) + .hasValueSatisfying( + dependency -> + assertThat(dependency.getClassifier()).hasValue("some_classifier")); + } +} diff --git a/tools/ci/flink-cdc-ci-tools/src/test/java/org/apache/flink/tools/ci/utils/dependency/DependencyParserTreeTest.java b/tools/ci/flink-cdc-ci-tools/src/test/java/org/apache/flink/tools/ci/utils/dependency/DependencyParserTreeTest.java new file mode 100644 index 00000000000..3d5e499d1be --- /dev/null +++ b/tools/ci/flink-cdc-ci-tools/src/test/java/org/apache/flink/tools/ci/utils/dependency/DependencyParserTreeTest.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.tools.ci.utils.dependency; + +import org.apache.flink.cdc.tools.ci.utils.shared.Dependency; +import org.apache.flink.cdc.tools.ci.utils.shared.DependencyTree; + +import org.junit.jupiter.api.Test; + +import java.util.Map; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests the parsing of {@code dependency:tree}. */ +class DependencyParserTreeTest { + + private static Stream getTestDependencyTree() { + return Stream.of( + "[INFO] --- maven-dependency-plugin:3.2.0:tree (default-cli) @ m1 ---", + "[INFO] internal:m1:jar:1.1", + "[INFO] +- external:dependency1:jar:2.1:compile", + "[INFO] | +- external:dependency2:jar:2.2:compile (optional)", + "[INFO] | | \\- external:dependency3:jar:2.3:provided", + "[INFO] | +- external:dependency4:jar:classifier:2.4:compile", + "[INFO]", + "[INFO] --- maven-dependency-plugin:3.2.0:tree (default-cli) @ m2 ---", + "[INFO] internal:m2:jar:1.2", + "[INFO] +- internal:m1:jar:1.1:compile", + "[INFO] | +- external:dependency4:jar:2.4:compile"); + } + + @Test + void testTreeParsing() { + final Map dependenciesByModule = + DependencyParser.parseDependencyTreeOutput(getTestDependencyTree()); + + assertThat(dependenciesByModule).containsOnlyKeys("m1", "m2"); + assertThat(dependenciesByModule.get("m1").flatten()) + .containsExactlyInAnyOrder( + Dependency.create("external", "dependency1", "2.1", null, "compile", false), + Dependency.create("external", "dependency2", "2.2", null, "compile", true), + Dependency.create( + "external", "dependency3", "2.3", null, "provided", false), + Dependency.create( + "external", "dependency4", "2.4", "classifier", "compile", false)); + assertThat(dependenciesByModule.get("m2").flatten()) + .containsExactlyInAnyOrder( + Dependency.create("internal", "m1", "1.1", null, "compile", false), + Dependency.create( + "external", "dependency4", "2.4", null, "compile", false)); + } + + @Test + void testTreeLineParsingGroupId() { + assertThat( + DependencyParser.parseTreeDependency( + "[INFO] +- external:dependency1:jar:1.0:compile")) + .hasValueSatisfying( + dependency -> assertThat(dependency.getGroupId()).isEqualTo("external")); + } + + @Test + void testTreeLineParsingArtifactId() { + assertThat( + DependencyParser.parseTreeDependency( + "[INFO] +- external:dependency1:jar:1.0:compile")) + .hasValueSatisfying( + dependency -> + assertThat(dependency.getArtifactId()).isEqualTo("dependency1")); + } + + @Test + void testTreeLineParsingVersion() { + assertThat( + DependencyParser.parseTreeDependency( + "[INFO] +- external:dependency1:jar:1.0:compile")) + .hasValueSatisfying( + dependency -> assertThat(dependency.getVersion()).isEqualTo("1.0")); + } + + @Test + void testTreeLineParsingScope() { + assertThat( + DependencyParser.parseTreeDependency( + "[INFO] +- external:dependency1:jar:1.0:provided")) + .hasValueSatisfying( + dependency -> assertThat(dependency.getScope()).hasValue("provided")); + } + + @Test + void testTreeLineParsingWithNonJarType() { + assertThat( + DependencyParser.parseTreeDependency( + "[INFO] +- external:dependency1:pom:1.0:compile")) + .hasValue( + Dependency.create( + "external", "dependency1", "1.0", null, "compile", false)); + } + + @Test + void testTreeLineParsingWithClassifier() { + assertThat( + DependencyParser.parseTreeDependency( + "[INFO] +- external:dependency1:jar:some_classifier:1.0:compile")) + .hasValue( + Dependency.create( + "external", + "dependency1", + "1.0", + "some_classifier", + "compile", + false)); + } + + @Test + void testTreeLineParsingWithoutOptional() { + assertThat( + DependencyParser.parseTreeDependency( + "[INFO] +- external:dependency1:jar:1.0:compile")) + .hasValueSatisfying( + dependency -> assertThat(dependency.isOptional()).hasValue(false)); + } + + @Test + void testTreeLineParsingWithOptional() { + assertThat( + DependencyParser.parseTreeDependency( + "[INFO] +- external:dependency1:jar:1.0:compile (optional)")) + .hasValueSatisfying( + dependency -> assertThat(dependency.isOptional()).hasValue(true)); + } +} diff --git a/tools/ci/flink-cdc-ci-tools/src/test/java/org/apache/flink/tools/ci/utils/deploy/DeployParserTest.java b/tools/ci/flink-cdc-ci-tools/src/test/java/org/apache/flink/tools/ci/utils/deploy/DeployParserTest.java new file mode 100644 index 00000000000..37ad5c86c98 --- /dev/null +++ b/tools/ci/flink-cdc-ci-tools/src/test/java/org/apache/flink/tools/ci/utils/deploy/DeployParserTest.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.tools.ci.utils.deploy; + +import org.junit.jupiter.api.Test; + +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + +class DeployParserTest { + @Test + void testParseDeployOutputDetectsDeployment() { + assertThat( + DeployParser.parseDeployOutput( + Stream.of( + "[INFO] --- maven-deploy-plugin:2.8.2:deploy (default-deploy) @ flink-parent ---", + "[INFO] "))) + .containsExactly("flink-parent"); + } + + @Test + void testParseDeployOutputDetectsDeploymentWithAltRepository() { + assertThat( + DeployParser.parseDeployOutput( + Stream.of( + "[INFO] --- maven-deploy-plugin:2.8.2:deploy (default-deploy) @ flink-parent ---", + "[INFO] Using alternate deployment repository.../tmp/flink-validation-deployment"))) + .containsExactly("flink-parent"); + } + + @Test + void testParseDeployOutputDetectsSkippedDeployments() { + assertThat( + DeployParser.parseDeployOutput( + Stream.of( + "[INFO] --- maven-deploy-plugin:2.8.2:deploy (default-deploy) @ flink-parent ---", + "[INFO] Skipping artifact deployment"))) + .isEmpty(); + } +} diff --git a/tools/ci/flink-cdc-ci-tools/src/test/java/org/apache/flink/tools/ci/utils/notice/NoticeParserTest.java b/tools/ci/flink-cdc-ci-tools/src/test/java/org/apache/flink/tools/ci/utils/notice/NoticeParserTest.java new file mode 100644 index 00000000000..89473e5b740 --- /dev/null +++ b/tools/ci/flink-cdc-ci-tools/src/test/java/org/apache/flink/tools/ci/utils/notice/NoticeParserTest.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.tools.ci.utils.notice; + +import org.apache.flink.cdc.tools.ci.utils.shared.Dependency; + +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +class NoticeParserTest { + @Test + void testParseNoticeFileCommonPath() { + final String module = "some-module"; + final Dependency dependency1 = + Dependency.create("groupId1", "artifactId1", "version1", null); + final Dependency dependency2 = + Dependency.create("groupId2", "artifactId2", "version2", "classifier2"); + final Dependency dependency3 = + Dependency.create("org.codehaus.woodstox", "stax2-api", "4.2.1", null); + final List noticeContents = + Arrays.asList( + module, + "", + "Some text about the applicable license", + "- groupId1:artifactId1:version1", + "- groupId2:artifactId2:classifier2:version2", + "- org.codehaus.woodstox:stax2-api:4.2.1 (https://github.com/FasterXML/stax2-api/tree/stax2-api-4.2.1)", + "", + "some epilogue"); + + assertThat(NoticeParser.parseNoticeFile(noticeContents)) + .hasValueSatisfying( + contents -> { + assertThat(contents.getNoticeModuleName()).isEqualTo(module); + assertThat(contents.getDeclaredDependencies()) + .containsExactlyInAnyOrder( + dependency1, dependency2, dependency3); + }); + } + + @Test + void testParseNoticeFileBundlesPath() { + final String module = "some-module"; + final Dependency dependency = + Dependency.create("groupId", "artifactId", "version", "classifier"); + final List noticeContents = + Arrays.asList( + module, "", "Something bundles \"groupId:artifactId:classifier:version\""); + + assertThat(NoticeParser.parseNoticeFile(noticeContents)) + .hasValueSatisfying( + contents -> { + assertThat(contents.getNoticeModuleName()).isEqualTo(module); + assertThat(contents.getDeclaredDependencies()) + .containsExactlyInAnyOrder(dependency); + }); + } + + @Test + void testParseNoticeFileMalformedDependencyIgnored() { + final String module = "some-module"; + final Dependency dependency = Dependency.create("groupId", "artifactId", "version", null); + final List noticeContents = Arrays.asList(module, "- " + dependency, "- a:b"); + + assertThat(NoticeParser.parseNoticeFile(noticeContents)) + .hasValueSatisfying( + contents -> { + assertThat(contents.getNoticeModuleName()).isEqualTo(module); + assertThat(contents.getDeclaredDependencies()) + .containsExactlyInAnyOrder(dependency); + }); + } +} diff --git a/tools/ci/flink-cdc-ci-tools/src/test/java/org/apache/flink/tools/ci/utils/shade/ShadeParserTest.java b/tools/ci/flink-cdc-ci-tools/src/test/java/org/apache/flink/tools/ci/utils/shade/ShadeParserTest.java new file mode 100644 index 00000000000..bb433f622d5 --- /dev/null +++ b/tools/ci/flink-cdc-ci-tools/src/test/java/org/apache/flink/tools/ci/utils/shade/ShadeParserTest.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.tools.ci.utils.shade; + +import org.apache.flink.cdc.tools.ci.utils.shared.Dependency; + +import org.junit.jupiter.api.Test; + +import java.util.Map; +import java.util.Set; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + +class ShadeParserTest { + + private static Stream getTestDependencyCopy() { + return Stream.of( + "[INFO] --- maven-shade-plugin:3.2.0:shade (shade-flink) @ m1 ---", + "[INFO] Including external:dependency1:jar:2.1 in the shaded jar.", + "[INFO] Excluding external:dependency3:jar:2.3 from the shaded jar.", + "[INFO] Including external:dependency4:jar:classifier:2.4 in the shaded jar.", + "[INFO] Replacing original artifact with shaded artifact.", + "[INFO] Replacing /some/path/m1.jar with /some/path/m1-shaded.jar", + "[INFO]", + "[INFO] --- maven-shade-plugin:3.2.0:shade (shade-flink) @ m2 ---", + "[INFO] Including internal:m1:jar:1.1 in the shaded jar.", + "[INFO] Replacing /some/path/m2.jar with /some/path/m2-shaded.jar"); + } + + @Test + void testParsing() { + final Map> dependenciesByModule = + ShadeParser.parseShadeOutput(getTestDependencyCopy()); + + assertThat(dependenciesByModule).containsOnlyKeys("m1", "m2"); + assertThat(dependenciesByModule.get("m1")) + .containsExactlyInAnyOrder( + Dependency.create("external", "dependency1", "2.1", null), + Dependency.create("external", "dependency4", "2.4", "classifier")); + assertThat(dependenciesByModule.get("m2")) + .containsExactlyInAnyOrder(Dependency.create("internal", "m1", "1.1", null)); + } + + @Test + void testLineParsingGroupId() { + assertThat( + ShadeParser.parseDependency( + "Including external:dependency1:jar:1.0 in the shaded jar.")) + .hasValueSatisfying( + dependency -> assertThat(dependency.getGroupId()).isEqualTo("external")); + } + + @Test + void testLineParsingArtifactId() { + assertThat( + ShadeParser.parseDependency( + "Including external:dependency1:jar:1.0 in the shaded jar.")) + .hasValueSatisfying( + dependency -> + assertThat(dependency.getArtifactId()).isEqualTo("dependency1")); + } + + @Test + void testLineParsingVersion() { + assertThat( + ShadeParser.parseDependency( + "Including external:dependency1:jar:1.0 in the shaded jar.")) + .hasValueSatisfying( + dependency -> assertThat(dependency.getVersion()).isEqualTo("1.0")); + } + + @Test + void testLineParsingWithNonJarType() { + assertThat( + ShadeParser.parseDependency( + "Including external:dependency1:pom:1.0 in the shaded jar.")) + .isPresent(); + } +} diff --git a/tools/ci/flink-cdc-ci-tools/src/test/java/org/apache/flink/tools/ci/utils/shared/DependencyTreeTest.java b/tools/ci/flink-cdc-ci-tools/src/test/java/org/apache/flink/tools/ci/utils/shared/DependencyTreeTest.java new file mode 100644 index 00000000000..c21edbf0467 --- /dev/null +++ b/tools/ci/flink-cdc-ci-tools/src/test/java/org/apache/flink/tools/ci/utils/shared/DependencyTreeTest.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.tools.ci.utils.shared; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +class DependencyTreeTest { + private static final Dependency DEPENDENCY = + Dependency.create("groupId", "artifactId", "version", null); + + @Test + void testDependencyKeyIncludesGroupId() { + testDependencyKeyInclusion( + Dependency.create( + "xxx", + DEPENDENCY.getArtifactId(), + DEPENDENCY.getVersion(), + DEPENDENCY.getClassifier().orElse(null))); + } + + @Test + void testDependencyKeyIncludesArtifactId() { + testDependencyKeyInclusion( + Dependency.create( + DEPENDENCY.getGroupId(), + "xxx", + DEPENDENCY.getVersion(), + DEPENDENCY.getClassifier().orElse(null))); + } + + @Test + void testDependencyKeyIncludesVersion() { + testDependencyKeyInclusion( + Dependency.create( + DEPENDENCY.getGroupId(), + DEPENDENCY.getArtifactId(), + "xxx", + DEPENDENCY.getClassifier().orElse(null))); + } + + @Test + void testDependencyKeyIncludesClassifier() { + testDependencyKeyInclusion( + Dependency.create( + DEPENDENCY.getGroupId(), + DEPENDENCY.getArtifactId(), + DEPENDENCY.getVersion(), + "xxx")); + } + + private static void testDependencyKeyInclusion(Dependency modifiedDependency) { + final DependencyTree dependencyTree = new DependencyTree(); + dependencyTree.addDirectDependency(DEPENDENCY); + dependencyTree.addDirectDependency(modifiedDependency); + + assertThat(dependencyTree.flatten()).containsExactly(DEPENDENCY, modifiedDependency); + } + + @Test + void testDependencyKeyIgnoresScopeAndOptionalFlag() { + final Dependency dependencyWithScopeAndOptionalFlag = + Dependency.create( + DEPENDENCY.getGroupId(), + DEPENDENCY.getArtifactId(), + DEPENDENCY.getVersion(), + DEPENDENCY.getClassifier().orElse(null), + "compile", + true); + + final DependencyTree dependencyTree = new DependencyTree(); + dependencyTree.addDirectDependency(DEPENDENCY); + dependencyTree.addDirectDependency(dependencyWithScopeAndOptionalFlag); + + assertThat(dependencyTree.flatten()).containsExactly(DEPENDENCY); + assertThat(dependencyTree.getPathTo(dependencyWithScopeAndOptionalFlag)) + .containsExactly(DEPENDENCY); + } +} diff --git a/tools/ci/flink-cdc-ci-tools/src/test/resources/META-INF/services/org.junit.jupiter.api.extension.Extension b/tools/ci/flink-cdc-ci-tools/src/test/resources/META-INF/services/org.junit.jupiter.api.extension.Extension new file mode 100644 index 00000000000..28999133c2b --- /dev/null +++ b/tools/ci/flink-cdc-ci-tools/src/test/resources/META-INF/services/org.junit.jupiter.api.extension.Extension @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.flink.util.TestLoggerExtension \ No newline at end of file diff --git a/tools/ci/license_check.rb b/tools/ci/license_check.rb deleted file mode 100755 index ac74c52ca83..00000000000 --- a/tools/ci/license_check.rb +++ /dev/null @@ -1,135 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -require 'zip' - -# These maven modules don't need to be checked -EXCLUDED_MODULES = %w[flink-cdc-dist flink-cdc-e2e-tests].freeze - -# Questionable license statements which shouldn't occur in packaged jar files -QUESTIONABLE_STATEMENTS = [ - 'Binary Code License', - 'Intel Simplified Software License', - 'JSR 275', - 'Microsoft Limited Public License', - 'Amazon Software License', - # Java SDK for Satori RTM license - 'as necessary for your use of Satori services', - 'REDIS SOURCE AVAILABLE LICENSE', - 'Booz Allen Public License', - 'Confluent Community License Agreement Version 1.0', - # “Commons Clause” License Condition v1.0 - 'the License does not grant to you, the right to Sell the Software.', - 'Sun Community Source License Version 3.0', - 'GNU General Public License', - 'GNU Affero General Public License', - 'GNU Lesser General Public License', - 'Q Public License', - 'Sleepycat License', - 'Server Side Public License', - 'Code Project Open License', - # BSD 4-Clause - ' All advertising materials mentioning features or use of this software must display the following acknowledgement', - # Facebook Patent clause v1 - 'The license granted hereunder will terminate, automatically and without notice, for anyone that makes any claim', - # Facebook Patent clause v2 - 'The license granted hereunder will terminate, automatically and without notice, if you (or any of your subsidiaries, corporate affiliates or agents) initiate directly or indirectly, or take a direct financial interest in, any Patent Assertion: (i) against Facebook', - 'Netscape Public License', - 'SOLIPSISTIC ECLIPSE PUBLIC LICENSE', - # DON'T BE A DICK PUBLIC LICENSE - "Do whatever you like with the original work, just don't be a dick.", - # JSON License - 'The Software shall be used for Good, not Evil.', - # can sometimes be found in "funny" licenses - 'Don’t be evil', - # IBM's non-FOSS license - 'International Program License Agreement', - # Oracle's non-FOSS license - 'Oracle Free Use Terms and Conditions' -].freeze - -# These file extensions are binary-formatted. No check needed. -BINARY_FILE_EXTENSIONS = %w[.class .dylib .so .dll .gif .ico].freeze - -# These packages are licensed under "Weak Copyleft" licenses. -# According to Apache official guidelines, such software could be -# packaged in jar if appropriately labelled. -# See https://www.apache.org/legal/resolved.html for more details. -EXCEPTION_PACKAGES = [ - 'org/glassfish/jersey/', # dual-licensed under GPL 2 and EPL 2.0 - 'org.glassfish.jersey', # dual-licensed under GPL 2 and EPL 2.0 - 'org.glassfish.hk2', # dual-licensed under GPL 2 and EPL 2.0 - 'javax.ws.rs-api', # dual-licensed under GPL 2 and EPL 2.0 - 'jakarta.ws.rs' # dual-licensed under GPL 2 and EPL 2.0 -].freeze - -puts 'Start license check...' - -# Extract Flink CDC revision number from global pom.xml -begin - REVISION_NUMBER = File.read('pom.xml').scan(%r{(.*)}).last[0] -rescue NoMethodError - abort 'Could not extract Flink CDC revision number from pom.xml' -end - -puts "Flink CDC version: '#{REVISION_NUMBER}'" - -# Traversing maven module in given path -def traverse_module(path) - module_name = File.basename path - return if EXCLUDED_MODULES.include?(module_name) - - jar_file = File.join path, 'target', "#{module_name}-#{REVISION_NUMBER}.jar" - check_jar_license jar_file if File.exist? jar_file - - File.read(File.join(path, 'pom.xml')).scan(%r{(.*)}).map(&:first).each do |submodule| - traverse_module File.join(path, submodule.to_s) unless submodule.nil? - end -end - -@tainted_records = [] - -# Check license issues in given jar file -def check_jar_license(jar_file) - puts "Checking jar file #{jar_file}" - Zip::File.open(jar_file) do |jar| - jar.filter { |e| e.ftype == :file } - .filter { |e| !File.basename(e.name).downcase.end_with?(*BINARY_FILE_EXTENSIONS) } - .filter { |e| !File.basename(e.name).downcase.start_with? 'license', 'dependencies' } - .filter { |e| EXCEPTION_PACKAGES.none? { |ex| e.name.include? ex } } - .map do |e| - content = e.get_input_stream.read.force_encoding('UTF-8') - next unless QUESTIONABLE_STATEMENTS.map { |stmt| content.include?(stmt) }.any? - - @tainted_records.push({ - jar_file: File.basename(jar_file), - suspicious_file: e.name - }) - end - end -end - -traverse_module '.' - -unless @tainted_records.empty? - puts "\nError: packaged jar contains files with incompatible licenses:" - puts @tainted_records.map { |e| " -> In #{e[:jar_file]}: #{e[:suspicious_file]}" }.join("\n") - abort 'See https://www.apache.org/legal/resolved.html for more details.' -end - -puts 'License check passed.'