diff --git a/.ci/validation.sh b/.ci/validation.sh index fcab7ea9..62656e29 100755 --- a/.ci/validation.sh +++ b/.ci/validation.sh @@ -41,7 +41,7 @@ checkstyle-tester-diff-groovy-patch) sed -i'' 's/^guava/#guava/' ../.ci-temp/projects-to-test-on.properties sed -i'' 's/#checkstyle/checkstyle/' ../.ci-temp/projects-to-test-on.properties groovy diff.groovy -l ../.ci-temp/projects-to-test-on.properties \ - -c my_check.xml -b master -p patch-branch -r ../.ci-temp/checkstyle + -c my_check.xml -b master -p patch-branch -r ../.ci-temp/checkstyle -h ;; checkstyle-tester-diff-groovy-base-patch) @@ -50,7 +50,7 @@ checkstyle-tester-diff-groovy-base-patch) git checkout -b patch-branch cd ../../checkstyle-tester groovy diff.groovy -l projects-to-test-on.properties \ - -bc my_check.xml -pc my_check.xml -b master -p patch-branch -r ../.ci-temp/checkstyle + -bc my_check.xml -pc my_check.xml -b master -p patch-branch -r ../.ci-temp/checkstyle -h ;; checkstyle-tester-diff-groovy-patch-only) @@ -59,7 +59,7 @@ checkstyle-tester-diff-groovy-patch-only) git checkout -b patch-branch cd ../../checkstyle-tester groovy diff.groovy -l projects-to-test-on.properties \ - -pc my_check.xml -p patch-branch -r ../.ci-temp/checkstyle -m single + -pc my_check.xml -p patch-branch -r ../.ci-temp/checkstyle -m single --useShallowClone ;; checkstyle-tester-diff-groovy-regression-single) @@ -76,7 +76,7 @@ checkstyle-tester-diff-groovy-regression-single) groovy ./diff.groovy --listOfProjects projects-to-test-on.properties \ -pc ../../../checkstyle-tester/diff-groovy-regression-config.xml \ -r ../../checkstyle -xm "-Dcheckstyle.failsOnError=false" \ - -m single -p master + -m single -p master -h # Run report with current branch cd ../../../checkstyle-tester/ @@ -85,7 +85,7 @@ checkstyle-tester-diff-groovy-regression-single) rm -rf reports repositories groovy ./diff.groovy --listOfProjects projects-to-test-on.properties \ -pc diff-groovy-regression-config.xml -r ../.ci-temp/checkstyle/ \ - -m single -p master -xm "-Dcheckstyle.failsOnError=false" + -m single -p master -xm "-Dcheckstyle.failsOnError=false" -h cd .. # We need to ignore file paths below, since they will be different between reports diff --git a/appveyor.yml b/appveyor.yml index edb4de1b..6293a594 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -82,7 +82,7 @@ environment: CMD3: " && cd checkstyle && git checkout -b patch-branch" CMD4: " " CMD5: " && cd ..\\checkstyle-tester " - CMD6: " && groovy diff.groovy -l projects-to-test-on.properties -bc my_check.xml -pc my_check.xml -b master -p patch-branch -r C:\\projects\\contribution\\checkstyle -s" + CMD6: " && groovy diff.groovy -l projects-to-test-on.properties -bc my_check.xml -pc my_check.xml -b master -p patch-branch -r C:\\projects\\contribution\\checkstyle -s -h" build_script: - ps: > diff --git a/checkstyle-tester/README.md b/checkstyle-tester/README.md index 18f0cda0..5528b950 100644 --- a/checkstyle-tester/README.md +++ b/checkstyle-tester/README.md @@ -72,6 +72,16 @@ would add `--extraMvnRegressionOptions "-Dmaven.site.skip=true"` to `diff.groovy (optional, default is false). This option is used for files that are not compilable or that Checkstyle cannot parse. +**useShallowClone** (h) - this option tells `diff.groovy` to use shallow clone for repositories +defined in the file +specified for the `--listOfProjects (-l)` argument. This option is a toggle (does not require an argument) +and convenient for cases when internet is not stable and it is better to clone minimal required sources. +Shallow cloning cannot be used for projects that reference SHA commit and, by default, +fallback to using normal cloning. ATTENTION: if you change tag/branch in config for project +that previously cloned shallowly there will be error to checkout to new tag/branch +(Example: `fatal: Could not parse object 'c2ac5b90a467aedb04b52ae50a99e83207d847b3'.`), to resovle +problem you need to remove impacted project folder(s) from repositories directory. + ## Outputs When the script finishes its work the following directory structure will be created diff --git a/checkstyle-tester/diff.groovy b/checkstyle-tester/diff.groovy index aa088f2a..5524949d 100644 --- a/checkstyle-tester/diff.groovy +++ b/checkstyle-tester/diff.groovy @@ -68,6 +68,7 @@ def getCliOptions(args) { + 'config file (required if baseConfig and patchConfig are not secified)') g(longOpt: 'allowExcludes', required: false, 'Whether to allow excludes specified in the list of ' \ + 'projects (optional, default is false)') + h(longOpt: 'useShallowClone', 'Enable shallow cloning') l(longOpt: 'listOfProjects', args: 1, required: true, argName: 'path', 'Path to file which contains projects to test on (required)') s(longOpt: 'shortFilePaths', required: false, 'Whether to save report file paths' \ @@ -264,6 +265,7 @@ def generateCheckstyleReport(cfg) { def checkstyleConfig = cfg.checkstyleCfg def checkstyleVersion = cfg.checkstyleVersion def allowExcludes = cfg.allowExcludes + def useShallowClone = cfg.useShallowClone def listOfProjectsFile = new File(cfg.listOfProjects) def projects = listOfProjectsFile.readLines() def extraMvnRegressionOptions = cfg.extraMvnRegressionOptions @@ -292,7 +294,11 @@ def generateCheckstyleReport(cfg) { if (repoType == 'local') { copyDir(repoUrl, getOsSpecificPath("$srcDir", "$repoName")) } else { - cloneRepository(repoName, repoType, repoUrl, commitId, reposDir) + if (useShallowClone && !isGitSha(commitId)) { + shallowCloneRepository(repoName, repoType, repoUrl, commitId, reposDir) + } else { + cloneRepository(repoName, repoType, repoUrl, commitId, reposDir) + } copyDir(getOsSpecificPath("$reposDir", "$repoName"), getOsSpecificPath("$srcDir", "$repoName")) } runMavenExecution(srcDir, excludes, checkstyleConfig, @@ -340,16 +346,15 @@ def getCommitSha(commitId, repoType, srcDestinationDir) { return sha.replace('\n', '') } -def getCloneCmd(repoType, repoUrl, srcDestinationDir) { - def cloneCmd = '' - switch (repoType) { - case 'git': - cloneCmd = "git clone $repoUrl $srcDestinationDir" - break - default: - throw new IllegalArgumentException("Error! Unknown $repoType repository.") +def shallowCloneRepository(repoName, repoType, repoUrl, commitId, srcDir) { + def srcDestinationDir = getOsSpecificPath("$srcDir", "$repoName") + if (!Files.exists(Paths.get(srcDestinationDir))) { + def cloneCmd = getCloneShallowCmd(repoType, repoUrl, srcDestinationDir, commitId) + println "Shallow clone $repoType repository '$repoName' to $srcDestinationDir folder ..." + executeCmdWithRetry(cloneCmd) + println "Cloning $repoType repository '$repoName' - completed\n" } - return cloneCmd + println "$repoName is synchronized" } def cloneRepository(repoName, repoType, repoUrl, commitId, srcDir) { @@ -365,6 +370,10 @@ def cloneRepository(repoName, repoType, repoUrl, commitId, srcDir) { def lastCommitSha = getLastProjectCommitSha(repoType, srcDestinationDir) def commitIdSha = getCommitSha(commitId, repoType, srcDestinationDir) if (lastCommitSha != commitIdSha) { + if (!isGitSha(commitId)) { + // If commitId is a branch or tag, fetch more data and then reset + fetchAdditionalData(repoType, srcDestinationDir, commitId) + } def resetCmd = getResetCmd(repoType, commitId) println "Resetting $repoType sources to commit '$commitId'" executeCmd(resetCmd, new File("$srcDestinationDir")) @@ -373,6 +382,51 @@ def cloneRepository(repoName, repoType, repoUrl, commitId, srcDir) { println "$repoName is synchronized" } +def getCloneCmd(repoType, repoUrl, srcDestinationDir) { + if ('git'.equals(repoType)) { + return "git clone $repoUrl $srcDestinationDir" + } + throw new IllegalArgumentException("Error! Unknown $repoType repository.") +} + +def getCloneShallowCmd(repoType, repoUrl, srcDestinationDir, commitId) { + if ('git'.equals(repoType)) { + return "git clone --depth 1 --branch $commitId $repoUrl $srcDestinationDir" + } + throw new IllegalArgumentException("Error! Unknown repository type: $repoType") +} + +def fetchAdditionalData(repoType, srcDestinationDir, commitId) { + String fetchCmd = '' + if ('git'.equals(repoType)) { + if (isGitSha(commitId)) { + fetchCmd = "git fetch" + } else { + // Check if commitId is a tag and handle accordingly + if (isTag(commitId, new File(srcDestinationDir))) { + fetchCmd = "git fetch --tags" + } else { + fetchCmd = "git fetch origin $commitId:$commitId" + } + } + } else { + throw new IllegalArgumentException("Error! Unknown $repoType repository.") + } + + executeCmd(fetchCmd, new File(srcDestinationDir)) +} + +def isTag(commitId, gitRepo) { + def tagCheckCmd = "git tag -l $commitId".execute(null, gitRepo) + tagCheckCmd.waitFor() + return tagCheckCmd.text.trim().equals(commitId) +} + +// it is not very accurate match, but in case of mismatch we will do full clone +def isGitSha(value) { + return value ==~ /[0-9a-f]{5,40}/ +} + def executeCmdWithRetry(cmd, dir = new File("").getAbsoluteFile(), retry = 5) { def osSpecificCmd = getOsSpecificCmd(cmd) def left = retry @@ -708,7 +762,11 @@ def getResetCmd(repoType, commitId) { def resetCmd = '' switch (repoType) { case 'git': - resetCmd = "git reset --hard $commitId" + if (isGitSha(commitId)) { + resetCmd = "git reset --hard $commitId" + } else { + resetCmd = "git reset --hard refs/tags/$commitId" + } break default: throw new IllegalArgumentException("Error! Unknown $repoType repository.") @@ -755,6 +813,7 @@ class Config { def checkstyleVersion def sevntuVersion def allowExcludes + def useShallowClone Config(cliOptions) { if (cliOptions.localGitRepo) { @@ -767,6 +826,7 @@ class Config { checkstyleVersion = cliOptions.checkstyleVersion allowExcludes = cliOptions.allowExcludes + useShallowClone = cliOptions.useShallowClone mode = cliOptions.mode if (!mode) { @@ -815,6 +875,7 @@ class Config { destDir: tmpMasterReportsDir, extraMvnRegressionOptions: extraMvnRegressionOptions, allowExcludes:allowExcludes, + useShallowClone: useShallowClone, ] } @@ -827,6 +888,7 @@ class Config { destDir: tmpPatchReportsDir, extraMvnRegressionOptions: extraMvnRegressionOptions, allowExcludes: allowExcludes, + useShallowClone: useShallowClone, ] } @@ -840,6 +902,7 @@ class Config { shortFilePaths: shortFilePaths, mode: mode, allowExcludes: allowExcludes, + useShallowClone: useShallowClone, ] } }