Skip to content

Commit

Permalink
chore: Initial workflow for running functional tests
Browse files Browse the repository at this point in the history
  • Loading branch information
rsenden committed Jul 13, 2023
1 parent 11aa40a commit 761832c
Show file tree
Hide file tree
Showing 11 changed files with 166 additions and 75 deletions.
8 changes: 4 additions & 4 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@ name: Build and release
on:
workflow_dispatch:

# push:
# branches:
# - '**'
push:
branches:
- '**'

env:
native_image_opts: --verbose -H:Log=registerResource:verbose -H:+PrintClassInitialization
Expand Down Expand Up @@ -67,7 +67,7 @@ jobs:
- name: Build release ${{env.RELEASE_VERSION}}
if: env.DO_BUILD
run: ./gradlew clean build dist distThirdParty -Pversion=${{env.RELEASE_VERSION}}
run: ./gradlew clean build dist distThirdPartyReleaseAsset distFtest -Pversion=${{env.RELEASE_VERSION}}

- name: Check fcli version
if: env.DO_BUILD
Expand Down
61 changes: 37 additions & 24 deletions .github/workflows/functional-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,19 @@ name: Functional Tests

on:
workflow_dispatch:
schedule:
- cron: '0 2 * * *'
push:
branches:
- 'feat-functest'
inputs:
runNumber: # Accessible through ${{ inputs.runNumber }}
description: 'Optional workflow run number for which to perform functional tests'
required: true
type: number
workflow_run:
workflows: [Build and release]
types: [completed]


jobs:
functional-test:
if: github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch'
strategy:
fail-fast: false
matrix:
Expand All @@ -19,40 +24,48 @@ jobs:
runs-on: ${{ matrix.os }}

steps:
# Check out source code to access the Gradle-based functional tests
- name: Check Out Source Code
uses: actions/checkout@v2
with:
# TODO Update this to 'develop' once merged
ref: feat-functest

# Java is required to run the Gradle-based functional tests and
# for running the Java version of fcli
# Java is required for running the functional tests
- name: Setup Java
uses: actions/setup-java@v1
with:
java-version: 17

- name: Get artifacts from triggering workflow
if: github.event_name == 'workflow_run'
uses: dawidd6/action-download-artifact@v2
with:
run_id: ${{ github.event.workflow_run.id }}
- name: Get artifacts from specified workflow
if: github.event_name == 'workflow_dispatch'
uses: dawidd6/action-download-artifact@v2
with:
run_number: ${{ inputs.runNumber }}
workflow: ci.yml

- name: List artifact contents
shell: bash
run: ls -lR

- name: Run Tests
shell: bash
run: |
case "${{ matrix.type }}" in
"java" )
./gradlew ftest ;;
java -jar -Dft.fcli=build fcli-ftest.jar ;;
"jar" )
./gradlew build
./gradlew ftest -Dft.fcli=$(pwd)/build/libs/fcli.jar ;;
java -jar -Dft.fcli=$(pwd)/release-assets/fcli.jar fcli-ftest.jar ;;
"native" )
case "${{ matrix.os }}" in
"ubuntu-latest" )
curl -fsSL -o - https://github.com/fortify-ps/fcli/releases/download/dev_develop/fcli-linux.tgz | tar -zxvf -
./gradlew ftest -Dft.fcli=$(pwd)/fcli ;;
tar -zxvf release-assets/fcli-linux.tgz
java -jar -Dft.fcli=$(pwd)/fcli fcli-ftest.jar ;;
"windows-latest" )
curl -fsSL -o /tmp/fcli-windows.zip https://github.com/fortify-ps/fcli/releases/download/dev_develop/fcli-windows.zip
7z e /tmp/fcli-windows.zip
./gradlew ftest -Dft.fcli=$(pwd)/fcli.exe ;;
7z e release-assets/fcli-windows.zip
java -jar -Dft.fcli=$(pwd)/fcli.exe fcli-ftest.jar ;;
"macos-latest" )
curl -fsSL -o - https://github.com/fortify-ps/fcli/releases/download/dev_develop/fcli-mac.tgz | tar -zxvf -
tar -zxvf release-assets/fcli-mac.tgz
./gradlew ftest -Dft.fcli=$(pwd)/fcli ;;
esac ;;
esac
esac
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -56,3 +56,4 @@ fcli.log
/ReportTemplateDefAnswerTemplate.json*
/ReportTemplateDefAnswerTemplate.y*ml*
/*.rptdesign
*.log
39 changes: 12 additions & 27 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,9 @@ allprojects {
apply plugin: 'eclipse'
version = rootProject.ext.getVersion();
ext {
distDir = "${buildDir}/dist"
distDir = "${rootProject.buildDir}/dist"
releaseAssetsDir = "${distDir}/release-assets"
sharedGradleScriptsDir = "$rootDir/$sharedGradleScriptsRelativeDir"
sharedGradleScriptsDir = "${rootDir}/$sharedGradleScriptsRelativeDir"
gradleHelpersLocation = "https://raw.githubusercontent.com/fortify/shared-gradle-helpers/1.8"
}
// Define *RefDir properties for each *Ref property defined in gradle.properties,
Expand All @@ -36,18 +36,13 @@ allprojects {
repositories {
mavenCentral()
}
tasks.register('createDistDir') {
doLast {
mkdir "${distDir}"
mkdir "${releaseAssetsDir}"
}
}
}

subprojects {
// Disallow sub-projects from having a dist task;
// they should have a distLocal task instead.
task dist {}

task createDistDir {
doFirst {
mkdir "${distDir}"
mkdir "${releaseAssetsDir}"
}
}
}

configure(allprojects.findAll {it.path.startsWith "$fcliModulePathPrefix"}) {
Expand All @@ -62,19 +57,9 @@ task build(type: Copy) {
into "build/libs"
}
task dist(type: Copy) {
dependsOn(createDistDir)
// For now, this tasks assumes that any subproject that needs to add anything to the dist
// directory implements a distLocal task, and adds this distLocal task as a dependency for
// this dist task, using: rootProject.dist.dependsOn(distLocal).
from { subprojects.collect { it.ext.distDir } }
from("${projectDir}") {
dependsOn(createDistDir)
from("${projectDir}") {
include "LICENSE.txt"
}
into "${distDir}"
}
task distThirdParty(type: Copy) {
dependsOn(createDistDir)
dependsOn("${fcliAppRef}:distThirdPartyReleaseAsset")
from "${fcliAppRefDir}/build/dist-thirdparty"
into "${distDir}"
into "${releaseAssetsDir}"
}
3 changes: 1 addition & 2 deletions buildRoot/app/fcli-autocomplete/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,10 @@ dependencies {
runtimeOnly 'info.picocli:picocli-codegen'
}

task distLocal(type: JavaExec) {
task dist(type: JavaExec) {
dependsOn(createDistDir)
description = "Generate fcli autocomplete"
classpath(configurations.runtimeClasspath, configurations.annotationProcessor)
main 'picocli.AutoComplete'
args fcliRootCommandsClassName, "-f", "--completionScript=${distDir}/fcli_completion"
}
rootProject.dist.dependsOn(distLocal)
3 changes: 1 addition & 2 deletions buildRoot/app/fcli-doc/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,6 @@ task distDocsStaticJekyll(type: Zip) {
}

// Task to combine all distDocs* tasks
task distLocal {
task dist {
dependsOn 'distDocsVersionedHtml', 'distDocsManpage', 'distDocsVersionedJekyll', 'distDocsStaticJekyll'
}
rootProject.dist.dependsOn(distLocal)
24 changes: 23 additions & 1 deletion buildRoot/app/fcli-functional-test/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ testing {
implementation platform("org.spockframework:spock-bom:2.3-groovy-4.0")
implementation "org.spockframework:spock-core"
implementation 'io.github.joke:spock-outputcapture:3.0.1'
implementation 'org.junit.platform:junit-platform-launcher:1.9.3'
if ( !project.hasProperty('ftest.fcli') || project.property('ftest.fcli')=='build' ) {
implementation project("${fcliAppRef}")
}
Expand All @@ -31,4 +32,25 @@ testing {
}
}
}
}
}

apply plugin: 'com.github.johnrengelman.shadow'
task ftestShadowJar(type: com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar) {
archiveBaseName.set('fcli')
archiveClassifier.set('ftest')
archiveVersion.set('')
from(sourceSets.ftest.output)
configurations = [project.configurations.ftestRuntimeClasspath]
manifest {
attributes 'Main-Class': 'com.fortify.cli.ftest.TestRunner'
}
}

task distFtest(type: Copy) {
dependsOn(ftestShadowJar)
dependsOn(createDistDir)
into "${distDir}"
from("${buildDir}/libs") {
include "fcli-ftest.jar"
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
package com.fortify.cli.ftest;

import java.text.SimpleDateFormat

import org.junit.platform.engine.TestExecutionResult
import org.junit.platform.engine.discovery.DiscoverySelectors
import org.junit.platform.launcher.LauncherDiscoveryRequest
import org.junit.platform.launcher.TestExecutionListener
import org.junit.platform.launcher.TestIdentifier
import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder
import org.junit.platform.launcher.core.LauncherFactory
import org.junit.platform.launcher.listeners.SummaryGeneratingListener
import org.junit.platform.launcher.listeners.TestExecutionSummary

/**
* @author Ruud Senden
*/
public class TestRunner {
static def orgOut = System.out
static def orgErr = System.err
public static void main(String[] args) {
new PrintStream(new File("test.log")).withCloseable
{ log ->
final LauncherDiscoveryRequest request =
LauncherDiscoveryRequestBuilder.request()
.selectors(DiscoverySelectors.selectPackage("com.fortify.cli.ftest"))
.build();
def fcliListener = new FcliTestExecutionListener(log);
def summaryListener = new SummaryGeneratingListener();

try {
System.out = log
System.err = log
LauncherFactory.create().execute(request, summaryListener, fcliListener);
TestExecutionSummary summary = summaryListener.getSummary();
new PrintWriter(log).withCloseable { summary.printTo it }
new PrintWriter(orgOut).withCloseable { summary.printTo it }
} finally {
System.out = orgOut
System.err = orgErr
}
}
}

private static class FcliTestExecutionListener implements TestExecutionListener {
private final SimpleDateFormat logPrefixFormat = new SimpleDateFormat("[yyyy-MM-dd HH:mm:ss] ")
private PrintStream log;
private final def startTimes = [:]

private FcliTestExecutionListener(PrintStream log) {
this.log = log;
}
@Override
public void executionSkipped(TestIdentifier testIdentifier, String reason) {
logStatus("SKIPPED: "+testIdentifier.displayName+": "+reason)
}
@Override
public void executionStarted(TestIdentifier testIdentifier) {
if ( !testIdentifier.isContainer() ) {
logStatus("STARTED: "+testIdentifier.displayName)
startTimes[testIdentifier.uniqueId] = System.currentTimeMillis();
}
}
public void executionFinished(TestIdentifier testIdentifier, TestExecutionResult testExecutionResult) {
if ( !testIdentifier.isContainer() ) {
testExecutionResult.throwable.ifPresent({
it.printStackTrace(orgErr)
it.printStackTrace(System.err)
})
logStatus(testExecutionResult.status.name()+": "+testIdentifier.displayName
+"("+(System.currentTimeMillis()-startTimes[testIdentifier.uniqueId])+" ms)")
}
}
private void logStatus(String msg) {
def msgWithPrefix = logPrefixFormat.format(new Date()) + msg;
orgOut.println(msgWithPrefix)
log.println(msgWithPrefix)
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,7 @@ public class FcliRunner {
if ( !runner ) {
throw new IllegalStateException("Runner not initialized")
}
try {
return runner.run(args)
} catch ( Exception e ) {
e.printStackTrace()
return false
}
return runner.run(args)
}

static void close() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,9 @@ import com.fortify.cli.ftest._common.spec.Prefix

@Prefix("core.basic-info")
class BasicInfoSpec extends BaseFcliSpec {
def "help"(String[] args, boolean success) {
def "help"(String[] args, boolean expectSuccess) {
expect:
fcli(args)==success
fcli(args)==expectSuccess
out.lines
verifyAll(out.lines) {
it.any { it ==~ /.*Command-line interface for working with various Fortify products.*/ }
Expand All @@ -20,7 +20,7 @@ class BasicInfoSpec extends BaseFcliSpec {
}

where:
args | success
args | expectSuccess
["-h"] | true // Explicitly invoke fcli -h
[] | false // Invoke fcli without args, resulting in a 'missing required subcommand' error
}
Expand Down
9 changes: 3 additions & 6 deletions buildRoot/app/fcli/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -77,17 +77,14 @@ apply from: "${gradleHelpersLocation}/thirdparty-helper.gradle"
task distThirdPartyReleaseAsset {
dependsOn 'distThirdParty'
doLast {
file("$buildDir/dist-thirdparty/release-assets").mkdirs()
file("$buildDir/dist/fcli-thirdparty.zip").renameTo(file("$buildDir/dist-thirdparty/release-assets/fcli-thirdparty.zip"))
file("$buildDir/dist/fcli-thirdparty.zip").renameTo(file("${releaseAssetsDir}/fcli-thirdparty.zip"))
}
}


task distLocal(type: Copy) {
task dist(type: Copy) {
dependsOn 'shadowJar'
into "${releaseAssetsDir}"
from("${buildDir}/libs") {
include "fcli.jar"
}
}
rootProject.dist.dependsOn(distLocal)
}

0 comments on commit 761832c

Please sign in to comment.