Skip to content

Commit

Permalink
Merge pull request #293 from ftsrg/portfolio-ci-update
Browse files Browse the repository at this point in the history
Portfolio CI update
  • Loading branch information
leventeBajczi authored Aug 3, 2024
2 parents 4177646 + d6c5170 commit 60a9e48
Show file tree
Hide file tree
Showing 10 changed files with 250 additions and 30 deletions.
24 changes: 14 additions & 10 deletions .github/actions/benchexec-report/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,11 @@ runs:
cd artifacts
EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64)
echo "Message<<$EOF" >> $GITHUB_OUTPUT
header1=$(printf '|task set|'; for f in BenchexecResults*; do echo "${f##*-}"; done | sort -u | xargs printf '%s|')
header2=$(printf '|--|'; for ff in $(for f in BenchexecResults*; do echo "${f##*-}"; done | sort -u); do printf -- '--|'; done)
echo "$header1" >> $GITHUB_OUTPUT
printf "$header2" >> $GITHUB_OUTPUT
lasttask=""
for i in *
do
if (ls $i/*xml.bz2 >/dev/null 2>/dev/null)
Expand All @@ -36,21 +41,20 @@ runs:
emoji=":white_check_mark:"
[ $correct -eq 0 ] && emoji=":question:"
[ $incorrect -eq 0 ] || emoji=":exclamation:"
echo "<details><summary> $emoji ${i#BenchexecResults-} ($correct / $incorrect / $all)</summary>" >> $GITHUB_OUTPUT
echo >> $GITHUB_OUTPUT
echo '`table-generator`'" output: [HTML](https://theta.mit.bme.hu/benchmark-results/${{ github.head_ref }}/$i/$(ls *.html))/[CSV](https://theta.mit.bme.hu/benchmark-results/${{ github.head_ref }}/$i/$(ls *.csv))" >> $GITHUB_OUTPUT
echo >> $GITHUB_OUTPUT
echo '```' >> $GITHUB_OUTPUT
cat *.txt >> $GITHUB_OUTPUT
echo '```' >> $GITHUB_OUTPUT
echo "</details>" >> $GITHUB_OUTPUT
echo >> $GITHUB_OUTPUT
echo >> $GITHUB_OUTPUT
filename=${i#BenchexecResults-}
portfolio=${filename##*-}
task=${filename%-$portfolio}
if [ "$task" != "$lasttask" ]
then printf "\n| $task | " >> $GITHUB_OUTPUT
fi
lasttask="$task"
printf " $emoji ($correct / $incorrect / $all) [HTML](https://theta.mit.bme.hu/benchmark-results/${{ github.head_ref }}/$i/$(ls *.html))/[CSV](https://theta.mit.bme.hu/benchmark-results/${{ github.head_ref }}/$i/$(ls *.csv)) | " >> $GITHUB_OUTPUT
popd
else
rm -rf $i
fi
done
echo >> $GITHUB_OUTPUT
echo "$EOF" >> $GITHUB_OUTPUT
- name: Upload results
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
Expand Down
7 changes: 5 additions & 2 deletions .github/actions/benchexec-test/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ inputs:
required: true
test_number:
required: true
portfolio:
required: true
runs:
using: "composite"
steps:
Expand Down Expand Up @@ -46,6 +48,7 @@ runs:
shell: bash
run: |
unzip release/Theta.zip
sed -i 's/ --input \$1/ --portfolio ${{ inputs.portfolio }} --input \$1/g' Theta/theta-start.sh
- name: Cut setfile if too long
id: setfile
shell: bash
Expand All @@ -59,10 +62,10 @@ runs:
shell: bash
if: steps.setfile.outputs.length != '0'
run: |
benchexec xml/theta.xml --no-container --tool-directory Theta -t ${{ inputs.task }}
benchexec xml/theta.xml -n 2 --no-container --tool-directory Theta -t ${{ inputs.task }}
- name: Upload results
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
if: steps.setfile.outputs.length != '0'
with:
name: BenchexecResults-${{ inputs.task }}
name: BenchexecResults-${{ inputs.task }}-${{ inputs.portfolio }}
path: results
3 changes: 1 addition & 2 deletions .github/actions/benchexec-test/theta.xml
Original file line number Diff line number Diff line change
@@ -1,14 +1,13 @@
<?xml version="1.0"?>
<!DOCTYPE benchmark PUBLIC "+//IDN sosy-lab.org//DTD BenchExec benchmark 1.9//EN" "https://www.sosy-lab.org/benchexec/benchmark-2.3.dtd">
<benchmark tool="theta" timelimit="120 s" hardtimelimit="130 s">
<benchmark tool="theta" timelimit="20 s" hardtimelimit="30 s">

<resultfiles>**/witness.*</resultfiles>

<option name="--disable-xcfa-serialization"/>
<option name="--disable-c-serialization"/>
<option name="--disable-arg-generation"/>
<option name="--backend">PORTFOLIO</option>
<option name="--portfolio">COMPLEX</option>
<option name="--loglevel">INFO</option>

<rundefinition name="SV-COMP24_unreach-call">
Expand Down
5 changes: 4 additions & 1 deletion .github/actions/build-archive/necessary-solvers.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
cvc5:1.0.8
mathsat:5.6.10
z3:4.12.2
princess:2023-06-19
z3:4.13.0
princess:2023-06-19
eldarica:2.1
golem:0.5.0
14 changes: 7 additions & 7 deletions .github/actions/cache-build/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,15 @@ inputs:
runs:
using: "composite"
steps:
- uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8
if: runner.os == 'Linux'
id: cache
with:
path: .
key: ${{ runner.os }}-${{github.sha}}
# - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8
# if: runner.os == 'Linux'
# id: cache
# with:
# path: .
# key: ${{ runner.os }}-${{github.sha}}

- name: build gradle
uses: gradle/gradle-build-action@40b6781dcdec2762ad36556682ac74e31030cfe2 # v2.5.1
with:
arguments: ${{ inputs.arguments }} --info --stacktrace --max-workers 2 --no-daemon
arguments: ${{ inputs.arguments }} --info --stacktrace # --max-workers 2 --no-daemon

8 changes: 5 additions & 3 deletions .github/workflows/linux-build-test-deploy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ jobs:
strategy:
matrix:
task: [ReachSafety-Arrays, ReachSafety-BitVectors, ReachSafety-ControlFlow, ReachSafety-ECA, ReachSafety-Floats, ReachSafety-Heap, ReachSafety-Loops, ReachSafety-ProductLines, ReachSafety-Recursive, ReachSafety-Sequentialized, ReachSafety-XCSP, ReachSafety-Combinations, ReachSafety-Hardware, ConcurrencySafety-Main, NoDataRace-Main, ConcurrencySafety-NoOverflows, ConcurrencySafety-MemSafety]
portfolio: [CEGAR, BOUNDED, HORN]
runs-on: ubuntu-latest
needs: create-archives
steps:
Expand All @@ -59,7 +60,8 @@ jobs:
uses: ./.github/actions/benchexec-test
with:
task: ${{ matrix.task }}
test_number: 5
portfolio: ${{ matrix.portfolio }}
test_number: 50

collect-results:
needs: test-benchexec
Expand All @@ -80,12 +82,12 @@ jobs:
uses: ./.github/actions/build-archive
with:
name: "Theta"
inputflag: "--portfolio COMPLEX"
inputflag: "--portfolio STABLE"
- name: Create emergentheta.zip
uses: ./.github/actions/build-archive
with:
name: "EmergenTheta"
inputflag: "--algorithm IMC_THEN_KIND"
inputflag: "--algorithm EMERGENT"


javadoc:
Expand Down
2 changes: 1 addition & 1 deletion build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ buildscript {

allprojects {
group = "hu.bme.mit.theta"
version = "6.3.3"
version = "6.3.4"

apply(from = rootDir.resolve("gradle/shared-with-buildSrc/mirrors.gradle.kts"))
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,7 @@ import hu.bme.mit.theta.xcfa.analysis.XcfaPrec
import hu.bme.mit.theta.xcfa.analysis.XcfaState
import hu.bme.mit.theta.xcfa.cli.params.PortfolioConfig
import hu.bme.mit.theta.xcfa.cli.params.XcfaConfig
import hu.bme.mit.theta.xcfa.cli.portfolio.STM
import hu.bme.mit.theta.xcfa.cli.portfolio.boundedPortfolio
import hu.bme.mit.theta.xcfa.cli.portfolio.complexPortfolio23
import hu.bme.mit.theta.xcfa.cli.portfolio.complexPortfolio24
import hu.bme.mit.theta.xcfa.cli.portfolio.*
import hu.bme.mit.theta.xcfa.model.XCFA
import java.io.File
import java.io.FileReader
Expand All @@ -51,11 +48,20 @@ fun getPortfolioChecker(xcfa: XCFA, mcm: MCM, config: XcfaConfig<*, *>,
val portfolioName = (config.backendConfig.specConfig as PortfolioConfig).portfolio

val portfolioStm = when (portfolioName) {
"STABLE",
"CEGAR",
"COMPLEX",
"COMPLEX24" -> complexPortfolio24(xcfa, mcm, parseContext, config, logger, uniqueLogger)

"COMPLEX23" -> complexPortfolio23(xcfa, mcm, parseContext, config, logger, uniqueLogger)

"EMERGENT",
"BOUNDED" -> boundedPortfolio(xcfa, mcm, parseContext, config, logger, uniqueLogger)

"TESTING",
"CHC",
"HORN" -> hornPortfolio(xcfa, mcm, parseContext, config, logger, uniqueLogger)

else -> {
if (File(portfolioName).exists()) {
val kotlinEngine: ScriptEngine = ScriptEngineManager().getEngineByExtension("kts")
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
/*
* Copyright 2024 Budapest University of Technology and Economics
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package hu.bme.mit.theta.xcfa.cli.portfolio

import hu.bme.mit.theta.common.logging.Logger
import hu.bme.mit.theta.frontend.ParseContext
import hu.bme.mit.theta.frontend.transformation.ArchitectureConfig
import hu.bme.mit.theta.graphsolver.patterns.constraints.MCM
import hu.bme.mit.theta.xcfa.analysis.isInlined
import hu.bme.mit.theta.xcfa.cli.params.*
import hu.bme.mit.theta.xcfa.cli.runConfig
import hu.bme.mit.theta.xcfa.model.XCFA
import hu.bme.mit.theta.xcfa.passes.LbePass
import hu.bme.mit.theta.xcfa.passes.LoopUnrollPass
import java.nio.file.Paths

fun hornPortfolio(
xcfa: XCFA,
mcm: MCM,
parseContext: ParseContext,
portfolioConfig: XcfaConfig<*, *>,
logger: Logger,
uniqueLogger: Logger): STM {

val checker = { config: XcfaConfig<*, *> -> runConfig(config, logger, uniqueLogger, true) }

var baseConfig = XcfaConfig(
inputConfig = InputConfig(
input = null,
xcfaWCtx = Triple(xcfa, mcm, parseContext),
propertyFile = null,
property = portfolioConfig.inputConfig.property),
frontendConfig = FrontendConfig(
lbeLevel = LbePass.level,
loopUnroll = LoopUnrollPass.UNROLL_LIMIT,
inputType = InputType.C,
specConfig = CFrontendConfig(arithmetic = ArchitectureConfig.ArithmeticType.efficient)),
backendConfig = BackendConfig(
backend = Backend.CHC,
solverHome = portfolioConfig.backendConfig.solverHome,
timeoutMs = 0,
specConfig = HornConfig(
solver = "z3:4.13.0",
validateSolver = false
)),
outputConfig = OutputConfig(
versionInfo = false,
resultFolder = Paths.get("./").toFile(), // cwd
cOutputConfig = COutputConfig(disable = true),
witnessConfig = WitnessConfig(disable = false, concretizerSolver = "Z3", validateConcretizerSolver = false),
argConfig = ArgConfig(disable = true),
enableOutput = portfolioConfig.outputConfig.enableOutput,
),
debugConfig = portfolioConfig.debugConfig
)

if (parseContext.multiThreading) {
throw UnsupportedOperationException("Multithreading for horn checkers not supported")
}

if (!xcfa.isInlined) {
throw UnsupportedOperationException("Recursive XCFA for horn checkers not supported")
}

val timeoutOrNotSolvableError = ExceptionTrigger(
fallthroughExceptions = setOf(
ErrorCodeException(ExitCodes.SOLVER_ERROR.code),
ErrorCodeException(ExitCodes.SERVER_ERROR.code),
),
label = "TimeoutOrNotSolvableError"
)

val timeoutOrSolverError = ExceptionTrigger(
fallthroughExceptions = setOf(
ErrorCodeException(ExitCodes.SERVER_ERROR.code),
),
label = "TimeoutOrSolverError"
)

val solverError = ExceptionTrigger(
ErrorCodeException(ExitCodes.SOLVER_ERROR.code),
label = "SolverError"
)

val anyError = ExceptionTrigger(label = "Anything")

fun XcfaConfig<*, HornConfig>.adaptConfig(
solver: String = "z3:4.13.0",
timeoutMs: Long = 0,
inProcess: Boolean = this.backendConfig.inProcess
): XcfaConfig<*, HornConfig> {
return copy(backendConfig = backendConfig.copy(
timeoutMs = timeoutMs,
inProcess = inProcess,
specConfig = backendConfig.specConfig!!.copy(
solver = solver,
)
))
}

fun getStm(inProcess: Boolean): STM {
val edges = LinkedHashSet<Edge>()
val configZ3 = ConfigNode("Z3-$inProcess",
baseConfig.adaptConfig(
inProcess = inProcess,
timeoutMs = 100_000
), checker)
val configEldarica = ConfigNode("Eldarica-$inProcess",
baseConfig.adaptConfig(
inProcess = inProcess,
solver = "eldarica:2.1",
timeoutMs = 500_000
), checker)
val configGolem = ConfigNode("Golem-$inProcess",
baseConfig.adaptConfig(
inProcess = inProcess,
solver = "golem:0.5.0",
timeoutMs = 300_000
), checker)

edges.add(Edge(configZ3, configEldarica, anyError))
edges.add(Edge(configEldarica, configGolem, anyError))

return STM(configZ3, edges)
}

logger.write(Logger.Level.RESULT, "Using CHC portfolio\n")

val inProcess = HierarchicalNode("InProcess", getStm(true))
val notInProcess = HierarchicalNode("NotInprocess", getStm(false))

val fallbackEdge = Edge(inProcess, notInProcess, ExceptionTrigger(label = "Anything"))

return if (portfolioConfig.debugConfig.debug) getStm(false) else STM(inProcess, setOf(fallbackEdge))
}
Loading

0 comments on commit 60a9e48

Please sign in to comment.