From 42fc464b3d0a85bbc3a8622adb737cfeb21c32f1 Mon Sep 17 00:00:00 2001 From: Dvegrod Date: Sun, 1 Sep 2024 21:19:14 +0200 Subject: [PATCH] Doc test --- docs/src/index.MD | 18 +++++++++---- src/PerfTest.jl | 29 ++++++++++++++++++-- src/auxiliar.jl | 68 +++++++++++++++++++++++++++++++++++++---------- src/config.jl | 6 +++-- src/structs.jl | 30 +++++++++++++++++---- 5 files changed, 123 insertions(+), 28 deletions(-) diff --git a/docs/src/index.MD b/docs/src/index.MD index d193fa6..a381150 100644 --- a/docs/src/index.MD +++ b/docs/src/index.MD @@ -1,13 +1,21 @@ # [PerfTest.jl] [![Star on GitHub](https://img.shields.io/github/stars/JuliaPerf/PerfTest.jl.svg)](https://github.com/JuliaPerf/PerfTest.jl/stargazers) -The package `PerfTest` provides ... +The package `PerfTest` provides the user with a performance regression unit testing framework. This framework consists of a collection of macros used to declaratively define a performance test suite. Scripts with said macros can then be transformed into performance suites using the `transform` method. This package is focused on providing an easy and fast way to develop performance suites, with additional features to customise them following the demands of the use case. ## Dependencies -`PerfTest` relies on [StaticArrays.jl], [Adapt.jl] and the Julia GPU packages [CUDA.jl] and [AMDGPU.jl]. +`PerfTest` relies on: + - [MacroTools] + - [JLD2] + - [MPI] + - [STREAMBenchmark] + - [GFlops] + - [UnicodePlots] + - [Test] + - [Suppressor] ## Contributors This project has been developed as a Master's thesis by Daniel Sergio Vega Rodriguez. Thus, the contributors to this project have been so far: - Daniel Sergio Vega Rodriguez ([@Dvegrod](https://github.com/Dvegrod)), Università della Svizzera italiana (USI): developer -- Dr. Samuel Omlin ([@omlins](https://github.com/omlins)), Swiss National Supercomputing Centre (CSCS), ETH Zurich: original idea and Master's thesis superviser -- Prof. Olaf Schenk, Università della Svizzera italiana (USI): Master's thesis superviser -- Dr. Pasadakis Dimosthenis, Università della Svizzera italiana (USI): additional adviser \ No newline at end of file +- Dr. Samuel Omlin ([@omlins](https://github.com/omlins)), Swiss National Supercomputing Centre (CSCS), ETH Zurich: original idea and Master's thesis supervisor +- Prof. Olaf Schenk, Università della Svizzera italiana (USI): Master's thesis supervisor +- Dr. Pasadakis Dimosthenis, Università della Svizzera italiana (USI): additional advisor diff --git a/src/PerfTest.jl b/src/PerfTest.jl index 7058dab..10fc5b3 100644 --- a/src/PerfTest.jl +++ b/src/PerfTest.jl @@ -5,7 +5,6 @@ __precompile__(false) # Temporary workaround to avoid error export @perftest, @on_perftest_exec, @on_perftest_ignore, @perftest_config, @define_eff_memory_throughput, @define_metric, @roofline -# Possibly redundant using MacroTools include("structs.jl") include("auxiliar.jl") @@ -70,6 +69,15 @@ rules = ASTRule[testset_macro_rule, # Main transform routine +""" +This method builds what is known as a rule set. Which is a function that will evaluate if an expression triggers a rule in a set and if that is the case apply the rule modifier. See the ASTRule documentation for more information. + +WARNING: the rule set will apply the FIRST rule that matches with the expression, therefore other matches will be ignored + +# Arguments + - `context` the context structure of the tree run, it will be ocassinally used by some rules on the set. + - `rules` the collection of rules that will belong to the resulting set. +""" function ruleSet(context::Context, rules :: Vector{ASTRule}) function _ruleSet(x) for rule in rules @@ -84,13 +92,28 @@ function ruleSet(context::Context, rules :: Vector{ASTRule}) end -function _treeRun(input_expr :: Expr, context :: Context, args...) +""" +This method gets a input julia expression, and a context register and executes a transformation of the input that converts a recipe script (input) into a fully-fledged testing suite (return value). + +# Arguments + - `input_expr` the recipe/source expression. (internally, a.k.a source code space) + - `context` a register that will store information useful for the transformation over its run over the AST of the input + +""" +function _treeRun(input_expr::Expr, context::Context, args...) return MacroTools.prewalk(ruleSet(context, rules), input_expr) end +""" +This method implements the transformation that converts a recipe script into a fully-fledged testing suite. +The function will return a Julia expression with the resulting performance testing suite. This can be then executed or saved in a file for later usage. + +# Arguments + - `path` the path of the script to be transformed. +""" function treeRun(path :: AbstractString) # Load original @@ -120,5 +143,7 @@ function treeRun(path :: AbstractString) return MacroTools.prettify(module_full) end +transform = treeRun + end # module perftest diff --git a/src/auxiliar.jl b/src/auxiliar.jl index 62c6ef1..e6470eb 100644 --- a/src/auxiliar.jl +++ b/src/auxiliar.jl @@ -1,23 +1,36 @@ using MacroTools: ismatch -# Function that generates a test name if needed +""" + Function that generates a test name if needed, it is used to name + test targets to distinguish them if several go in the same testset. +""" function genTestName!(state::Context) v = (last(state.depth).depth_test_count += 1) return "Test $v" end + +""" + Function used to register a new test set in the hierarchy record of the context, where `name` is the name of the test set. +""" function testsetUpdate!(state::Context, name::String) push!(state.depth, ASTWalkDepthRecord(name)) end -### EXPRESSION LOADER -function loadFileAsExpr(path ::AbstractString) +""" + Utility to get an expression from a Julia file stored at `path` +""" +function loadFileAsExpr(path::AbstractString) file = open(path, "r") str = read(file, String) return Meta.parse("begin $str end") end -### EXPRESSION PRINTER +""" + Utility to save an expression (`expr`) to a Julia file stored at `path` + + Requires a :toplevel symbol to be the head of the expression. +""" function saveExprAsFile(expr::Expr, path = "out.jl" :: AbstractString) #Get the module @@ -31,7 +44,10 @@ function saveExprAsFile(expr::Expr, path = "out.jl" :: AbstractString) end -## Pops expr block or quote and returns array of nested expressions +""" +Pops `expr` which has a head that is :block or :quote and returns array of nested expressions which are the arguments of such head. + +""" function removeBlock(expr::Expr)::Vector result = [] @@ -47,7 +63,9 @@ function removeBlock(expr::Expr)::Vector end -### Useful to move expressions to the toplevel +""" +This function is useful to move expressions to the toplevel when they are enclosed inside a block +""" function unblockAndConcat(exprs::Vector{Expr})::Expr result = Expr(:toplevel) @@ -62,10 +80,11 @@ function unblockAndConcat(exprs::Vector{Expr})::Expr return result end - -### Useful to correct operations limited by the tree walking -# Will remove quote blocks inside the main block without recursion and push -# their expressions into the main block +""" + Useful to correct operations limited by the tree walking + Will remove quote blocks inside the main block without recursion and push + their expressions into the main block +""" function popQuoteBlocks(expr::Expr)::Expr result = [] @@ -87,7 +106,20 @@ function popQuoteBlocks(expr::Expr)::Expr end end +""" +This method interpolates the `inside_expr` into `outside_expr` anywhere it finds the token `substitution_token`, which is a symbol. The `outside_expr` has to be a block or a quote block. It has the particularity that it will remove block heads from the `inside_expr` and add the nested elements onto the location where the token it. + +# Example: + +outside_expr = :(:A; 4) + +inside_expr = :(begin 2;3 end) + +substitution_token = :A +returns = :(2;3;4) + +""" function flattenedInterpolation(outside_expr::Expr, inside_expr::Expr, substitution_token::Symbol)::Expr @@ -176,7 +208,9 @@ function metaGet(expr_array :: AbstractVector, sym :: Symbol) return Nothing end - +""" + +""" function metaGetString(expr_array::AbstractVector) for expr in expr_array @@ -196,7 +230,9 @@ macro inRange(min, max, value) return :($min < $value < $max) end - +""" + From a string, it will divide it by lines and retrieve the ones that match the regular expression provided. +""" function grepOutput(output :: String, regex_or_string :: Union{Regex, String}):: Vector{SubString{String}} lines = split(output, '\n') @@ -206,7 +242,9 @@ function grepOutput(output :: String, regex_or_string :: Union{Regex, String}):: return cleaned_lines end -"""WARNING, SUPPORTS ONLY 1 NUMBER PER LINE""" +""" +From a string (`field`), it will parse the first number it finds as a Float +""" function getNumber(field :: String)::Float64 clean = replace(field, r"[^0-9.]" => "") @@ -214,7 +252,9 @@ function getNumber(field :: String)::Float64 return parse(Float64, clean) end -## Abbreviation gets from the first match +""" + Given a string `output`, it will retrieve the first number in the first line that contains the string `string`. +""" function grepOutputXGetNumber(output :: String, string ::String)::Float64 return getNumber(String(grepOutput(output, string)[1])) diff --git a/src/config.jl b/src/config.jl index f0df245..d44e05b 100644 --- a/src/config.jl +++ b/src/config.jl @@ -3,7 +3,9 @@ using MacroTools # CONFIG STRUCTURE DEFINITION # FOR DEFAULTS SEE BELOW COMMENT "DEFAULTCONFIG": - +""" + TEST +""" @kwdef mutable struct Struct_Regression enabled::Bool @@ -53,7 +55,7 @@ end # DEFAULTCONFIG -""" Where the regression tests are saved """ +# Where the regression tests are saved save_folder = ".perftests" save_test_results = true diff --git a/src/structs.jl b/src/structs.jl index 5fa058d..ee5bd73 100644 --- a/src/structs.jl +++ b/src/structs.jl @@ -2,12 +2,17 @@ using BenchmarkTools OPTIONAL_Float = Union{Nothing, Float64} - +""" + Tolerance interval structure. Used to save intervals around a threshold during test comparisons. +""" @kwdef mutable struct Struct_Tolerance max_percentage::OPTIONAL_Float = nothing min_percentage::OPTIONAL_Float = nothing end +""" + This structure is used to record a test set frame during a AST walk. See `ASTWalkDepthRecord` for more info. +""" mutable struct DepthRecord depth_name::String depth_flag::Bool @@ -15,6 +20,8 @@ mutable struct DepthRecord DepthRecord(name) = new(name, false) end +""" + This structure is used to record a test set hierarchy during a AST walk. In any specific point of the walk the array will TODO""" mutable struct ASTWalkDepthRecord depth_name::Union{String,Expr} depth_test_count::Int @@ -32,7 +39,10 @@ struct FloatRange end """ - Saves data needed during one specific execution of the test generation process. +Saves flags needed during the execution of the AST walk. It holds if: + - The walk is on an expression that is a test target + - The walk is on an expression that is inside a config macro + - Several flags that affect the roofline methodology """ mutable struct EnvironmentFlags inside_target::Bool @@ -46,6 +56,12 @@ mutable struct EnvironmentFlags end +""" +Saves flags needed during the execution of the AST walk. It holds if: + - The walk is on an expression that is a test target + - The walk is on an expression that is inside a config macro + - Several flags that affect the roofline methodology +""" @kwdef struct CustomMetric name::AbstractString units::AbstractString @@ -60,7 +76,8 @@ end end """ - Saves important state information when going through the AST of an expression. +In order to perform with the test suite generation, the AST walk needs to keep a context register to integrate features that rely on the scope hierarchy. + """ mutable struct Context # To register the current testset tree depth @@ -90,8 +107,11 @@ end """ - Used by the tree traverser to check for expressions that match "condition", - if they do then "modifier" will be applied to the expression. +Used by the AST walker to check for expressions that match `condition`, +if they do then `modifier` will be applied to the expression. + +This is the basic building block of the code transformer, a set of these rules compounds to all the needed manipulations to create the testing suite. + """ struct ASTRule condition::Function