Doc test

Author: Dvegrod

docs/src/index.MD

@@ -1,13 +1,21 @@
 # [PerfTest.jl] [![Star on GitHub](https://img.shields.io/github/stars/JuliaPerf/PerfTest.jl.svg)](https://github.com/JuliaPerf/PerfTest.jl/stargazers)
-The package `PerfTest` provides ...
+The package `PerfTest` provides the user with a performance regression unit testing framework. This framework consists of a collection of macros used to declaratively define a performance test suite. Scripts with said macros can then be transformed into performance suites using the `transform` method. This package is focused on providing an easy and fast way to develop performance suites, with additional features to customise them following the demands of the use case.
 
 ## Dependencies
-`PerfTest` relies on [StaticArrays.jl], [Adapt.jl] and the Julia GPU packages [CUDA.jl] and [AMDGPU.jl].
+`PerfTest` relies on:
+ - [MacroTools]
+ - [JLD2]
+ - [MPI]
+ - [STREAMBenchmark]
+ - [GFlops]
+ - [UnicodePlots]
+ - [Test]
+ - [Suppressor]
 
 ## Contributors
 This project has been developed as a Master's thesis by Daniel Sergio Vega Rodriguez.
 Thus, the contributors to this project have been so far:
 - Daniel Sergio Vega Rodriguez ([@Dvegrod](https://github.com/Dvegrod)), Università della Svizzera italiana (USI): developer
-- Dr. Samuel Omlin ([@omlins](https://github.com/omlins)), Swiss National Supercomputing Centre (CSCS), ETH Zurich: original idea and Master's thesis superviser
-- Prof. Olaf Schenk, Università della Svizzera italiana (USI): Master's thesis superviser
-- Dr. Pasadakis Dimosthenis, Università della Svizzera italiana (USI): additional adviser
+- Dr. Samuel Omlin ([@omlins](https://github.com/omlins)), Swiss National Supercomputing Centre (CSCS), ETH Zurich: original idea and Master's thesis supervisor
+- Prof. Olaf Schenk, Università della Svizzera italiana (USI): Master's thesis supervisor
+- Dr. Pasadakis Dimosthenis, Università della Svizzera italiana (USI): additional advisor

src/PerfTest.jl

@@ -5,7 +5,6 @@ __precompile__(false) # Temporary workaround to avoid error
 export @perftest, @on_perftest_exec, @on_perftest_ignore, @perftest_config,
     @define_eff_memory_throughput, @define_metric, @roofline
 
-# Possibly redundant
 using MacroTools
 include("structs.jl")
 include("auxiliar.jl")
@@ -70,6 +69,15 @@ rules = ASTRule[testset_macro_rule,
 
 # Main transform routine
 
+"""
+This method builds what is known as a rule set. Which is a function that will evaluate if an expression triggers a rule in a set and if that is the case apply the rule modifier. See the ASTRule documentation for more information.
+
+WARNING: the rule set will apply the FIRST rule that matches with the expression, therefore other matches will be ignored
+
+# Arguments
+ - `context` the context structure of the tree run, it will be ocassinally used by some rules on the set.
+ - `rules` the collection of rules that will belong to the resulting set.
+"""
 function ruleSet(context::Context, rules :: Vector{ASTRule})
     function _ruleSet(x)
         for rule in rules
@@ -84,13 +92,28 @@ function ruleSet(context::Context, rules :: Vector{ASTRule})
 end
 
 
-function _treeRun(input_expr :: Expr, context :: Context, args...)
+"""
+This method gets a input julia expression, and a context register and executes a transformation of the input that converts a recipe script (input) into a fully-fledged testing suite (return value).
+
+# Arguments
+ - `input_expr` the recipe/source expression. (internally, a.k.a source code space)
+ - `context` a register that will store information useful for the transformation over its run over the AST of the input
+
+"""
+function _treeRun(input_expr::Expr, context::Context, args...)
 
     return MacroTools.prewalk(ruleSet(context, rules), input_expr)
 end
 
 
+"""
+This method implements the transformation that converts a recipe script into a fully-fledged testing suite.
+The function will return a Julia expression with the resulting performance testing suite. This can be then executed or saved in a file for later usage.
+
+# Arguments
+ - `path` the path of the script to be transformed.
 
+"""
 function treeRun(path :: AbstractString)
 
     # Load original
@@ -120,5 +143,7 @@ function treeRun(path :: AbstractString)
     return MacroTools.prettify(module_full)
 end
 
+transform = treeRun
+
 
 end # module perftest

src/auxiliar.jl

@@ -1,23 +1,36 @@
 using MacroTools: ismatch
 
-# Function that generates a test name if needed
+"""
+  Function that generates a test name if needed, it is used to name
+  test targets to distinguish them if several go in the same testset.
+"""
 function genTestName!(state::Context)
     v = (last(state.depth).depth_test_count += 1)
     return "Test $v"
 end
 
+
+"""
+  Function used to register a new test set in the hierarchy record of the context, where `name` is the name of the test set.
+"""
 function testsetUpdate!(state::Context, name::String)
     push!(state.depth, ASTWalkDepthRecord(name))
 end
 
-### EXPRESSION LOADER
-function loadFileAsExpr(path ::AbstractString)
+"""
+  Utility to get an expression from a Julia file stored at `path`
+"""
+function loadFileAsExpr(path::AbstractString)
     file = open(path, "r")
     str = read(file, String)
     return Meta.parse("begin $str end")
 end
 
-### EXPRESSION PRINTER
+"""
+  Utility to save an expression (`expr`) to a Julia file stored at `path`
+
+  Requires a :toplevel symbol to be the head of the expression.
+"""
 function saveExprAsFile(expr::Expr, path = "out.jl" :: AbstractString)
 
     #Get the module
@@ -31,7 +44,10 @@ function saveExprAsFile(expr::Expr, path = "out.jl" :: AbstractString)
 
 end
 
-## Pops expr block or quote and returns array of nested expressions
+"""
+Pops `expr` which has a head that is :block or :quote and returns array of nested expressions which are the arguments of such head.
+
+"""
 function removeBlock(expr::Expr)::Vector
     result = []
 
@@ -47,7 +63,9 @@ function removeBlock(expr::Expr)::Vector
 end
 
 
-### Useful to move expressions to the toplevel
+"""
+This function is useful to move expressions to the toplevel when they are enclosed inside a block
+"""
 function unblockAndConcat(exprs::Vector{Expr})::Expr
 
     result = Expr(:toplevel)
@@ -62,10 +80,11 @@ function unblockAndConcat(exprs::Vector{Expr})::Expr
     return result
 end
 
-
-### Useful to correct operations limited by the tree walking
-# Will remove quote blocks inside the main block without recursion and push
-# their expressions into the main block
+"""
+ Useful to correct operations limited by the tree walking
+ Will remove quote blocks inside the main block without recursion and push
+ their expressions into the main block
+"""
 function popQuoteBlocks(expr::Expr)::Expr
     result = []
 
@@ -87,7 +106,20 @@ function popQuoteBlocks(expr::Expr)::Expr
     end
 end
 
+"""
+This method interpolates the `inside_expr` into `outside_expr` anywhere it finds the token `substitution_token`, which is a symbol. The `outside_expr` has to be a block or a quote block. It has the particularity that it will remove block heads from the `inside_expr` and add the nested elements onto the location where the token it.
+
+# Example:
+
+outside_expr = :(:A; 4)
+
+inside_expr = :(begin 2;3 end)
+
+substitution_token = :A
 
+returns = :(2;3;4)
+
+"""
 function flattenedInterpolation(outside_expr::Expr,
     inside_expr::Expr,
     substitution_token::Symbol)::Expr
@@ -176,7 +208,9 @@ function metaGet(expr_array :: AbstractVector, sym :: Symbol)
     return Nothing
 end
 
-
+"""
+  
+"""
 function metaGetString(expr_array::AbstractVector)
 
     for expr in expr_array
@@ -196,7 +230,9 @@ macro inRange(min, max, value)
     return :($min < $value < $max)
 end
 
-
+"""
+  From a string, it will divide it by lines and retrieve the ones that match the regular expression provided.
+"""
 function grepOutput(output :: String, regex_or_string :: Union{Regex, String}):: Vector{SubString{String}}
     lines = split(output, '\n')
 
@@ -206,15 +242,19 @@ function grepOutput(output :: String, regex_or_string :: Union{Regex, String})::
     return cleaned_lines
 end
 
-"""WARNING, SUPPORTS ONLY 1 NUMBER PER LINE"""
+"""
+From a string (`field`), it will parse the first number it finds as a Float
+"""
 function getNumber(field :: String)::Float64
     clean = replace(field, r"[^0-9.]" => "")
 
 
     return parse(Float64, clean)
 end
 
-## Abbreviation gets from the first match
+"""
+  Given a string `output`, it will retrieve the first number in the first line that contains the string `string`.
+"""
 function grepOutputXGetNumber(output :: String, string ::String)::Float64
 
     return getNumber(String(grepOutput(output, string)[1]))

src/config.jl

@@ -3,7 +3,9 @@ using MacroTools
 
 # CONFIG STRUCTURE DEFINITION
 # FOR DEFAULTS SEE BELOW COMMENT "DEFAULTCONFIG":
-
+"""
+ TEST
+"""
 @kwdef mutable struct Struct_Regression
     enabled::Bool
 
@@ -53,7 +55,7 @@ end
 
 # DEFAULTCONFIG
 
-""" Where the regression tests are saved """
+# Where the regression tests are saved
 save_folder = ".perftests"
 
 save_test_results = true

src/structs.jl

@@ -2,19 +2,26 @@ using BenchmarkTools
 
 OPTIONAL_Float = Union{Nothing, Float64}
 
-
+"""
+  Tolerance interval structure. Used to save intervals around a threshold during test comparisons.
+"""
 @kwdef mutable struct Struct_Tolerance
     max_percentage::OPTIONAL_Float = nothing
     min_percentage::OPTIONAL_Float = nothing
 end
 
+"""
+  This structure is used to record a test set frame during a AST walk. See `ASTWalkDepthRecord` for more info.
+"""
 mutable struct DepthRecord
     depth_name::String
     depth_flag::Bool
 
     DepthRecord(name) = new(name, false)
 end
 
+"""
+  This structure is used to record a test set hierarchy during a AST walk. In any specific point of the walk the array will TODO"""
 mutable struct ASTWalkDepthRecord
     depth_name::Union{String,Expr}
     depth_test_count::Int
@@ -32,7 +39,10 @@ struct FloatRange
 end
 
 """
-  Saves data needed during one specific execution of the test generation process.
+Saves flags needed during the execution of the AST walk. It holds if:
+ - The walk is on an expression that is a test target
+ - The walk is on an expression that is inside a config macro
+ - Several flags that affect the roofline methodology
 """
 mutable struct EnvironmentFlags
     inside_target::Bool
@@ -46,6 +56,12 @@ mutable struct EnvironmentFlags
 end
 
 
+"""
+Saves flags needed during the execution of the AST walk. It holds if:
+ - The walk is on an expression that is a test target
+ - The walk is on an expression that is inside a config macro
+ - Several flags that affect the roofline methodology
+"""
 @kwdef struct CustomMetric
     name::AbstractString
     units::AbstractString
@@ -60,7 +76,8 @@ end
 end
 
 """
-  Saves important state information when going through the AST of an expression.
+In order to perform with the test suite generation, the AST walk needs to keep a context register to integrate features that rely on the scope hierarchy.
+
 """
 mutable struct Context
     # To register the current testset tree depth
@@ -90,8 +107,11 @@ end
 
 
 """
-  Used by the tree traverser to check for expressions that match "condition",
-  if they do then "modifier" will be applied to the expression.
+Used by the AST walker to check for expressions that match `condition`,
+if they do then `modifier` will be applied to the expression.
+
+This is the basic building block of the code transformer, a set of these rules compounds to all the needed manipulations to create the testing suite.
+
 """
 struct ASTRule
     condition::Function