diff --git a/docs/src/api.MD b/docs/src/api.MD index 67c3020..1b53812 100644 --- a/docs/src/api.MD +++ b/docs/src/api.MD @@ -12,24 +12,11 @@ help?> PerfTest ``` - - - - - - - - - - - - - - -## Convenience type aliases and constructors +## Types #### Index -* [`PerfTest.retvalExpressionParser`](@ref) -* [`PerfTest.metaGet`](@ref) +```@index +Order = [:type] +``` #### Documentation ```@autodocs diff --git a/src/benchmarking.jl b/src/benchmarking.jl index a76d720..d3452bd 100644 --- a/src/benchmarking.jl +++ b/src/benchmarking.jl @@ -3,7 +3,9 @@ using LinearAlgebra # Memory and CPU benchmarks used by different methodologies - +""" + This method is used to generate the code responsible for sampling the maximum memory bandwith in every resulting suite. +""" function setupMemoryBandwidthBenchmark()::Expr # TODO MPI extra behaviour #println("="^26 * "Maximum memory throughput calculation" * "="^26) @@ -49,6 +51,9 @@ function setupMemoryBandwidthBenchmark()::Expr end +""" + This method is used to generate the code responsible for sampling the maximum CPU FLOPS based on the avaiable threads in every resulting suite. +""" function setupCPUPeakFlopBenchmark()::Expr return mpi_enabled ? quote diff --git a/src/config.jl b/src/config.jl index d44e05b..1ffcb69 100644 --- a/src/config.jl +++ b/src/config.jl @@ -4,7 +4,15 @@ using MacroTools # CONFIG STRUCTURE DEFINITION # FOR DEFAULTS SEE BELOW COMMENT "DEFAULTCONFIG": """ - TEST +This struct holds the configuration of the basic metric regression methodology. + +`enabled` is used to enable or disable the methodology +`save_failed` will record historical measurements of failed tests if true +`general_regression_threshold` sets the torelance interval for the test comparison + +`regression_calculation` can be: + - :latest The reference will be the latest saved result + - :average The reference will be the average of all saved results """ @kwdef mutable struct Struct_Regression enabled::Bool @@ -13,20 +21,26 @@ using MacroTools general_regression_threshold::Struct_Tolerance - - """ - Can be: - - :latest The reference will be the latest saved result - - :average The reference will be the average of all saved results - """ regression_calculation::Symbol end +""" +This struct holds the configuration of the basic effective memory throughput methodology. + + - `enabled` is used to enable or disable the methodology + - `tolerance` defines the interval of ratios (eff.mem.through. / max. bandwidth) that make the test succeed. +""" @kwdef mutable struct Struct_Eff_Mem_Throughput enabled::Bool tolerance::Struct_Tolerance end +""" +This struct holds the configuration of the basic roofline methodology. + + - `enabled` is used to enable or disable the methodology + - `tolerance` defines the interval of ratios (eff.mem.through. / max. bandwidth) that make the test succeed. +""" @kwdef mutable struct Struct_Roofline_Config enabled::Bool @@ -37,6 +51,13 @@ end tolerance::Struct_Tolerance end + +""" +This struct can hold the configuration of any metric. + + - `enabled` is used to enable or disable the methodology + - `regression_threshold`, when comparing the measure with a reference, defines how far can the measurement be from the reference +""" @kwdef mutable struct Struct_Metric_Config enabled::Bool @@ -151,7 +172,10 @@ metrics = Struct_Metrics( # AST MODIFIERS # Perftest_config AST Manipulation -function perftestConfigEnter(expr :: Expr, context :: Context)::Expr +""" + Function to trigger the configuration mode on the context register +""" +function perftestConfigEnter(expr::Expr, context::Context)::Expr block = escCaptureGetblock(expr, Symbol("@perftest_config")) # TODO Enable environment flag @@ -160,10 +184,13 @@ function perftestConfigEnter(expr :: Expr, context :: Context)::Expr eval(block) return quote - nothing + nothing end end +""" + Function to deactivate the configuration mode on the context register +""" function perftestConfigExit(_ :: Expr, context :: Context)::Expr # TODO # Disable environment flag @@ -174,9 +201,6 @@ function perftestConfigExit(_ :: Expr, context :: Context)::Expr end end -function perftestConfigParseField(expr :: Expr, context::Context)::Expr - #TODO -end # CONFIG UTILS diff --git a/src/macros.jl b/src/macros.jl index 8f71145..25d4a92 100644 --- a/src/macros.jl +++ b/src/macros.jl @@ -144,6 +144,8 @@ Any formula block specified in this macro supports these symbols. :autoflop / mem end +The code block defines operational intensity, whilst the other arguments define how to measure and compare the actual performance with the roofline performance. If the actual to projected performance ratio goes below the target, the test fails. + """ macro roofline(opint_formula, cpu_peak=nothing, membw_peak=nothing) return :( diff --git a/src/metrics.jl b/src/metrics.jl index 0524e1f..89db356 100644 --- a/src/metrics.jl +++ b/src/metrics.jl @@ -3,25 +3,42 @@ sym_set = Set([:(:median_time), :(:minimum_time)]) -function customMetricExpressionParser(expr :: Expr) :: Expr +""" +This is one of the parser functions that expand any formula block for metric definition. +This function will parse all primitive metric symbols with the structure where the corresponding value of the metric is. +""" +function customMetricExpressionParser(expr::Expr)::Expr return MacroTools.postwalk(x -> (x in sym_set ? :(metric_results[$x].value) : x), expr) end +""" +This is one of the parser functions that expand any formula block for metric definition. +This function will parse all primitive metric symbols with the structure where the corresponding reference value for the metric is. +""" function customMetricReferenceExpressionParser(expr::Expr)::Expr return MacroTools.postwalk(x -> (x in sym_set ? :(metric_references[$x]) : x), expr) end """ - Checks for the return symbol +This is one of the parser functions that expand any formula block for metric definition. +This function will parse the appropiate symbol and substitute it by the return value of the test target execution. """ function retvalExpressionParser(expr::Expr)::Expr return MacroTools.postwalk(x -> (x == :(:return) ? :(PerfTest.by_index(export_tree, depth)[:ret_value]) : x), expr) end +""" +This is one of the parser functions that expand any formula block for metric definition. +This function will parse the `:autoflop` symbol and substitute it with the flop count of the test target +""" function autoflopExpressionParser(expr::Expr)::Expr - return MacroTools.postwalk(x -> ((@show x;x == :(:autoflop)) ? :(PerfTest.by_index(export_tree, depth)[:autoflop]) : x), expr) + return MacroTools.postwalk(x -> ((@show x; x == :(:autoflop)) ? :(PerfTest.by_index(export_tree, depth)[:autoflop]) : x), expr) end +""" +This is one of the parser functions that expand any formula block for metric definition. +This function will parse the `:printed_output` symbol and substitute it with the standard output of the test target execution +""" function printedOutputExpressionParser(expr::Expr)::Expr return MacroTools.postwalk(x -> (x == :(:printed_output) ? :(PerfTest.by_index(export_tree, depth)[:printed_output]) : x), expr) end @@ -30,10 +47,17 @@ function printedOutputAbbreviationExpressionParser(expr::Expr)::Expr return MacroTools.postwalk(x -> (x == :(:out) ? :(PerfTest.grepOutputXGetNumber(PerfTest.by_index(export_tree, depth)[:printed_output])) : x), expr) end +""" +This is one of the parser functions that expand any formula block for metric definition. +This function will parse the `:iterator` symbol and substitute it with the current value of the innermost test set loop of the current test target execution +""" function iteratorExpressionParser(expr::Expr)::Expr return MacroTools.postwalk(x -> (x == :(:iterator) ? :(PerfTest.by_index(export_tree, depth)[:iterator]) : x), expr) end +""" +This function combines a collection of rules to turn a formula block into a functioning expression to calculate any metric defined by said formula +""" function fullParsingSuite(expr::Expr)::Expr # Fill primitives t = customMetricExpressionParser(expr) @@ -50,6 +74,9 @@ function fullParsingSuite(expr::Expr)::Expr return t end +""" +This function is called to register a custom metric, it will parse the arguments of the definition macro and add the metric to the context to be later used in test targets on the same scope. +""" function onCustomMetricDefinition(expr ::Expr, context :: Context, flags::Set{Symbol}) :: Expr # Special case @@ -117,13 +144,18 @@ function onCustomMetricDefinition(expr ::Expr, context :: Context, flags::Set{Sy ) end +""" +This function is used to register a special custom metric, which is the effective memory throughput calculation, and is registered in the same way as any other but with a special flag that the EMT metholodogy will use to get and use the metric. +""" function onMemoryThroughputDefinition(expr::Expr, context::Context)::Expr # Communicate that this can be used with the mem throughput methodology flags = Set{Symbol}([:mem_throughput]) return onCustomMetricDefinition(expr, context, flags) end -# TODO flops memory +""" +This function generates the code that make primitive metrics values available to all metodologies and custom metrics. +""" function buildPrimitiveMetrics() :: Expr return quote metric_results = Dict{Symbol, PerfTest.Metric_Result}() @@ -144,9 +176,14 @@ function buildPrimitiveMetrics() :: Expr end end -# WARNING Predefined symbols needed before this quote -# reference_value -# metric_results +""" +This function is used to generate the code that evaluates if the median time of execution of a target is within a specified reference. + +# WARNING +Predefined symbols needed before this code is added to the generated space: + - `reference_value` + - `metric_results` +""" function checkMedianTime(thresholds ::Struct_Tolerance)::Expr return metrics.median_time.enabled ? quote @@ -179,9 +216,14 @@ function checkMedianTime(thresholds ::Struct_Tolerance)::Expr end : quote nothing end end -# WARNING Predefined symbols needed before this quote -# reference_value -# metric_results +""" +This function is used to generate the code that evaluates if the minimum time of execution of a target is within a specified reference. + +# WARNING +Predefined symbols needed before this code is added to the generated space: + - `reference_value` + - `metric_results` +""" function checkMinTime(thresholds::Struct_Tolerance)::Expr return metrics.median_time.enabled ? quote @@ -215,10 +257,14 @@ function checkMinTime(thresholds::Struct_Tolerance)::Expr end -# WARNING Predefined symbols needed before this quote -# metric_results -# msym -# metric (see loop on function below this one) +""" +This function is used to generate the code that evaluates if a custom metric result f a target is within a specified reference. + +# WARNING +Predefined symbols needed before this code is added to the generated space: + - `reference_value` + - `metric_results` +""" function checkCustomMetric(metric :: CustomMetric)::Expr if :aux in metric.flags @@ -263,8 +309,14 @@ function checkCustomMetric(metric :: CustomMetric)::Expr end -# WARNING Predefined symbols needed before this quote -# local_customs, global_customs +""" +This function is used to generate the code that evaluates if a custom metric result f a target is within a specified reference. + +# WARNING +Predefined symbols needed before this code is added to the generated space: + - `reference_value` + - `metric_results` +""" function checkCustomMetrics(context::Context)::Expr result = :( begin end diff --git a/src/perftest/data_handling.jl b/src/perftest/data_handling.jl index 61e126a..b53adee 100644 --- a/src/perftest/data_handling.jl +++ b/src/perftest/data_handling.jl @@ -2,16 +2,25 @@ using JLD2: StringDatatype using JLD2 -function openDataFile(path :: AbstractString) :: Perftest_Datafile_Root +""" +This method is used to get historical data of a performance test suite from a save file located in `path`. +""" +function openDataFile(path::AbstractString)::Perftest_Datafile_Root return JLD2.load(path)["contents"] end +""" +This method is used to save historical data of a performance test suite to a save file located in `path`. +""" function saveDataFile(path :: AbstractString, contents:: Perftest_Datafile_Root) return jldsave(path; contents) end -## DepthRecord -# Auxiliar by index Dict access function +""" +This method expects a hierarchy tree (`dict`) in the form of nested dictionaries and a vector of dictionary keys `idx`. The function will recursively apply the keys to get to a final element. + +It is usually put to work with the `DepthRecord` struct. +""" function by_index(dict::Union{Dict,BenchmarkGroup}, idx::Vector{DepthRecord}) e = dict for idx_elem in idx @@ -22,7 +31,22 @@ function by_index(dict::Union{Dict,BenchmarkGroup}, idx::Vector{DepthRecord}) end -## To extract values from methology results +""" +This method will return a flattened array of all of the results for all the methodologies exercised in the provided dictionary. + +# Example: + "Test Set 1" + -> "Test 1" + -> Methodology A result + -> Methodology B result + "Test Set 2" + -> "Test 1" + -> Methodology A result +Returns: + M. A result (Test 1) + M. B result (Test 1) + M. A result (Test 2) +""" function extractMethodologyResultArray(methodology_dict :: Dict, methodology :: Symbol) :: Vector{Methodology_Result} retval = [] for (key, elem) in methodology_dict @@ -35,6 +59,9 @@ function extractMethodologyResultArray(methodology_dict :: Dict, methodology :: return retval end +""" +Given a series of methodology results, the the raw values of all the metrics contained in the methodology results. +""" function getMetricValue(mresult_vector :: Vector{Methodology_Result}, name :: String) retval = [] @@ -54,6 +81,23 @@ function getMetricValue(mresult_vector :: Vector{Methodology_Result}, name :: St return retval end +""" +This method will return a flattened array of the whole test result hierarchy. + +# Example +# Example: + "Test Set 1" + -> "Test 1" + -> Methodology A result + -> Methodology B result + "Test Set 2" + -> "Test 1" + -> Methodology A result +Returns: + "Test Set 1 -> Test 1 -> Methodology A" + "Test Set 1 -> Test 1 -> Methodology B" + "Test Set 2 -> Test 1 -> Methodology A" +""" function extractNamesResultArray(methodology_dict::Dict, methodology :: Symbol)::Vector{String} retval = String[] for (key, elem) in methodology_dict diff --git a/src/perftest/structs.jl b/src/perftest/structs.jl index 1b70fbe..527ec32 100644 --- a/src/perftest/structs.jl +++ b/src/perftest/structs.jl @@ -2,6 +2,10 @@ using BenchmarkTools StrOrSym = Union{String,Symbol} +""" +This struct is used in the test suite to save a metric measurement, +therefore its saves the metric `name`, its `units` space and its `value`. +""" @kwdef struct Metric_Result{N} name::AbstractString # Used to identify the metric in some situations @@ -10,7 +14,13 @@ StrOrSym = Union{String,Symbol} end -# This is used when the struct below has not been built yet +""" +This struct is used in the test suite to save a metric reference, +a reference is meant to be later compared with a result, its combination gives the `Metric_Constraint` struct. +It holds: + - A `reference` value. + - `low_is_bad` registers if in this metric lower values are less desired than higher ones, or the opposite (e.g. time vs FLOP/s). +""" @kwdef struct Metric_Reference{N} reference::N low_is_bad::Bool @@ -18,6 +28,9 @@ end custom_elements::Vector{Symbol} = Symbol[] end +""" +This struct is used in the test suite to save a metric test result and its associated data, it saves the reference used and the toreance intervals in absolute and percentual values, also it shows if the test succeded and some additional variables for data printing +""" @kwdef struct Metric_Constraint{N} reference::N threshold_min::N @@ -27,14 +40,19 @@ end low_is_bad::Bool succeeded::Bool # Additional metric X methodology data - custom_plotting :: Vector{Symbol} + custom_plotting::Vector{Symbol} full_print::Bool end -# Fields to save additional data for a methodology -# For example cpu peak in roofline methodology -Custom_Methodology_Elements = Union{Metric_Result, Float64, Function} +""" + A custom methodology element is used to save informational metrics, other special values and custom functions to be executed after testing. +""" +Custom_Methodology_Elements = Union{Metric_Result,Float64,Function} + +""" +This struct is used in the test suite to save a methodology result, which in turn is constituted of a group of metric results and their references. Additionally, custom elements that are not subject to test are also saved, e.g. informational metrics, printing functions. +""" @kwdef struct Methodology_Result name::AbstractString metrics :: Vector{Pair{Metric_Result, Metric_Constraint}} @@ -42,13 +60,18 @@ Custom_Methodology_Elements = Union{Metric_Result, Float64, Function} custom_auto_print::Bool = true end - +""" +This struct saves a complete test suite result for one execution. It also saves the raw measurements obtained from the targets. +""" @kwdef struct Perftest_Result timestamp :: Float64 benchmarks :: BenchmarkGroup perftests :: Dict end +""" +This struct is the root of the data recording file, it can save several performance test suite execution results. +""" @kwdef struct Perftest_Datafile_Root results :: Vector{Perftest_Result} methodologies_history :: Vector{Dict{StrOrSym, Any}} diff --git a/src/prints.jl b/src/prints.jl index 4af87d5..4237740 100644 --- a/src/prints.jl +++ b/src/prints.jl @@ -1,28 +1,48 @@ using Printf using BenchmarkTools - +""" + Macro that adds a space at the beggining of a string +""" macro lpad(pad) return :(" " ^ $(esc(pad))) end +""" + Prints the element in color blue +""" function p_blue(printable) printstyled(printable, color=:blue) end +""" + Prints the element in color red +""" function p_red(printable) - printstyled(printable, color=:red) + printstyled(printable, color=:red) end + +""" + Prints the element in color yellow +""" function p_yellow(printable) - printstyled(printable, color=:yellow) + printstyled(printable, color=:yellow) end + + +""" + Prints the element in color green +""" function p_green(printable) printstyled(printable, color=:green) end -# Auxiliar print functions -function printDepth!(depth :: AbstractArray) - for i in eachindex(depth) - if depth[i].depth_flag == false +""" + This method is used to print the test names, with consideration on +the hierarchy and adding indentation whenever necessary +""" +function printDepth!(depth::AbstractArray) + for i in eachindex(depth) + if depth[i].depth_flag == false if firstindex(depth) == i printstyled("PERFORMANCE TEST:\n", color=:yellow) end @@ -36,6 +56,9 @@ function printDepth!(depth :: AbstractArray) end +""" +This method dumps into the output a test result in case of failure. The output will be formatted to make it easy to read. +""" function printfail(judgement::BenchmarkTools.TrialJudgement, trial::BenchmarkTools.Trial, reference :: BenchmarkTools.Trial, tolerance :: FloatRange, tab::Int) print(lpad(">", tab)) @@ -53,6 +76,9 @@ function printfail(judgement::BenchmarkTools.TrialJudgement, trial::BenchmarkToo println("") end +""" +This method is used to print a graphical representation on a test result and the admisible intervals it can take. The result will and the two bounds will be printed in order. +""" function printIntervalLanding(bot, top, landing, down_is_bad::Bool = true) @assert bot < top @@ -84,6 +110,9 @@ function printIntervalLanding(bot, top, landing, down_is_bad::Bool = true) end end +""" +This method is used to dump into the output the information about a metric and the value obtained in a specific test. +""" function printMetric(metric :: Metric_Result, constraint:: Metric_Constraint, tab::Int) println(@lpad(tab) * "-" ^ 72) @@ -117,14 +146,18 @@ function printMetric(metric :: Metric_Result, constraint:: Metric_Constraint, ta end end - +""" +This function is used to dump metric information regading auxiliar metrics, which are not used in testing. +""" function auxiliarMetricPrint(metric :: Metric_Result, tab::Int) println(" " ^ tab * "Metric: " * metric.name * " [" * metric.units * "]") println(" " ^ tab * " = ", metric.value) println("") end - +""" +This function is used to print the information relative to a methodology, relative to a a specific test execution result. This will usually print a series of metrics and might also print plots. +""" function printMethodology(methodology :: Methodology_Result, tab :: Int) println(@lpad(tab) * "═"^72)