Add general tests testing inputs, clean up utils, rename ARC grammar …

…to follow convention

src/HerbBenchmarks.jl

@@ -51,5 +51,6 @@ export
     get_default_grammar,
 
     make_public,
-    make_public_rec
+    make_public_rec,
+    get_submodules
 end # module HerbBenchmarks

src/data/Abstract_Reasoning_2019/grammar.jl

@@ -1,4 +1,4 @@
-grammar = @cfgrammar begin
+grammar_arc = @cfgrammar begin
     Start = (state = Grid; returnState(state))
     InputGrid = initState(_arg_1)
     Return = returnState(state)

src/export_module.jl

@@ -106,53 +106,20 @@ export make_public_rec
 """
 
 """
-function all_problems(module_name::Module)
-    all_problems = [module_name.eval(var) for var in filter(v -> startswith(string(v), "problem_"), names(module_name; all=true))]
-    return all_problems
-end
-
-"""
-
-"""
-function all_grammars(module_name::Module)
-    all_grammars = [module_name.eval(var) for var in filter(v -> startswith(string(v), "grammar_"), names(module_name; all=true))]
-    return all_grammars
-end
-
-"""
-
-"""
-function find_corresponding_grammar(problem_name::AbstractString, module_name::Module)
-    # Extract the problem number from the problem identifier
-    problem_number = match(r"_.*$", string(problem_name)).match
-
-    # Construct the grammar identifier
-    grammar_id = Symbol("grammar" * problem_number)
-
-    # Check if this specific grammar exists in the module
-    if isdefined(module_name, grammar_id)
-        return grammar_id
-    else
-        # Find the default grammar that starts with "grammar_"
-        for name in names(module_name; all=true)
-            if startswith(string(name), "grammar_")
-                return name
+function get_submodules(mod::Module)
+    submodules = []
+    for name in names(mod, all = true)
+        try
+            item = getfield(mod, name)
+            if isa(item, Module) && item !== mod
+                push!(submodules, item)
+            end
+        catch e
+            # Catch errors for names that can't be accessed with getfield
+            if !isa(e, UndefVarError)
+                throw(e)
             end
         end
     end
-
-    # Return nothing or throw an error if no grammar is found
-    throw(KeyError("No corresponding grammar found for problem $problem"))
-end
-
-"""
-    all_problem_grammar_pairs(module_name::Module)
-
-Takes a module and returns a dict mapping from the problem name to a tuple of problem and grammar. If there is no grammar with a matching name it searches for the default grammar.
-"""
-function all_problem_grammar_pairs(module_name::Module)
-    # Find the corresponding grammar for each problem
-    problem_grammar_dict = Dict(string(var) => (module_name.eval(var), module_name.eval(find_corresponding_grammar(string(var), module_name))) for var in filter(v -> startswith(string(v), "problem_"), names(module_name; all=true)))
-
-    return problem_grammar_dict
+    return submodules
 end

test/runtests.jl

@@ -4,6 +4,7 @@ using HerbSpecification
 using Test
 
 @testset "HerbBenchmarks.jl" verbose = true begin
+    include("test_general.jl")
     include("test_string_transformations_2020.jl")
     include("test_robots_2020.jl")
     include("test_pixels_2020.jl")

test/test_general.jl

@@ -0,0 +1,34 @@
+input_rules(grammar::AbstractGrammar) = findall(rule -> occursin("_arg_", string(rule)), grammar.rules)
+
+@testset verbose=true "General tests on all submodules" begin
+    modules = get_submodules(HerbBenchmarks)
+               for mod in modules
+        @testset verbose=true "Module $mod" begin
+            problems = get_all_problems(mod)
+            if length(problems) == 0 
+                continue
+            end
+            @testset verbose=true "Inputs are well-typed" begin
+                @test typeof(problems[1]) <: HerbSpecification.Problem
+                @test typeof(problems[1].spec[1]) == HerbSpecification.IOExample
+            end
+
+            if mod ∉ [HerbBenchmarks.String_transformations_2020,
+                      HerbBenchmarks.Pixels_2020,
+                      HerbBenchmarks.Robots_2020]
+
+                @testset verbose=true "Inputs align in grammar and problem" begin
+                    pairs = get_all_problem_grammar_pairs(mod)
+                    for pair in pairs
+                        p,g = pair.problem, pair.grammar
+                        p_args = length(p.spec[1].in)
+                        g_args = length(input_rules(g))
+                        # test whether 
+                        @test p_args == g_args
+                    end            
+                end
+            end
+        end
+    end
+end
+