Use Common Test for testing when Gradualizer should pass, fail, and its known problems

Makefile

@@ -149,6 +149,9 @@ eunit: compile-tests
 	erl $(ERL_OPTS) -noinput -pa ebin -pa test -eval \
 	 '$(erl_run_eunit), halt().'
 
+ct:
+	@rebar3 ct --label "git: $$(git describe --tags --always) $$(git diff --no-ext-diff --quiet --exit-code || echo '(modified)')"
+
 cli-tests: bin/gradualizer test/arg.beam
 	# CLI test cases
 	# 1. When checking a dir with erl files, erl file names are printed

rebar.config

@@ -4,10 +4,7 @@
           {deps,
            [
             {proper, {git, "https://github.com/proper-testing/proper.git", {branch, "master"}}}
-           ]},
-          %% see the maybe expression fail;
-          %% the VM also needs to be configured to load the module
-          {erl_opts, [{feature,maybe_expr,enable}]}
+           ]}
          ]}
  ]}.
 

src/typechecker.erl

@@ -5746,8 +5746,10 @@ type_check_forms(Forms, Opts) ->
 %% a Gradualizer (NOT the checked program!) error.
 -spec type_check_form_with_timeout(expr(), [any()], boolean(), env(), [any()]) -> [any()].
 type_check_form_with_timeout(Function, Errors, StopOnFirstError, Env, Opts) ->
-    %% TODO: make FormCheckTimeOut configurable
-    FormCheckTimeOut = ?form_check_timeout_ms,
+    FormCheckTimeOut = case lists:keyfind(form_check_timeout_ms, 1, Opts) of
+                           false -> ?form_check_timeout_ms;
+                           {form_check_timeout_ms, MS} -> MS
+                       end,
     ?verbose(Env, "Spawning async task...~n", []),
     Self = self(),
     Task = fun () ->

test/gradualizer_dynamic_suite.erl

@@ -0,0 +1,60 @@
+-module(gradualizer_dynamic_suite).
+
+-export([reload/1]).
+
+-include_lib("common_test/include/ct.hrl").
+-include_lib("stdlib/include/assert.hrl").
+
+reload(Config) ->
+    Module = ?config(dynamic_suite_module, Config),
+    ?assert(Module /= undefined),
+    case erlang:function_exported(Module, generated_tests, 0) of
+        true ->
+            {ok, Module:generated_tests()};
+        false ->
+            Path = ?config(dynamic_suite_test_path, Config),
+            ?assert(Path /= undefined),
+            Forms = get_forms(Module),
+            FilesForms = map_erl_files(fun (File) ->
+                                               make_test_form(Forms, File, Config)
+                                       end, Path),
+            {TestFiles, TestForms} = lists:unzip(FilesForms),
+            TestNames = [ list_to_atom(filename:basename(File, ".erl")) || File <- TestFiles ],
+            ct:pal("All tests found under ~s:\n~p\n", [Path, TestNames]),
+            GeneratedTestsForm = make_generated_tests_form(TestNames),
+            NewForms = Forms ++ TestForms ++ [GeneratedTestsForm, {eof, 0}],
+            {ok, _} = merl:compile_and_load(NewForms),
+            {ok, TestNames}
+    end.
+
+map_erl_files(Fun, Dir) ->
+    Files = filelib:wildcard(filename:join(Dir, "*.erl")),
+    [{filename:basename(File), Fun(File)} || File <- Files].
+
+make_test_form(Forms, File, Config) ->
+    TestTemplateName = ?config(dynamic_test_template, Config),
+    ?assert(TestTemplateName /= undefined),
+    TestTemplate = merl:quote("'@Name'(_) -> _@Body."),
+    {function, _Anno, _Name, 1, Clauses} = lists:keyfind(TestTemplateName, 3, Forms),
+    [{clause, _, _Args, _Guards, ClauseBodyTemplate}] = Clauses,
+    TestName = filename:basename(File, ".erl"),
+    ClauseBody = merl:subst(ClauseBodyTemplate, [{'File', erl_syntax:string(File)}]),
+    TestEnv = [
+               {'Name', erl_syntax:atom(TestName)},
+               {'Body', ClauseBody}
+              ],
+    erl_syntax:revert(merl:subst(TestTemplate, TestEnv)).
+
+make_generated_tests_form(TestNames) ->
+    Template = merl:quote("generated_tests() -> _@Body."),
+    erl_syntax:revert(merl:subst(Template, [{'Body', merl:term(TestNames)}])).
+
+get_forms(Module) ->
+    ModPath = code:which(Module),
+    {ok, {Module, [Abst]}} = beam_lib:chunks(ModPath, [abstract_code]),
+    {abstract_code, {raw_abstract_v1, Forms}} = Abst,
+    StripEnd = fun
+                   ({eof, _}) -> false;
+                   (_) -> true
+               end,
+    lists:filter(StripEnd, Forms).

test/known_problems_should_fail_SUITE.erl

@@ -0,0 +1,63 @@
+-module(known_problems_should_fail_SUITE).
+
+-compile([export_all, nowarn_export_all]).
+
+%% EUnit has some handy macros, so let's use it, too
+-include_lib("eunit/include/eunit.hrl").
+
+%% Test server callbacks
+-export([suite/0,
+         all/0,
+         groups/0,
+         init_per_suite/1, end_per_suite/1]).
+
+suite() ->
+    [{timetrap, {minutes, 10}}].
+
+all() ->
+    [{group, all_tests}].
+
+groups() ->
+    Config = [
+              {dynamic_suite_module, ?MODULE},
+              {dynamic_suite_test_path, filename:join(code:lib_dir(gradualizer), "test/known_problems/should_fail")},
+              {dynamic_test_template, known_problems_should_fail_template}
+             ],
+    {ok, GeneratedTests} = gradualizer_dynamic_suite:reload(Config),
+    [{all_tests, [parallel], GeneratedTests}].
+
+init_per_suite(Config) ->
+    {ok, _} = application:ensure_all_started(gradualizer),
+    ok = load_prerequisites(code:lib_dir(gradualizer)),
+    Config.
+
+load_prerequisites(AppBase) ->
+    %% exhaustive_user_type.erl is referenced by exhaustive_remote_user_type.erl
+    gradualizer_db:import_erl_files([filename:join(AppBase, "test/should_fail/exhaustive_user_type.erl")]),
+    ok.
+
+end_per_suite(_Config) ->
+    ok = application:stop(gradualizer),
+    ok.
+
+known_problems_should_fail_template(_@File) ->
+    Result = safe_type_check_file(_@File, [return_errors, {form_check_timeout_ms, 2000}]),
+    case Result of
+        crash ->
+            ok;
+        Errors ->
+            ErrorsExceptTimeouts = lists:filter(
+                                     fun ({_File, {form_check_timeout, _}}) -> false; (_) -> true end,
+                                     Errors),
+            ?assertEqual(0, length(ErrorsExceptTimeouts))
+    end.
+
+safe_type_check_file(File) ->
+    safe_type_check_file(File, []).
+
+safe_type_check_file(File, Opts) ->
+    try
+        gradualizer:type_check_file(File, Opts)
+    catch
+        _:_ -> crash
+    end.

test/known_problems_should_pass_SUITE.erl

@@ -0,0 +1,55 @@
+-module(known_problems_should_pass_SUITE).
+
+-compile([export_all, nowarn_export_all]).
+
+%% EUnit has some handy macros, so let's use it, too
+-include_lib("eunit/include/eunit.hrl").
+
+%% Test server callbacks
+-export([suite/0,
+         all/0,
+         groups/0,
+         init_per_suite/1, end_per_suite/1]).
+
+suite() ->
+    [{timetrap, {minutes, 10}}].
+
+all() ->
+    [{group, all_tests}].
+
+groups() ->
+    Config = [
+              {dynamic_suite_module, ?MODULE},
+              {dynamic_suite_test_path, filename:join(code:lib_dir(gradualizer), "test/known_problems/should_pass")},
+              {dynamic_test_template, known_problems_should_pass_template}
+             ],
+    {ok, GeneratedTests} = gradualizer_dynamic_suite:reload(Config),
+    [{all_tests, [parallel], GeneratedTests}].
+
+init_per_suite(Config) ->
+    {ok, _} = application:ensure_all_started(gradualizer),
+    ok = load_prerequisites(code:lib_dir(gradualizer)),
+    Config.
+
+load_prerequisites(_AppBase) ->
+    ok.
+
+end_per_suite(_Config) ->
+    ok = application:stop(gradualizer),
+    ok.
+
+known_problems_should_pass_template(_@File) ->
+    {ok, Forms} = gradualizer_file_utils:get_forms_from_erl(_@File, []),
+    ExpectedErrors = typechecker:number_of_exported_functions(Forms),
+    ReturnedErrors = length(safe_type_check_file(_@File, [return_errors, {form_check_timeout_ms, 2000}])),
+    ?assertEqual(ExpectedErrors, ReturnedErrors).
+
+safe_type_check_file(File) ->
+    safe_type_check_file(File, []).
+
+safe_type_check_file(File, Opts) ->
+    try
+        gradualizer:type_check_file(File, Opts)
+    catch
+        _:_ -> crash
+    end.

test/should_fail_SUITE.erl

@@ -0,0 +1,56 @@
+-module(should_fail_SUITE).
+
+-compile([export_all, nowarn_export_all]).
+
+%% EUnit has some handy macros, so let's use it, too
+-include_lib("eunit/include/eunit.hrl").
+
+%% Test server callbacks
+-export([suite/0,
+         all/0,
+         groups/0,
+         init_per_suite/1, end_per_suite/1]).
+
+suite() ->
+    [{timetrap, {minutes, 10}}].
+
+all() ->
+    [{group, all_tests}].
+
+groups() ->
+    Config = [
+              {dynamic_suite_module, ?MODULE},
+              {dynamic_suite_test_path, filename:join(code:lib_dir(gradualizer), "test/should_fail")},
+              {dynamic_test_template, should_fail_template}
+             ],
+    {ok, GeneratedTests} = gradualizer_dynamic_suite:reload(Config),
+    [{all_tests, [parallel], GeneratedTests}].
+
+init_per_suite(Config) ->
+    {ok, _} = application:ensure_all_started(gradualizer),
+    ok = load_prerequisites(code:lib_dir(gradualizer)),
+    Config.
+
+load_prerequisites(AppBase) ->
+    %% user_types.erl is referenced by opaque_fail.erl.
+    %% It is not in the sourcemap of the DB so let's import it manually
+    gradualizer_db:import_erl_files([filename:join(AppBase, "test/should_pass/user_types.erl")]),
+    %% exhaustive_user_type.erl is referenced by exhaustive_remote_user_type.erl
+    gradualizer_db:import_erl_files([filename:join(AppBase, "test/should_fail/exhaustive_user_type.erl")]),
+    ok.
+
+end_per_suite(_Config) ->
+    ok = application:stop(gradualizer),
+    ok.
+
+should_fail_template(_@File) ->
+    Errors = gradualizer:type_check_file(_@File, [return_errors, {form_check_timeout_ms, 2000}]),
+    Timeouts = [ E || {_File, {form_check_timeout, _}} = E <- Errors],
+    ?assertEqual(0, length(Timeouts)),
+    %% Test that error formatting doesn't crash
+    Opts = [{fmt_location, brief},
+            {fmt_expr_fun, fun erl_prettypr:format/1}],
+    lists:foreach(fun({_, Error}) -> gradualizer_fmt:handle_type_error(Error, Opts) end, Errors),
+    {ok, Forms} = gradualizer_file_utils:get_forms_from_erl(_@File, []),
+    ExpectedErrors = typechecker:number_of_exported_functions(Forms),
+    ?assertEqual(ExpectedErrors, length(Errors)).

test/should_pass/module_info.erl → test/should_pass/module_info_pass.erl

@@ -1,4 +1,4 @@
--module(module_info).
+-module(module_info_pass).
 
 -compile([export_all, nowarn_export_all]).
 
@@ -18,4 +18,4 @@ unary_direct() ->
 -spec unary_var() -> atom().
 unary_var() ->
     I = erlang:module_info(module),
-    I.
+    I.

test/should_pass_SUITE.erl

@@ -0,0 +1,48 @@
+-module(should_pass_SUITE).
+
+-compile([export_all, nowarn_export_all]).
+
+%% EUnit has some handy macros, so let's use it, too
+-include_lib("eunit/include/eunit.hrl").
+
+%% Test server callbacks
+-export([suite/0,
+         all/0,
+         groups/0,
+         init_per_suite/1, end_per_suite/1]).
+
+suite() ->
+    [{timetrap, {minutes, 10}}].
+
+all() ->
+    [{group, all_tests}].
+
+groups() ->
+    Opts = [
+            {dynamic_suite_module, ?MODULE},
+            {dynamic_suite_test_path, filename:join(code:lib_dir(gradualizer), "test/should_pass")},
+            {dynamic_test_template, should_pass_template}
+           ],
+    {ok, GeneratedTests} = gradualizer_dynamic_suite:reload(Opts),
+    [{all_tests, [parallel], GeneratedTests}].
+
+init_per_suite(Config) ->
+    {ok, _} = application:ensure_all_started(gradualizer),
+    ok = load_prerequisites(code:lib_dir(gradualizer)),
+    Config.
+
+load_prerequisites(AppBase) ->
+    %% user_types.erl is referenced by remote_types.erl and opaque.erl.
+    %% It is not in the sourcemap of the DB so let's import it manually
+    gradualizer_db:import_erl_files([filename:join(AppBase, "test/should_pass/user_types.erl")]),
+    gradualizer_db:import_erl_files([filename:join(AppBase, "test/should_pass/other_module.erl")]),
+    %% imported.erl references any.erl
+    gradualizer_db:import_erl_files([filename:join(AppBase, "test/should_pass/any.erl")]),
+    ok.
+
+end_per_suite(_Config) ->
+    ok = application:stop(gradualizer),
+    ok.
+
+should_pass_template(_@File) ->
+    ?assertEqual(ok, gradualizer:type_check_file(_@File, [{form_check_timeout_ms, 2000}])).