tests: added new tests + utils code improved (#221)

* tests: added new tests + utils code improved

* new tests:
- test_preset_args_combination
- test_preset_mxnet_framework

* added more flags to test_preset_args
* added validation for flags in utils

* tests: added new tests + fixed utils

* tests: added new checkpoint test

* tests: added checkpoint test improve utils

* tests: added tests + improve validations

* bump integration CI run timeout.

* tests: improve timerun + add functional test marker

.circleci/config.yml

@@ -157,7 +157,7 @@ jobs:
           name: run integration tests
           command: |
             python3 rl_coach/tests/test_eks.py  -c coach-test -bn ${CIRCLE_BUILD_NUM} -tn integration-test -tc 'make integration_tests_without_docker' -i 316971102342.dkr.ecr.us-west-2.amazonaws.com/coach:$(git describe --tags --always --dirty) -cpu 2048 -mem 4096
-          no_output_timeout: 20m
+          no_output_timeout: 30m
       - run:
           name: cleanup
           command: |

rl_coach/tests/conftest.py

@@ -15,63 +15,30 @@
 #
 """PyTest configuration."""
 
-import configparser as cfgparser
 import os
-import platform
 import shutil
+import sys
 import pytest
 import rl_coach.tests.utils.args_utils as a_utils
 import rl_coach.tests.utils.presets_utils as p_utils
 from rl_coach.tests.utils.definitions import Definitions as Def
 from os import path
 
 
-def pytest_collection_modifyitems(config, items):
-    """pytest built in method to pre-process cli options"""
-    global test_config
-    test_config = cfgparser.ConfigParser()
-    str_rootdir = str(config.rootdir)
-    str_inifile = str(config.inifile)
-    # Get the relative path of the inifile
-    # By default is an absolute path but relative path when -c option used
-    config_path = os.path.relpath(str_inifile, str_rootdir)
-    config_path = os.path.join(str_rootdir, config_path)
-    assert (os.path.exists(config_path))
-    test_config.read(config_path)
-
-
-def pytest_runtest_setup(item):
-    """Called before test is run."""
-    if len(item.own_markers) < 1:
-        return
-    if (item.own_markers[0].name == "unstable" and
-            "unstable" not in item.config.getoption("-m")):
-        pytest.skip("skipping unstable test")
-
-    if item.own_markers[0].name == "linux_only":
-        if platform.system() != 'Linux':
-            pytest.skip("Skipping test that isn't Linux OS.")
-
-    if item.own_markers[0].name == "golden_test":
-        """ do some custom configuration for golden tests. """
-        # TODO: add custom functionality
-        pass
-
-
 @pytest.fixture(scope="module", params=list(p_utils.collect_presets()))
 def preset_name(request):
     """
     Return all preset names
     """
-    return request.param
+    yield request.param
 
 
 @pytest.fixture(scope="function", params=list(a_utils.collect_args()))
 def flag(request):
     """
     Return flags names in function scope
     """
-    return request.param
+    yield request.param
 
 
 @pytest.fixture(scope="module", params=list(a_utils.collect_preset_for_args()))
@@ -80,7 +47,26 @@ def preset_args(request):
     Return preset names that can be used for args testing only; working in
     module scope
     """
-    return request.param
+    yield request.param
+
+
+@pytest.fixture(scope="module", params=list(a_utils.collect_preset_for_seed()))
+def preset_args_for_seed(request):
+    """
+    Return preset names that can be used for args testing only and for special
+    action when using seed argument; working in module scope
+    """
+    yield request.param
+
+
+@pytest.fixture(scope="module",
+                params=list(a_utils.collect_preset_for_mxnet()))
+def preset_for_mxnet_args(request):
+    """
+    Return preset names that can be used for args testing only; this special
+    fixture will be used for mxnet framework only. working in module scope
+    """
+    yield request.param
 
 
 @pytest.fixture(scope="function")
@@ -105,6 +91,7 @@ def __init__(self, csv, log, pattern):
 
     p_valid_params = p_utils.validation_params(p_name)
 
+    sys.path.append('.')
     test_name = 'ExpName_{}'.format(p_name)
     test_path = os.path.join(Def.Path.experiments, test_name)
     if path.exists(test_path):
@@ -113,7 +100,7 @@ def __init__(self, csv, log, pattern):
     # get the stdout for logs results
     log_file_name = 'test_log_{}.txt'.format(p_name)
     stdout = open(log_file_name, 'w')
-    fn_pattern = 'worker_0*.csv' if p_valid_params.num_workers > 1 else '*.csv'
+    fn_pattern = '*.csv' if p_valid_params.num_workers > 1 else 'worker_0*.csv'
 
     res = CreateCsvLog(test_path, stdout, fn_pattern)
 
@@ -123,5 +110,5 @@ def __init__(self, csv, log, pattern):
     if path.exists(res.exp_path):
         shutil.rmtree(res.exp_path)
 
-    if os.path.exists(res.exp_path):
-        os.remove(res.stdout)
+    if path.exists(res.stdout.name):
+        os.remove(res.stdout.name)

rl_coach/tests/test_checkpoint.py

@@ -1,24 +1,125 @@
+#
+# Copyright (c) 2019 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
 import os
+import shutil
+import subprocess
+import time
 import pytest
+import signal
 import tempfile
-
+import numpy as np
+import pandas as pd
+import rl_coach.tests.utils.args_utils as a_utils
+import rl_coach.tests.utils.test_utils as test_utils
 from rl_coach import checkpoint
+from rl_coach.tests.utils.definitions import Definitions as Def
 
 
 @pytest.mark.unit_test
 def test_get_checkpoint_state():
-    files = ['4.test.ckpt.ext', '2.test.ckpt.ext', '3.test.ckpt.ext', '1.test.ckpt.ext', 'prefix.10.test.ckpt.ext']
+    files = ['4.test.ckpt.ext', '2.test.ckpt.ext', '3.test.ckpt.ext',
+             '1.test.ckpt.ext', 'prefix.10.test.ckpt.ext']
     with tempfile.TemporaryDirectory() as temp_dir:
         [open(os.path.join(temp_dir, fn), 'a').close() for fn in files]
-        checkpoint_state = checkpoint.get_checkpoint_state(temp_dir, all_checkpoints=True)
-        assert checkpoint_state.model_checkpoint_path == os.path.join(temp_dir, '4.test.ckpt')
+        checkpoint_state = \
+            checkpoint.get_checkpoint_state(temp_dir,
+                                            all_checkpoints=True)
+        assert checkpoint_state.model_checkpoint_path == os.path.join(
+            temp_dir, '4.test.ckpt')
         assert checkpoint_state.all_model_checkpoint_paths == \
                [os.path.join(temp_dir, f[:-4]) for f in sorted(files[:-1])]
 
-        reader = checkpoint.CheckpointStateReader(temp_dir, checkpoint_state_optional=False)
+        reader = \
+            checkpoint.CheckpointStateReader(temp_dir,
+                                             checkpoint_state_optional=False)
         assert reader.get_latest() is None
         assert len(reader.get_all()) == 0
 
         reader = checkpoint.CheckpointStateReader(temp_dir)
         assert reader.get_latest().num == 4
         assert [ckp.num for ckp in reader.get_all()] == [1, 2, 3, 4]
+
+
+@pytest.mark.functional_test
+def test_restore_checkpoint(preset_args, clres, start_time=time.time()):
+    """ Create checkpoint and restore them in second run."""
+
+    def _create_cmd_and_run(flag):
+
+        run_cmd = [
+            'python3', 'rl_coach/coach.py',
+            '-p', '{}'.format(preset_args),
+            '-e', '{}'.format("ExpName_" + preset_args),
+        ]
+        test_flag = a_utils.add_one_flag_value(flag=flag)
+        run_cmd.extend(test_flag)
+
+        p = subprocess.Popen(run_cmd, stdout=clres.stdout, stderr=clres.stdout)
+
+        return p
+
+    create_cp_proc = _create_cmd_and_run(flag=['--checkpoint_save_secs', '5'])
+
+    # wait for checkpoint files
+    csv_list = a_utils.get_csv_path(clres=clres)
+    assert len(csv_list) > 0
+    exp_dir = os.path.dirname(csv_list[0])
+
+    checkpoint_dir = os.path.join(exp_dir, Def.Path.checkpoint)
+
+    checkpoint_test_dir = os.path.join(Def.Path.experiments, Def.Path.test_dir)
+    if os.path.exists(checkpoint_test_dir):
+        shutil.rmtree(checkpoint_test_dir)
+
+    entities = a_utils.get_files_from_dir(checkpoint_dir)
+
+    while not any("10_Step" in file for file in entities) and time.time() - \
+            start_time < Def.TimeOuts.test_time_limit:
+        entities = a_utils.get_files_from_dir(checkpoint_dir)
+        time.sleep(1)
+
+    assert len(entities) > 0
+    assert "checkpoint" in entities
+    assert any(".ckpt." in file for file in entities)
+
+    # send CTRL+C to close experiment
+    create_cp_proc.send_signal(signal.SIGINT)
+
+    csv = pd.read_csv(csv_list[0])
+    rewards = csv['Evaluation Reward'].values
+    rewards = rewards[~np.isnan(rewards)]
+    min_reward = np.amin(rewards)
+
+    if os.path.isdir(checkpoint_dir):
+        shutil.copytree(exp_dir, checkpoint_test_dir)
+        shutil.rmtree(exp_dir)
+
+    create_cp_proc.kill()
+    checkpoint_test_dir = "{}/{}".format(checkpoint_test_dir,
+                                         Def.Path.checkpoint)
+    # run second time with checkpoint folder  (restore)
+    restore_cp_proc = _create_cmd_and_run(flag=['-crd', checkpoint_test_dir,
+                                                '--evaluate'])
+
+    new_csv_list = test_utils.get_csv_path(clres=clres)
+    time.sleep(10)
+
+    csv = pd.read_csv(new_csv_list[0])
+    res = csv['Episode Length'].values[-1]
+    assert res >= min_reward, \
+        Def.Consts.ASSERT_MSG.format(str(res) + ">=" + str(min_reward),
+                                     str(res) + " < " + str(min_reward))
+    restore_cp_proc.kill()