Require Python 3.6 + Changes to CI configuration (#452)

* Change build_*_env jobs to pull base image of current "tag" instead of "master" image * Change nightly flow so build_*_env jobs now gated by build_base (so change in previous bullet works in nightly) * Bugfix in CheckpointDataStore: Call to object.__init__ with parameters * Disabling unstable Doom A3C and ACER golden tests
IntelLabs · Jul 26, 2020 · c973828 · c973828
1 parent a6689b6
commit c973828
Show file tree

Hide file tree

Showing 7 changed files with 46 additions and 34 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -75,9 +75,9 @@ jobs:
           command: |
             REGISTRY=316971102342.dkr.ecr.us-west-2.amazonaws.com
             TAG=$(git describe --tags --always --dirty)
-            docker pull ${REGISTRY}/coach-base:${MASTER_BRANCH}
-            docker tag ${REGISTRY}/coach-base:${MASTER_BRANCH} coach-base:master
-            docker build --cache-from ${REGISTRY}/coach-base:${MASTER_BRANCH} -t ${REGISTRY}/coach-gym_environment:${TAG} -f docker/Dockerfile.gym_environment .
+            docker pull ${REGISTRY}/coach-base:${TAG}
+            docker tag ${REGISTRY}/coach-base:${TAG} coach-base:master
+            docker build --cache-from ${REGISTRY}/coach-base:${TAG} -t ${REGISTRY}/coach-gym_environment:${TAG} -f docker/Dockerfile.gym_environment .
             docker push ${REGISTRY}/coach-gym_environment:${TAG}
           no_output_timeout: 10m
 
@@ -93,9 +93,9 @@ jobs:
           command: |
             REGISTRY=316971102342.dkr.ecr.us-west-2.amazonaws.com
             TAG=$(git describe --tags --always --dirty)
-            docker pull ${REGISTRY}/coach-base:${MASTER_BRANCH}
-            docker tag ${REGISTRY}/coach-base:${MASTER_BRANCH} coach-base:master
-            docker build --cache-from ${REGISTRY}/coach-base:${MASTER_BRANCH} -t ${REGISTRY}/coach-doom_environment:${TAG} -f docker/Dockerfile.doom_environment .
+            docker pull ${REGISTRY}/coach-base:${TAG}
+            docker tag ${REGISTRY}/coach-base:${TAG} coach-base:master
+            docker build --cache-from ${REGISTRY}/coach-base:${TAG} -t ${REGISTRY}/coach-doom_environment:${TAG} -f docker/Dockerfile.doom_environment .
             docker push ${REGISTRY}/coach-doom_environment:${TAG}
           no_output_timeout: 10m
 
@@ -111,9 +111,9 @@ jobs:
           command: |
             REGISTRY=316971102342.dkr.ecr.us-west-2.amazonaws.com
             TAG=$(git describe --tags --always --dirty)
-            docker pull ${REGISTRY}/coach-base:${MASTER_BRANCH}
-            docker tag ${REGISTRY}/coach-base:${MASTER_BRANCH} coach-base:master
-            docker build --cache-from ${REGISTRY}/coach-base:${MASTER_BRANCH} --build-arg MUJOCO_KEY=${MUJOCO_KEY} -t ${REGISTRY}/coach-mujoco_environment:${TAG} -f docker/Dockerfile.mujoco_environment .
+            docker pull ${REGISTRY}/coach-base:${TAG}
+            docker tag ${REGISTRY}/coach-base:${TAG} coach-base:master
+            docker build --cache-from ${REGISTRY}/coach-base:${TAG} --build-arg MUJOCO_KEY=${MUJOCO_KEY} -t ${REGISTRY}/coach-mujoco_environment:${TAG} -f docker/Dockerfile.mujoco_environment .
             docker push ${REGISTRY}/coach-mujoco_environment:${TAG}
           no_output_timeout: 10m
 
@@ -130,9 +130,9 @@ jobs:
 #          command: |
 #            REGISTRY=316971102342.dkr.ecr.us-west-2.amazonaws.com
 #            TAG=$(git describe --tags --always --dirty)
-#            docker pull ${REGISTRY}/coach-base:${MASTER_BRANCH}
-#            docker tag ${REGISTRY}/coach-base:${MASTER_BRANCH} coach-base:master
-#            docker build --cache-from ${REGISTRY}/coach-base:${MASTER_BRANCH} -t ${REGISTRY}/coach-starcraft_environment:${TAG} -f docker/Dockerfile.starcraft_environment .
+#            docker pull ${REGISTRY}/coach-base:${TAG}
+#            docker tag ${REGISTRY}/coach-base:${TAG} coach-base:master
+#            docker build --cache-from ${REGISTRY}/coach-base:${TAG} -t ${REGISTRY}/coach-starcraft_environment:${TAG} -f docker/Dockerfile.starcraft_environment .
 #            docker push ${REGISTRY}/coach-starcraft_environment:${TAG}
 #          no_output_timeout: 10m
 
@@ -614,6 +614,9 @@ workflows:
       - multinode_test:
           requires:
             - multinode_approval
+      # NOTE: build_gym/doom/mujoco_env MUST occur after successful build_base stage
+      #       In this workflow this is satisfied by having this flow:
+      #       build_base --> e2e_approval --> build_*_env
       - e2e_approval:
           type: approval
           requires:
@@ -718,10 +721,17 @@ workflows:
               only:
                 - master
     jobs:
-      - build_gym_env
-      - build_doom_env
-      - build_mujoco_env
       - build_base
+      # NOTE: build_gym/doom/mujoco_env MUST occur after successful build_base stage
+      - build_gym_env:
+          requires:
+            - build_base
+      - build_doom_env:
+          requires:
+            - build_base
+      - build_mujoco_env:
+          requires:
+            - build_base
       - unit_tests:
           requires:
             - build_base

diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -2,7 +2,7 @@ FROM coach-base:master as builder
 
 # prep some of the more common environments
 # Gym (installed with coach)
-Run pip3 install gym[atari]==0.12.5 box2d
+RUN pip3 install gym[atari]==0.12.5 box2d
 # Mujoco
 RUN mkdir -p ~/.mujoco \
     && wget https://www.roboti.us/download/mjpro150_linux.zip -O mujoco.zip \

diff --git a/docker/Dockerfile.base b/docker/Dockerfile.base
@@ -29,6 +29,16 @@ RUN apt-get update && \
     apt-get clean autoclean && \
     apt-get autoremove -y
 
+################################
+# Install Python 3.6           #
+################################
+
+RUN DEBIAN_FRONTEND=noninteractive add-apt-repository --yes ppa:deadsnakes/ppa && apt-get update
+RUN DEBIAN_FRONTEND=noninteractive apt-get install --yes python3.6-dev python3.6 python3-pip
+
+RUN rm /usr/bin/python3
+RUN ln -s /usr/bin/python3.6 /usr/bin/python3
+
 ############################
 # Install Pip Requirements #
 ############################

diff --git a/rl_coach/data_stores/checkpoint_data_store.py b/rl_coach/data_stores/checkpoint_data_store.py
@@ -25,7 +25,6 @@ class CheckpointDataStore(object):
     A DataStore which relies on the GraphManager check pointing methods to communicate policies.
     """
     def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
         self.checkpoint_num = 0
 
     def end_of_policies(self) -> bool:

diff --git a/rl_coach/presets/Doom_Basic_A3C.py b/rl_coach/presets/Doom_Basic_A3C.py
@@ -42,10 +42,10 @@
 # Test #
 ########
 preset_validation_params = PresetValidationParameters()
-preset_validation_params.test = True
-preset_validation_params.min_reward_threshold = 20
-preset_validation_params.max_episodes_to_achieve_reward = 400
-preset_validation_params.num_workers = 8
+# preset_validation_params.test = True
+# preset_validation_params.min_reward_threshold = 20
+# preset_validation_params.max_episodes_to_achieve_reward = 400
+# preset_validation_params.num_workers = 8
 
 
 graph_manager = BasicRLGraphManager(agent_params=agent_params, env_params=env_params,

diff --git a/rl_coach/presets/Doom_Basic_ACER.py b/rl_coach/presets/Doom_Basic_ACER.py
@@ -44,10 +44,10 @@
 # Test #
 ########
 preset_validation_params = PresetValidationParameters()
-preset_validation_params.test = True
-preset_validation_params.min_reward_threshold = 20
-preset_validation_params.max_episodes_to_achieve_reward = 400
-preset_validation_params.num_workers = 8
+# preset_validation_params.test = True
+# preset_validation_params.min_reward_threshold = 20
+# preset_validation_params.max_episodes_to_achieve_reward = 400
+# preset_validation_params.num_workers = 8
 
 
 graph_manager = BasicRLGraphManager(agent_params=agent_params, env_params=env_params,

diff --git a/setup.py b/setup.py
@@ -66,14 +66,7 @@
 
 if not using_GPU:
     if not slim_package:
-        # For linux wth no GPU, we install the Intel optimized version of TensorFlow
-        if sys.platform == "linux" or sys.platform == "linux2":
-            # CI: limiting version to 1.13.1 due to
-            # https://github.com/tensorflow/tensorflow/issues/29617
-            # (reproduced with intel-tensorflow 1.14.0 but not with 1.13.1)
-            install_requires.append('intel-tensorflow==1.13.1')
-        else:
-            install_requires.append('tensorflow>=1.9.0,<=1.14.0')
+        install_requires.append('tensorflow>=1.9.0,<=1.14.0')
     extras['mxnet'] = ['mxnet-mkl>=1.3.0']
 else:
     if not slim_package:
@@ -94,7 +87,7 @@
     author='Intel AI Lab',
     author_email='[email protected]',
     packages=find_packages(),
-    python_requires=">=3.5.*",
+    python_requires=">=3.6.*",
     install_requires=install_requires,
     extras_require=extras,
     package_data={'rl_coach': ['dashboard_components/*.css',