From 106884d182323668d2330dcde3780b1e90a88c4c Mon Sep 17 00:00:00 2001 From: John Agapiou Date: Fri, 28 Jul 2023 10:54:28 +0000 Subject: [PATCH] Update workflows to use composite actions --- .github/actions/install-examples/action.yml | 13 ++++ .github/actions/install-meltingpot/action.yml | 61 +++++++++++++++++++ .github/workflows/pylint-examples.yml | 43 +++++++++++++ .github/workflows/pylint-meltingpot.yml | 41 +++++++++++++ .github/workflows/pylint.yml | 47 -------------- .github/workflows/pytype-examples.yml | 43 +++++++++++++ .github/workflows/pytype-meltingpot.yml | 41 +++++++++++++ .github/workflows/pytype.yml | 55 ----------------- .github/workflows/test-examples.yml | 28 +++------ .github/workflows/test-meltingpot.yml | 24 +++----- meltingpot/testing/mocks_test.py | 9 ++- .../utils/policies/saved_model_policy.py | 2 +- 12 files changed, 267 insertions(+), 140 deletions(-) create mode 100644 .github/actions/install-examples/action.yml create mode 100644 .github/actions/install-meltingpot/action.yml create mode 100644 .github/workflows/pylint-examples.yml create mode 100644 .github/workflows/pylint-meltingpot.yml delete mode 100644 .github/workflows/pylint.yml create mode 100644 .github/workflows/pytype-examples.yml create mode 100644 .github/workflows/pytype-meltingpot.yml delete mode 100644 .github/workflows/pytype.yml diff --git a/.github/actions/install-examples/action.yml b/.github/actions/install-examples/action.yml new file mode 100644 index 000000000..5fa69f25c --- /dev/null +++ b/.github/actions/install-examples/action.yml @@ -0,0 +1,13 @@ +name: install-examples + +runs: + using: composite + steps: + - name: Install Melting Pot + uses: ./.github/actions/install-meltingpot + - name: Install requirements for examples + shell: bash + run: pip -r examples/requirements.txt + - name: Show installed dependencies + shell: bash + run: pip list diff --git a/.github/actions/install-meltingpot/action.yml b/.github/actions/install-meltingpot/action.yml new file mode 100644 index 000000000..f53a3e822 --- /dev/null +++ b/.github/actions/install-meltingpot/action.yml @@ -0,0 +1,61 @@ +name: install-meltingpot + +inputs: + python-version: + description: Python version + required: false + default: '3.10' + type: string + +runs: + using: composite + steps: + - name: Set up Python + uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 + with: + python-version: ${{ inputs.python-version }} + + - name: Restore install cache + id: cache-restore + uses: actions/cache/restore@v3 + with: + path: | + assets + meltingpot/assets + venv + key: install-meltingpot-${{ runner.name }}-py${{ inputs.python-version}}-${{ hashFiles('setup.py') }} + restore-keys: + install-meltingpot-${{ runner.name }}-py${{ inputs.python-version }} + + - name: Debug + shell: bash + run: | + echo "cache-hit=${{ steps.cache-restore.outputs.cache-hit }}" + pwd + ls -la + ls -la meltingpot + + - name: Install Python dependencies + if: steps.cache-restore.outputs.cache-hit == 'false' + shell: bash + run: | + pip install --upgrade pip + pip install virtualenv + virtualenv venv + source venv/bin/activate + pip install -e .[dev] + + - name: Save install cache + uses: actions/cache/save@v3 + with: + path: | + assets + venv + meltingpot/assets + key: ${{ steps.cache-restore.outputs.cache-primary-key }} + + - name: Show installed dependencies + shell: bash + run: | + source venv/bin/activate + pip list diff --git a/.github/workflows/pylint-examples.yml b/.github/workflows/pylint-examples.yml new file mode 100644 index 000000000..17dfe0354 --- /dev/null +++ b/.github/workflows/pylint-examples.yml @@ -0,0 +1,43 @@ +name: pylint-examples + +on: + push: + branches: + - main + paths: + - '.github/actions/install-examples/action.yml' + - '.github/actions/install-meltingpot/action.yml' + - '.github/workflows/pylint-examples.yml' + - '.pylintrc' + - 'examples/**' + - 'meltingpot/**' + pull_request: + branches: + - main + paths: + - '.github/actions/install-examples/action.yml' + - '.github/actions/install-meltingpot/action.yml' + - '.github/workflows/pylint-examples.yml' + - '.pylintrc' + - 'examples/**' + - 'meltingpot/**' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +permissions: read-all + +jobs: + pylint: + name: Lint examples + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout Melting Pot + uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 + - name: Install examples + uses: ./.github/actions/install-examples + - name: Run PyLint on examples + run: pylint --errors-only examples diff --git a/.github/workflows/pylint-meltingpot.yml b/.github/workflows/pylint-meltingpot.yml new file mode 100644 index 000000000..053f03d83 --- /dev/null +++ b/.github/workflows/pylint-meltingpot.yml @@ -0,0 +1,41 @@ +name: pylint-meltingpot + +on: + push: + branches: + - main + paths: + - '.github/actions/install-meltingpot/action.yml' + - '.github/workflows/pylint-meltingpot.yml' + - '.pylintrc' + - 'examples/**' + - 'meltingpot/**' + pull_request: + branches: + - main + paths: + - '.github/actions/install-meltingpot/action.yml' + - '.github/workflows/pylint-meltingpot.yml' + - '.pylintrc' + - 'examples/**' + - 'meltingpot/**' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +permissions: read-all + +jobs: + pylint: + name: Lint Melting Pot + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout Melting Pot + uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 + - name: Install Melting Pot + uses: ./.github/actions/install-meltingpot + - name: Run PyLint on Melting Pot + run: pylint --errors-only meltingpot diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml deleted file mode 100644 index 686c99615..000000000 --- a/.github/workflows/pylint.yml +++ /dev/null @@ -1,47 +0,0 @@ -# Continuous integration tests. - -name: pylint - -on: - push: - branches: - - main - paths: - - '.github/workflows/pylint.yml' - - '.pylintrc' - - 'examples/**' - - 'meltingpot/**' - pull_request: - branches: - - main - paths: - - '.github/workflows/pylint.yml' - - '.pylintrc' - - 'examples/**' - - 'meltingpot/**' - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} - cancel-in-progress: true - -permissions: read-all - -jobs: - pylint: - name: Lint Melting Pot - runs-on: ubuntu-latest - timeout-minutes: 5 - steps: - - name: Checkout Melting Pot - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 - - name: Set up Python - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 - with: - python-version: '3.10' - - name: Install Python dependencies - run: | - pip install --upgrade pip - pip install pylint - - name: Run PyLint - run: pylint --disable=C,W,R --recursive=y meltingpot examples diff --git a/.github/workflows/pytype-examples.yml b/.github/workflows/pytype-examples.yml new file mode 100644 index 000000000..328476d84 --- /dev/null +++ b/.github/workflows/pytype-examples.yml @@ -0,0 +1,43 @@ +name: pytype-examples + +on: + push: + branches: + - main + paths: + - '.github/actions/install-examples/action.yml' + - '.github/actions/install-meltingpot/action.yml' + - '.github/workflows/pytype-examples.yml' + - 'examples/**' + - 'meltingpot/**' + - 'pyproject.toml' + pull_request: + branches: + - main + paths: + - '.github/actions/install-examples/action.yml' + - '.github/actions/install-meltingpot/action.yml' + - '.github/workflows/pytype-examples.yml' + - 'examples/**' + - 'meltingpot/**' + - 'pyproject.toml' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +permissions: read-all + +jobs: + pytype: + name: Typecheck examples + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout Melting Pot + uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 + - name: Install examples + uses: ./.github/actions/install-examples + - name: Run PyType on examples + run: pytype examples diff --git a/.github/workflows/pytype-meltingpot.yml b/.github/workflows/pytype-meltingpot.yml new file mode 100644 index 000000000..19fbc9831 --- /dev/null +++ b/.github/workflows/pytype-meltingpot.yml @@ -0,0 +1,41 @@ +name: pytype-meltingpot + +on: + push: + branches: + - main + paths: + - '.github/actions/install-meltingpot/action.yml' + - '.github/workflows/pytype-meltingpot.yml' + - 'examples/**' + - 'meltingpot/**' + - 'pyproject.toml' + pull_request: + branches: + - main + paths: + - '.github/actions/install-meltingpot/action.yml' + - '.github/workflows/pytype-meltingpot.yml' + - 'examples/**' + - 'meltingpot/**' + - 'pyproject.toml' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +permissions: read-all + +jobs: + pytype: + name: Typecheck Melting Pot + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout Melting Pot + uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 + - name: Install Melting Pot + uses: ./.github/actions/install-meltingpot + - name: Run PyType on Melting Pot + run: pytype meltingpot diff --git a/.github/workflows/pytype.yml b/.github/workflows/pytype.yml deleted file mode 100644 index fd1b22ea2..000000000 --- a/.github/workflows/pytype.yml +++ /dev/null @@ -1,55 +0,0 @@ -# Continuous integration tests. - -name: pytype - -on: - push: - branches: - - main - paths: - - '.github/workflows/pytype.yml' - - 'examples/**' - - 'meltingpot/**' - - 'pyproject.toml' - pull_request: - branches: - - main - paths: - - '.github/workflows/pytype.yml' - - 'examples/**' - - 'meltingpot/**' - - 'pyproject.toml' - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} - cancel-in-progress: true - -permissions: read-all - -jobs: - pytype: - name: Typecheck Melting Pot - runs-on: ubuntu-latest - timeout-minutes: 30 - strategy: - fail-fast: ${{ github.event_name != 'workflow_dispatch' }} - steps: - - name: Checkout Melting Pot - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 - - name: Set up Python - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 - with: - python-version: '3.10' - - name: Install Python dependencies - run: | - pip install --upgrade pip - pip install setuptools pytype - - name: Install Melting Pot - run: pip -vvv install . - - name: Run PyType on Melting Pot - run: pytype meltingpot - - name: Install example requirements - run: pip -vvv install -r examples/requirements.txt - - name: Run PyType on examples - run: pytype examples diff --git a/.github/workflows/test-examples.yml b/.github/workflows/test-examples.yml index 1bd4033b3..35284bf7d 100644 --- a/.github/workflows/test-examples.yml +++ b/.github/workflows/test-examples.yml @@ -1,5 +1,3 @@ -# Continuous integration tests. - name: test-examples on: @@ -7,18 +5,24 @@ on: branches: - main paths: + - '.github/actions/install-examples/action.yml' + - '.github/actions/install-meltingpot/action.yml' - '.github/workflows/test-examples.yml' - 'examples/**' - 'meltingpot/**' - 'setup.py' + - 'pyproject.toml' pull_request: branches: - main paths: + - '.github/actions/install-examples.yml' + - '.github/actions/install-meltingpot.yml' - '.github/workflows/test-examples.yml' - 'examples/**' - 'meltingpot/**' - 'setup.py' + - 'pyproject.toml' workflow_dispatch: concurrency: @@ -31,27 +35,11 @@ jobs: test-examples: name: Test examples runs-on: ubuntu-latest - env: - SYSTEM_VERSION_COMPAT: 0 # See https://github.com/actions/setup-python/issues/279. timeout-minutes: 90 steps: - name: Checkout Melting Pot - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 - - name: Set up Python - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 - with: - python-version: '3.10' - - name: Install Python dependencies - run: | - pip install --upgrade pip - pip install setuptools pytest-xdist - - name: Install Melting Pot - run: | - pip -vvv install . - pip list + uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 - name: Install examples - run: | - pip install -r examples/requirements.txt - pip list + uses: ./.github/actions/install-examples - name: Test examples run: pytest examples diff --git a/.github/workflows/test-meltingpot.yml b/.github/workflows/test-meltingpot.yml index 744b0d66c..d174bf2b1 100644 --- a/.github/workflows/test-meltingpot.yml +++ b/.github/workflows/test-meltingpot.yml @@ -1,5 +1,3 @@ -# Continuous integration tests. - name: test-meltingpot on: @@ -7,16 +5,20 @@ on: branches: - main paths: + - '.github/actions/install-meltingpot/action.yml' - '.github/workflows/test-meltingpot.yml' - 'meltingpot/**' - 'setup.py' + - 'pyproject.toml' pull_request: branches: - main paths: + - '.github/actions/install-meltingpot/action.yml' - '.github/workflows/test-meltingpot.yml' - 'meltingpot/**' - 'setup.py' + - 'pyproject.toml' workflow_dispatch: concurrency: @@ -26,7 +28,7 @@ concurrency: permissions: read-all jobs: - test-meltingpot: + pytest: name: Test Melting Pot runs-on: ${{ matrix.os }} env: @@ -41,17 +43,11 @@ jobs: python-version: - '3.10' steps: - - name: Set up Python - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 + - name: Checkout Melting Pot + uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 + - name: Install Melting Pot + uses: ./.github/actions/install-meltingpot with: python-version: ${{ matrix.python-version }} - - name: Install Python dependencies - run: | - pip install --upgrade pip - pip install setuptools pytest-xdist - - name: Install Melting Pot - run: | - pip -vvv install https://github.com/deepmind/meltingpot/archive/${{ github.sha }}.tar.gz - pip list - name: Test Melting Pot - run: pytest --pyargs meltingpot + run: pytest meltingpot diff --git a/meltingpot/testing/mocks_test.py b/meltingpot/testing/mocks_test.py index 166ad3456..7b94b5725 100644 --- a/meltingpot/testing/mocks_test.py +++ b/meltingpot/testing/mocks_test.py @@ -30,8 +30,8 @@ def test_value_from_specs(self): ) actual = mocks._values_from_specs(specs) expected = ( - {'a': np.zeros([1, 2, 3], dtype=np.uint8) + 0}, - {'b': np.zeros([1, 2, 3], dtype=np.uint8) + 1}, + {'a': np.zeros([1, 2, 3], dtype=np.uint8)}, + {'b': np.ones([1, 2, 3], dtype=np.uint8)}, ) np.testing.assert_equal(actual, expected) @@ -44,7 +44,10 @@ def test_mock_substrate(self): num_actions=num_actions, observation_spec=observation_spec) - expected_observation = ({'a': np.uint8()}, {'a': np.uint8() + 1}) + expected_observation = ( + {'a': np.zeros([], dtype=np.uint8)}, + {'a': np.ones([], dtype=np.uint8)}, + ) expected_reward = tuple(float(n) for n in range(num_players)) with self.subTest('is_substrate'): diff --git a/meltingpot/utils/policies/saved_model_policy.py b/meltingpot/utils/policies/saved_model_policy.py index 8df461f39..01fd4ea04 100644 --- a/meltingpot/utils/policies/saved_model_policy.py +++ b/meltingpot/utils/policies/saved_model_policy.py @@ -136,7 +136,7 @@ def __init__(self, model_path: str, device_name: str = 'cpu') -> None: @contextlib.contextmanager def _build_context(self): - with self._graph.as_default(): + with self._graph.as_default(): # pylint: disable=not-context-manager with tf.compat.v1.device(self._device_name): yield