repo setup

.gitignore

@@ -1,3 +1,11 @@
+# setuptools_scm
+helx/version.py
+
+# vscode
+.vscode/
+wandb/
+.DS_Store
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
@@ -20,6 +28,7 @@ parts/
 sdist/
 var/
 wheels/
+pip-wheel-metadata/
 share/python-wheels/
 *.egg-info/
 .installed.cfg
@@ -49,7 +58,6 @@ coverage.xml
 *.py,cover
 .hypothesis/
 .pytest_cache/
-cover/
 
 # Translations
 *.mo
@@ -72,7 +80,6 @@ instance/
 docs/_build/
 
 # PyBuilder
-.pybuilder/
 target/
 
 # Jupyter Notebook
@@ -83,9 +90,7 @@ profile_default/
 ipython_config.py
 
 # pyenv
-#   For a library or package, you might want to ignore these files since the code is
-#   intended to run in multiple environments; otherwise, check them in:
-# .python-version
+.python-version
 
 # pipenv
 #   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
@@ -94,22 +99,7 @@ ipython_config.py
 #   install all needed dependencies.
 #Pipfile.lock
 
-# poetry
-#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
-#   This is especially recommended for binary packages to ensure reproducibility, and is more
-#   commonly ignored for libraries.
-#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
-#poetry.lock
-
-# pdm
-#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
-#pdm.lock
-#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
-#   in version control.
-#   https://pdm.fming.dev/#use-with-ide
-.pdm.toml
-
-# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
 __pypackages__/
 
 # Celery stuff
@@ -145,16 +135,3 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
-
-# pytype static type analyzer
-.pytype/
-
-# Cython debug symbols
-cython_debug/
-
-# PyCharm
-#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
-#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
-#  and can be added to the global gitignore or merged into this file.  For a more nuclear
-#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/

AUTHORS

@@ -0,0 +1 @@
+Eduardo Pignatelli

COPYRIGHT

@@ -0,0 +1,18 @@
+Copyright 2023 The Helx Authors.
+
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.

LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright [yyyy] [name of copyright owner]
+   Copyright [2023] The Helx Authors.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

NOTICE

@@ -0,0 +1,5 @@
+Apache epignatelli/helx
+Copyright 2023 The Helx Authors.
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).

README.md

@@ -1,2 +1,170 @@
-# navix
-Minigrid in jax
+[![CI](https://github.com/epignatelli/helx/actions/workflows/CI.yml/badge.svg)](https://github.com/epignatelli/helx/actions/workflows/CI.yml)
+[![CD](https://github.com/epignatelli/helx/actions/workflows/CD.yml/badge.svg)](https://github.com/epignatelli/helx/actions/workflows/CD.yml)
+
+![GitHub release (latest by date)](https://img.shields.io/github/v/release/epignatelli/helx?color=%23216477&label=Release)
+![PyPI - Python Version](https://img.shields.io/pypi/pyversions/helx)
+![PyPI - Wheel](https://img.shields.io/pypi/wheel/helx)
+
+[![Project Status: WIP – Initial development is in progress, but there has not yet been a stable, usable release suitable for the public.](https://www.repostatus.org/badges/latest/wip.svg)](https://www.repostatus.org/#wip)
+[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
+
+--------------
+
+# Helx
+
+Helx aims to excel at two things:
+
+*1.* To **interoperate** between a variety of Reinforcement Learning (RL) environments and Deep Learning libraries with a **single interface**.
+For example, you can use `gym` with a `pytorch`, `jax` or `tensofrlow` agent, and reuse the same agents with a `bsuite` or a `dm_control` environment **without changing your code**.
+
+*2.* To easily **implement** a new Deep RL algorithm that one can plug-and-play in the ecosystem described above.
+
+
+## Installation
+```bash
+pip install git+https://github.com/epignatelli/helx
+```
+
+If you also want to download the binaries for `mujoco`, both `gym` and `dm_control`, and `atari`:
+```bash
+helx-download-extras
+```
+
+And then tell the system where the mujoco binaries are:
+```bash
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/path/to/mujoco/lib
+export MJLIB_PATH=/path/to/home/.mujoco/mujoco210/bin/libmujoco210.so
+export MUJOCO_PY_MUJOCO_PATH=/path/to/home/.mujoco/mujoco210
+```
+
+---
+## Example
+
+A typical use case is to design an agent, and toy-test it on `catch` before evaluating it on more complex environments, such as atari, procgen or mujoco.
+
+```python
+import bsuite
+import gym
+
+import helx.environment
+import helx.experiment
+import helx.agents
+
+# create the enviornment in you favourite way
+env = bsuite.load_from_id("catch/0")
+# convert it to an helx environment
+env = helx.environment.to_helx(env)
+# create the agent
+hparams = helx.agents.Hparams(env.obs_space(), env.action_space())
+agent = helx.agents.Random(hparams)
+
+# run the experiment
+helx.experiment.run(env, agent, episodes=100)
+```
+
+
+Switching to a different environment is as simple as changing the `env` variable.
+
+
+```diff
+import bsuite
+import gym
+
+import helx.environment
+import helx.experiment
+import helx.agents
+
+# create the enviornment in you favourite way
+-env = bsuite.load_from_id("catch/0")
++env = gym.make("procgen:procgen-coinrun-v0")
+# convert it to an helx environment
+env = helx.environment.to_helx(env)
+# create the agent
+hparams = helx.agents.Hparams(env.obs_space(), env.action_space())
+agent = helx.agents.Random(hparams)
+
+# run the experiment
+helx.experiment.run(env, agent, episodes=100)
+```
+
+---
+## Supported libraries
+
+We currently support these external environment models:
+- [dm_env](https://github.com/deepmind/dm_env)
+- [bsuite](https://github.com/deepmind/bsuite)
+- [dm_control](https://github.com/deepmind/dm_control), including
+  - [Mujoco](https://mujoco.org)
+- [gym](https://github.com/openai/gym) and [gymnasium](https://github.com/Farama-Foundation/Gymnasium), including
+  - The [minigrid]() family
+  - The [minihack]() family
+  - The [atari](https://github.com/mgbellemare/Arcade-Learning-Environment) family
+  - The legacy [mujoco](https://www.roboti.us/download.html) family
+  - And the standard gym family
+- [gym3](https://github.com/openai/gym3), including
+  - [procgen](https://github.com/openai/procgen)
+
+#### On the road:
+- [gymnax](https://github.com/RobertTLange/gymnax)
+- [ivy_gym](https://github.com/unifyai/gym)
+---
+## Adding a new agent (`helx.agents.Agent`)
+
+An `helx` agent interface is designed as the minimal set of functions necessary to *(i)* interact with an environment and *(ii)* reinforcement learn.
+
+```python
+class Agent(ABC):
+    """A minimal RL agent interface."""
+
+    @abstractmethod
+    def sample_action(self, timestep: Timestep) -> Action:
+        """Applies the agent's policy to the current timestep to sample an action."""
+
+    @abstractmethod
+    def update(self, timestep: Timestep) -> Any:
+        """Updates the agent's internal state (knowledge), such as a table,
+        or some function parameters, e.g., the parameters of a neural network."""
+```
+
+---
+## Adding a new environment library (`helx.environment.Environment`)
+
+To add a new library requires three steps:
+1. Implement the `helx.environment.Environment` interface for the new library.
+See the [dm_env](helx/environment/dm_env.py) implementation for an example.
+1. Implement serialisation (to `helx`) of the following objects:
+    - `helx.environment.Timestep`
+    - `helx.spaces.Discrete`
+    - `helx.spaces.Continuous`
+2. Add the new library to the [`helx.environment.to_helx`](helx/environment/interop.py#L16) function to tell `helx` about the new protocol.
+
+---
+## Cite
+If you use `helx` please consider citing it as:
+
+```bibtex
+@misc{helx,
+  author = {Pignatelli, Eduardo},
+  title = {Helx: Interoperating between Reinforcement Learning Experimental Protocols},
+  year = {2021},
+  publisher = {GitHub},
+  journal = {GitHub repository},
+  howpublished = {\url{https://github.com/epignatelli/helx}}
+  }
+```
+
+---
+## A note on maintainance
+This repository was born as the recipient of personal research code that was developed over the years.
+Its maintainance is limited by the time and the resources of a research project resourced with a single person.
+Even if I would like to automate many actions, I do not have the time to maintain the whole body of automation that a well maintained package deserves.
+This is the reason of the WIP badge, which I do not plan to remove soon.
+Maintainance will prioritise the code functionality over documentation and automation.
+
+Any help is very welcome.
+A quick guide to interacting with this repository:
+- If you find a bug, please open an issue, and I will fix it as soon as I can.
+- If you want to request a new feature, please open an issue, and I will consider it as soon as I can.
+- If you want to contribute yourself, please open an issue first, let's discuss objective, plan a proposal, and open a pull request to act on it.
+
+If you would like to be involved further in the development of this repository, please contact me directly at: `edu dot pignatelli at gmail dot com`.

pyproject.toml

@@ -0,0 +1,94 @@
+# This toml file is in comliance with PEP 517, PEP 518 and PEP 621
+# However, setuptools support for these is only in Beta
+# For now, `setup.py` is still the first option for pip
+[build-system]
+build-backend = "setuptools.build_meta"
+requires = ["setuptools >= 50", "setuptools-scm[toml]>=6.2", "wheel"]
+
+[project]
+name = "Helx"
+dynamic = ["version", "dependencies"]
+description = "Interoperate among reinforcement learning libraries with jax, pytorch, gym and dm_env"
+requires-python = ">=3.9"
+readme = "README.md"
+license = {file = "LICENSE", name = "Apache-2.0"}
+authors = [
+  {name = "Eduardo Pignatelli", email = "[email protected]"},
+]
+maintainers = [
+  {name = "Eduardo Pignatelli", email = "[email protected]"},
+]
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "Intended Audience :: Developers",
+  "Intended Audience :: Science/Research",
+  "Intended Audience :: Financial and Insurance Industry",
+  "Intended Audience :: Healthcare Industry",
+  "Environment :: GPU",
+  "Environment :: GPU :: NVIDIA CUDA",
+  "Environment :: GPU :: NVIDIA CUDA :: 11.0",
+  "Environment :: GPU :: NVIDIA CUDA :: 11.1",
+  "Environment :: GPU :: NVIDIA CUDA :: 11.2",
+  "Environment :: GPU :: NVIDIA CUDA :: 11.3",
+  "Environment :: GPU :: NVIDIA CUDA :: 11.4",
+  "Environment :: GPU :: NVIDIA CUDA :: 11.5",
+  "Environment :: GPU :: NVIDIA CUDA :: 11.6",
+  "Environment :: GPU :: NVIDIA CUDA :: 11.7",
+  "Environment :: GPU :: NVIDIA CUDA :: 11.8",
+  "Framework :: Pytest",
+  "License :: OSI Approved :: Apache Software License",
+  "Operating System :: OS Independent",
+  "Programming Language :: Python",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.9",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Topic :: Scientific/Engineering",
+  "Topic :: Scientific/Engineering :: Artificial Intelligence",
+  "Topic :: Software Development",
+  "Topic :: Software Development :: Libraries",
+  "Topic :: Software Development :: Libraries :: Python Modules",
+]
+
+
+[project.urls]
+homepage = "https://github.com/epignatelli/helx"
+repository = "https://github.com/epignatelli/helx"
+bug_tracker = "https://github.com/epignatelli/helx/issues"
+
+
+[project.optional-dependencies]
+atari = [
+    "AutoROM.accept-rom-license==0.4.2"
+    ]
+mujoco = [
+
+  ]
+all = [
+    "AutoROM.accept-rom-license==0.4.2"
+    ]
+
+
+[project.scripts]
+helx-download-extras = "helx.download:download_all"
+
+
+[tool.setuptools.dynamic]
+dependencies = {file = "./requirements.txt"}
+
+
+[tool.setuptools.packages.find]
+include = ["helx*"]
+exclude = ["test", "examples"]
+
+
+[tool.distutils.bdist_wheel]
+universal = true
+
+
+[tool.setuptools_scm]
+write_to = "helx/version.py"
+
+
+[tool.black]
+line-length = 88