diff --git a/.github/actions/build-image/action.yml b/.github/actions/build-image/action.yml
new file mode 100644
index 0000000..efa0aba
--- /dev/null
+++ b/.github/actions/build-image/action.yml
@@ -0,0 +1,48 @@
+inputs:
+ dockerhub_username:
+ description: "The DockerHub username."
+ required: true
+
+ dockerhub_token:
+ description: "The DockerHub login token."
+ required: true
+
+ docker_file:
+ description: "The docker file."
+ required: true
+
+ tags:
+ description: "Image tags (csv)."
+ required: true
+
+ platforms:
+ description: "Platforms to build for (csv)."
+ default: "linux/amd64,linux/arm64"
+
+ push_image:
+ description: "Whether to push to DockerHub."
+ default: false
+
+runs:
+ using: "composite"
+ steps:
+ - name: Set up QEMU 🌈
+ uses: docker/setup-qemu-action@v3
+
+ - name: Set up Docker Buildx ✨
+ uses: docker/setup-buildx-action@v3
+
+ - name: Login to Docker Hub 🎪
+ uses: docker/login-action@v3
+ with:
+ username: ${{ inputs.dockerhub_username }}
+ password: ${{ inputs.dockerhub_token }}
+
+ - name: Build and deploy image 🐳
+ uses: docker/build-push-action@v5
+ with:
+ context: docker
+ file: ${{ inputs.docker_file }}
+ platforms: ${{ inputs.platforms }}
+ push: ${{ inputs.push_image }}
+ tags: ${{ inputs.tags }}
diff --git a/.github/workflows/deploy_images.yml b/.github/workflows/deploy_images.yml
index e7a334b..bdc109a 100644
--- a/.github/workflows/deploy_images.yml
+++ b/.github/workflows/deploy_images.yml
@@ -3,167 +3,185 @@ name: Deploy images
on:
workflow_dispatch:
inputs:
- skip_push:
- description: Skip pushing images? (true|false)
- required: false
- default: 'false'
- skip_docs:
- description: Skip the docs image? (true|false)
- required: false
- default: 'false'
+ push_image:
+ description: Push to DockerHub
+ type: boolean
+ default: false
+
+ specific_job:
+ description: Specific job to run
+ type: choice
+ default: all
+ options:
+ - all
+ - py37
+ - py38
+ - py39
+ - py310
+ - py311
+ - py311-cuda
+ - example
+
+ platforms:
+ description: Platforms to build for
+ type: choice
+ default: linux/amd64,linux/arm64
+ options:
+ - linux/amd64,linux/arm64
+ - linux/amd64
+ - linux/arm64
jobs:
py37:
+ if: ${{ github.event.inputs.specific_job == 'all' || contains(github.event.inputs.specific_job, 'py37') }}
runs-on: ubuntu-latest
steps:
- name: Checkout 🛎️
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
with:
persist-credentials: false
- name: Build and deploy image 🐳
- uses: docker/build-push-action@v1
- with:
- path: docker
- dockerfile: docker/Dockerfile_37
- repository: cmsml/cmsml
- tags: "3.7"
- push: ${{ github.event.inputs.skip_push != 'true' }}
- username: ${{ secrets.DOCKERHUB_USERNAME }}
- password: ${{ secrets.DOCKERHUB_TOKEN }}
+ uses: ./.github/actions/build-image
+ with:
+ dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
+ dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
+ docker_file: docker/Dockerfile_37
+ platforms: ${{ github.event.inputs.platforms }}
+ tags: cmsml/cmsml:3.7
+ push_image: ${{ github.event.inputs.push_image == 'true' }}
py38:
+ if: ${{ github.event.inputs.specific_job == 'all' || contains(github.event.inputs.specific_job, 'py38') }}
runs-on: ubuntu-latest
steps:
- name: Checkout 🛎️
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
with:
persist-credentials: false
- name: Build and deploy image 🐳
- uses: docker/build-push-action@v1
- with:
- path: docker
- dockerfile: docker/Dockerfile_38
- repository: cmsml/cmsml
- tags: "3.8"
- push: ${{ github.event.inputs.skip_push != 'true' }}
- username: ${{ secrets.DOCKERHUB_USERNAME }}
- password: ${{ secrets.DOCKERHUB_TOKEN }}
+ uses: ./.github/actions/build-image
+ with:
+ dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
+ dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
+ docker_file: docker/Dockerfile_38
+ platforms: ${{ github.event.inputs.platforms }}
+ tags: cmsml/cmsml:3.8
+ push_image: ${{ github.event.inputs.push_image == 'true' }}
py39:
+ if: ${{ github.event.inputs.specific_job == 'all' || contains(github.event.inputs.specific_job, 'py39') }}
runs-on: ubuntu-latest
steps:
- name: Checkout 🛎️
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
with:
persist-credentials: false
- name: Build and deploy image 🐳
- uses: docker/build-push-action@v1
- with:
- path: docker
- dockerfile: docker/Dockerfile_39
- repository: cmsml/cmsml
- tags: 3.9,3,latest
- push: ${{ github.event.inputs.skip_push != 'true' }}
- username: ${{ secrets.DOCKERHUB_USERNAME }}
- password: ${{ secrets.DOCKERHUB_TOKEN }}
+ uses: ./.github/actions/build-image
+ with:
+ dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
+ dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
+ docker_file: docker/Dockerfile_39
+ platforms: ${{ github.event.inputs.platforms }}
+ tags: cmsml/cmsml:3.9,cmsml/cmsml:3,cmsml/cmsml:latest
+ push_image: ${{ github.event.inputs.push_image == 'true' }}
py39_base:
+ if: ${{ github.event.inputs.specific_job == 'all' || contains(github.event.inputs.specific_job, 'py39') }}
runs-on: ubuntu-latest
- if: ${{ github.event.inputs.skip_push != 'true' }}
steps:
- name: Checkout 🛎️
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
with:
persist-credentials: false
- name: Build and deploy image 🐳
- uses: docker/build-push-action@v1
- with:
- path: docker
- dockerfile: docker/Dockerfile_39_base
- repository: cmsml/cmsml
- tags: 3.9_base
- push: ${{ github.event.inputs.skip_push != 'true' }}
- username: ${{ secrets.DOCKERHUB_USERNAME }}
- password: ${{ secrets.DOCKERHUB_TOKEN }}
+ uses: ./.github/actions/build-image
+ with:
+ dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
+ dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
+ docker_file: docker/Dockerfile_39_base
+ platforms: ${{ github.event.inputs.platforms }}
+ tags: cmsml/cmsml:3.9_base
+ push_image: ${{ github.event.inputs.push_image == 'true' }}
py310:
+ if: ${{ github.event.inputs.specific_job == 'all' || contains(github.event.inputs.specific_job, 'py310') }}
runs-on: ubuntu-latest
steps:
- name: Checkout 🛎️
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
with:
persist-credentials: false
- name: Build and deploy image 🐳
- uses: docker/build-push-action@v1
- with:
- path: docker
- dockerfile: docker/Dockerfile_310
- repository: cmsml/cmsml
- tags: "3.10"
- push: ${{ github.event.inputs.skip_push != 'true' }}
- username: ${{ secrets.DOCKERHUB_USERNAME }}
- password: ${{ secrets.DOCKERHUB_TOKEN }}
+ uses: ./.github/actions/build-image
+ with:
+ dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
+ dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
+ docker_file: docker/Dockerfile_310
+ platforms: ${{ github.event.inputs.platforms }}
+ tags: cmsml/cmsml:3.10
+ push_image: ${{ github.event.inputs.push_image == 'true' }}
py311:
+ if: ${{ github.event.inputs.specific_job == 'all' || contains(github.event.inputs.specific_job, 'py311') }}
runs-on: ubuntu-latest
steps:
- name: Checkout 🛎️
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
with:
persist-credentials: false
- name: Build and deploy image 🐳
- uses: docker/build-push-action@v1
- with:
- path: docker
- dockerfile: docker/Dockerfile_311
- repository: cmsml/cmsml
- tags: "3.11"
- push: ${{ github.event.inputs.skip_push != 'true' }}
- username: ${{ secrets.DOCKERHUB_USERNAME }}
- password: ${{ secrets.DOCKERHUB_TOKEN }}
-
+ uses: ./.github/actions/build-image
+ with:
+ dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
+ dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
+ docker_file: docker/Dockerfile_311
+ platforms: ${{ github.event.inputs.platforms }}
+ tags: cmsml/cmsml:3.11
+ push_image: ${{ github.event.inputs.push_image == 'true' }}
+
py311-cuda:
+ if: ${{ github.event.inputs.specific_job == 'all' || contains(github.event.inputs.specific_job, 'py311-cuda') }}
runs-on: ubuntu-latest
steps:
- name: Checkout 🛎️
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
with:
persist-credentials: false
- name: Build and deploy image 🐳
- uses: docker/build-push-action@v1
- with:
- path: docker
- dockerfile: docker/Dockerfile_311_cuda
- repository: cmsml/cmsml
- tags: "3.11-cuda"
- push: ${{ github.event.inputs.skip_push != 'true' }}
- username: ${{ secrets.DOCKERHUB_USERNAME }}
- password: ${{ secrets.DOCKERHUB_TOKEN }}
-
+ uses: ./.github/actions/build-image
+ with:
+ dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
+ dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
+ docker_file: docker/Dockerfile_311_cuda
+ # gpu / cuda only available on x86, not on arm
+ platforms: linux/amd64
+ tags: cmsml/cmsml:3.11-cuda
+ push_image: ${{ github.event.inputs.push_image == 'true' }}
+
docs:
+ if: ${{ github.event.inputs.push_image == 'true' && (github.event.inputs.specific_job == 'all' || contains(github.event.inputs.specific_job, 'example')) }}
needs: py39
runs-on: ubuntu-latest
- if: ${{ github.event.inputs.skip_docs != 'true' && github.event.inputs.skip_push != 'true' }}
steps:
- name: Checkout 🛎️
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
with:
persist-credentials: false
- name: Build and deploy image 🐳
- uses: docker/build-push-action@v1
- with:
- path: docker
- dockerfile: docker/Dockerfile_docs
- repository: cmsml/cmsml
- tags: docs
- push: ${{ github.event.inputs.skip_push != 'true' }}
- username: ${{ secrets.DOCKERHUB_USERNAME }}
- password: ${{ secrets.DOCKERHUB_TOKEN }}
+ uses: ./.github/actions/build-image
+ with:
+ dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
+ dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
+ docker_file: docker/Dockerfile_docs
+ platforms: ${{ github.event.inputs.platforms }}
+ tags: cmsml/cmsml:docs
+ push_image: true
diff --git a/.github/workflows/lint_and_test.yml b/.github/workflows/lint_and_test.yml
index 5ec6aaa..30f2ce0 100644
--- a/.github/workflows/lint_and_test.yml
+++ b/.github/workflows/lint_and_test.yml
@@ -37,6 +37,7 @@ jobs:
- {tag: "3.9_base", tf: "2.11.1"}
- {tag: "3.9_base", tf: "2.12.1"}
- {tag: "3.9_base", tf: "2.13.0"}
+ - {tag: "3.9_base", tf: "2.16.1"}
name: test (image=${{ matrix.versions.tag }}, tf=${{ matrix.versions.tf }})
steps:
- name: Checkout 🛎️
diff --git a/.gitignore b/.gitignore
index 5fe43c3..2dbcbc1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -19,3 +19,4 @@ build
tmp
docs/_build
.ipynb_checkpoints
+.python-version
diff --git a/README.md b/README.md
index 266a272..000cacb 100644
--- a/README.md
+++ b/README.md
@@ -40,7 +40,7 @@ The documentation of this Python package is hosted on [readthedocs](http://cmsml
**However**, note that this documentation only covers the API and technical aspects of the package itself.
Usage examples and further techniques for working with machine learning tools in CMS, alongside a collection of useful guidelines can be found in the [general CMS ML group documentation](https://cms-ml.github.io/documentation).
-Click [here](https://github.com/cms-ml/cmsml/issues/new?labels=suggestion&template=feature-suggestion.md&) to submit a feature suggestion!
+Click [here](https://github.com/cms-ml/cmsml/issues/new?labels=suggestion&template=feature-suggestion.md) to submit a feature suggestion!
@@ -53,6 +53,15 @@ Click [here](https://github.com/cms-ml/cmsml/issues/new?labels=suggestion&templa
To use the cmsml package via docker, checkout our [DockerHub](https://hub.docker.com/repository/docker/cmsml/cmsml) which contains tags for several Python versions.
+| Image | Python version | TF Version | PyTorch Version | GPU support |
+| :----------------------------------------------------------- | :------------: | :--------------: | :-------------: | :---------: |
+| `cmsml/cmsml:3.7` | 3.7 | 2.11.1 | 1.13.1 | ✘ |
+| `cmsml/cmsml:3.8` | 3.8 | 2.13.1 | latest (~2.3.0) | ✘ |
+| `cmsml/cmsml:3.9`
`cmsml/cmsml:3`
`cmsml/cmsml:latest` | 3.9 | latest (~2.16.1) | latest (~2.3.0) | ✘ |
+| `cmsml/cmsml:3.10` | 3.10 | latest (~2.16.1) | latest (~2.3.0) | ✘ |
+| `cmsml/cmsml:3.11` | 3.11 | latest (~2.16.1) | latest (~2.3.0) | ✘ |
+| `cmsml/cmsml:3.11-cuda` | 3.11 | latest (~2.16.1) | latest (~2.3.0) | ✔︎ |
+
diff --git a/cmsml/tensorflow/__init__.py b/cmsml/tensorflow/__init__.py
index a6fc657..c6be046 100644
--- a/cmsml/tensorflow/__init__.py
+++ b/cmsml/tensorflow/__init__.py
@@ -6,16 +6,16 @@
"""
__all__ = [
- "import_tf", "save_frozen_graph", "save_graph", "load_frozen_graph", "load_graph",
- "write_graph_summary", "load_model", "load_graph_def",
+ "import_tf", "tf_version_check", "tf_keras_version_check", "save_frozen_graph", "save_graph",
+ "load_frozen_graph", "load_graph", "write_graph_summary", "load_model", "load_graph_def",
"OpsData", "get_graph_ops",
]
# provisioning imports
from cmsml.tensorflow.tools import (
- import_tf, save_frozen_graph, save_graph, load_frozen_graph, load_graph, write_graph_summary,
- load_model, load_graph_def,
+ import_tf, tf_version_check, tf_keras_version_check, save_frozen_graph, save_graph,
+ load_frozen_graph, load_graph, write_graph_summary, load_model, load_graph_def,
)
from cmsml.tensorflow.aot import (
diff --git a/cmsml/tensorflow/tools.py b/cmsml/tensorflow/tools.py
index 08b08a3..20ea707 100644
--- a/cmsml/tensorflow/tools.py
+++ b/cmsml/tensorflow/tools.py
@@ -14,7 +14,7 @@
from typing import Any
from tensorflow.core.framework.graph_pb2 import GraphDef
-from cmsml.util import MockModule
+from cmsml.util import MockModule, _op_map
tf = MockModule("tensorflow")
@@ -71,6 +71,59 @@ def import_tf(
return tf, tf1, tf_version
+def tf_version_check(op: str, version: int | tuple[int, ...]) -> bool:
+ """
+ Compares the installed TensorFlow version with *version* using an operator *op*, which should be
+ any of ``"=="``, ``"!="``, ``"<"``, ``"<="``, ``">"`` or ``">="``. Examples:
+
+ .. code-block:: python
+
+ # actual version is 2.16
+ tf_version_check("==", (2, 16)) # -> True
+ tf_version_check(">", (2, 15)) # -> True
+ tf_version_check("!=", (2, 16)) # -> False
+ """
+ if op not in _op_map:
+ raise ValueError(f"unsupported operator '{op}'")
+ if not isinstance(version, tuple):
+ version = (version,)
+ if len(version) > 3:
+ raise ValueError("version must be at most a 3-tuple")
+ # get the tf version
+ tf_version = import_tf()[2][:len(version)]
+ # comparison
+ return _op_map[op](tf_version, version)
+
+
+def tf_keras_version_check(op: str, version: int | tuple[int, ...]) -> bool:
+ """
+ Compares the installed Keras version shipped with TensorFlow with *version* using an operator
+ *op*, which should be any of ``"=="``, ``"!="``, ``"<"``, ``"<="``, ``">"`` or ``">="``.
+ Examples:
+
+ .. code-block:: python
+
+ # actual version is 3.3
+ keras_version_check("==", (3, 3)) # -> True
+ keras_version_check(">", (3, 2)) # -> True
+ keras_version_check("!=", (3, 3)) # -> False
+ """
+ if op not in _op_map:
+ raise ValueError(f"unsupported operator '{op}'")
+ if not isinstance(version, tuple):
+ version = (version,)
+ if len(version) > 3:
+ raise ValueError("version must be at most a 3-tuple")
+ # get the keras version, which is tf version dependent
+ if tf_version_check(">=", (2, 16)):
+ keras_version = tuple(map(int, import_tf()[0].keras.__version__.split(".", 2)))
+ else:
+ import keras
+ keras_version = tuple(map(int, keras.__version__.split(".", 2)))
+ # comparison
+ return _op_map[op](keras_version, version)
+
+
def save_graph(
path: str,
obj: Any,
@@ -138,16 +191,20 @@ def save_frozen_graph(
from tensorflow.python.eager.function import ConcreteFunction
if isinstance(obj, tf.keras.Model):
- learning_phase_orig = tf.keras.backend.get_value(tf.keras.backend.learning_phase())
- tf.keras.backend.set_learning_phase(False)
- model_func = saving_utils.trace_model_call(obj)
- if model_func.function_spec.arg_names and not model_func.input_signature:
- raise ValueError(
- "when obj is a keras model callable accepting arguments, its "
- "input signature must be frozen by building the model",
- )
- obj = model_func.get_concrete_function()
- tf.keras.backend.set_learning_phase(learning_phase_orig)
+ if tf_keras_version_check(">=", 3):
+ obj = obj._default_save_signature.get_concrete_function()
+ else:
+ # set the learning phase to False for the duration of the export for keras <= 3
+ learning_phase_orig = tf.keras.backend.get_value(tf.keras.backend.learning_phase())
+ tf.keras.backend.set_learning_phase(False)
+ model_func = saving_utils.trace_model_call(obj)
+ if model_func.function_spec.arg_names and not model_func.input_signature:
+ raise ValueError(
+ "when obj is a keras model callable accepting arguments, its "
+ "input signature must be frozen by building the model",
+ )
+ obj = model_func.get_concrete_function()
+ tf.keras.backend.set_learning_phase(learning_phase_orig)
elif isinstance(obj, Function):
if obj.function_spec.arg_names and not obj.input_signature:
@@ -295,8 +352,8 @@ def load_graph_def(
serving_key: str = tf.saved_model.DEFAULT_SERVING_SIGNATURE_DEF_KEY,
) -> GraphDef:
"""
- Loads the model saved at *model_path* and returns the GraphDef of it. Supported input types are tensorflow and keras
- SavedModels, as well as frozen graphs.
+ Loads the model saved at *model_path* and returns the GraphDef of it. Supported input types are
+ tensorflow and keras SavedModels, as well as frozen graphs.
"""
tf, tf1, tf_version = import_tf()
@@ -304,8 +361,6 @@ def load_graph_def(
# if model_path is directory try load as saved model
if os.path.isdir(model_path) and tf.saved_model.contains_saved_model(model_path):
- # if keras model try to load as keras model
- # else load as tensorflow saved model
loaded_saved_model = load_model(model_path)
# extract graph
@@ -325,6 +380,17 @@ def load_graph_def(
return graph_def
+ # load from new keras model interface
+ if (
+ tf_keras_version_check(">=", 3) and
+ os.path.isfile(model_path) and
+ model_path.endswith((".keras", ".h5"))
+ ):
+ loaded_keras_model = load_model(model_path)
+ # TODO: no interace yet to choose the serving key
+ concrete_function = loaded_keras_model._default_save_signature.get_concrete_function()
+ return concrete_function.graph.as_graph_def()
+
raise FileNotFoundError(f"{model_path} contains neither frozen graph nor SavedModel")
@@ -337,7 +403,15 @@ def load_model(model_path: str) -> tf.Model:
model_path = os.path.expandvars(os.path.expanduser(str(model_path)))
- if os.path.isdir(model_path) and os.path.exists(os.path.join(model_path, "keras_metadata.pb")):
+ if (
+ (
+ os.path.isdir(model_path) and
+ os.path.exists(os.path.join(model_path, "keras_metadata.pb"))
+ ) or (
+ os.path.isfile(model_path) and
+ model_path.endswith((".keras", ".h5"))
+ )
+ ):
model = tf.keras.models.load_model(model_path)
else:
model = tf.saved_model.load(model_path)
@@ -359,7 +433,6 @@ def write_graph_summary(
.. note::
When used with TensorFlow v1, eager mode must be disabled.
"""
-
# prepare the summary dir
if not os.path.exists(summary_dir):
os.makedirs(summary_dir)
diff --git a/cmsml/util.py b/cmsml/util.py
index e152834..f8e7ac0 100644
--- a/cmsml/util.py
+++ b/cmsml/util.py
@@ -16,6 +16,7 @@
import tempfile
import contextlib
import subprocess
+import operator
import signal
import importlib
import six
@@ -33,6 +34,15 @@
enumerate,
)
+_op_map = {
+ "==": operator.eq,
+ "!=": operator.ne,
+ "<": operator.lt,
+ "<=": operator.le,
+ ">": operator.gt,
+ ">=": operator.ge,
+}
+
def is_lazy_iterable(obj: Any) -> bool:
"""
diff --git a/docker/Dockerfile_310 b/docker/Dockerfile_310
index 541be8f..3fb5112 100644
--- a/docker/Dockerfile_310
+++ b/docker/Dockerfile_310
@@ -5,27 +5,26 @@ WORKDIR /root
# minimal software stack
RUN apt-get update; apt-get clean
-RUN apt-get install -y nano less htop git; apt-get clean
+RUN apt-get install -y nano less htop git libhdf5-serial-dev; apt-get clean
# python software stack
RUN pip install --no-cache-dir --upgrade pip setuptools
RUN pip install --no-cache-dir --upgrade ipython
-RUN pip install --no-cache-dir \
- numpy \
- scipy \
- matplotlib \
- pandas \
- numexpr \
- jupyterlab \
- notebook \
- scikit-learn \
- scikit-optimize \
- tensorflow \
- xgboost \
- scinum \
- nvidia_smi \
- py3nvml \
- torch
+RUN pip install --no-cache-dir numpy
+RUN pip install --no-cache-dir scipy
+RUN pip install --no-cache-dir matplotlib
+RUN pip install --no-cache-dir pandas
+RUN pip install --no-cache-dir numexpr
+RUN pip install --no-cache-dir jupyterlab
+RUN pip install --no-cache-dir notebook
+RUN pip install --no-cache-dir scikit-learn
+RUN pip install --no-cache-dir scikit-optimize
+RUN pip install --no-cache-dir tensorflow
+RUN pip install --no-cache-dir xgboost
+RUN pip install --no-cache-dir scinum
+RUN pip install --no-cache-dir nvidia_smi
+RUN pip install --no-cache-dir py3nvml
+RUN pip install --no-cache-dir torch
# install cmsml from master
RUN git clone https://github.com/cms-ml/cmsml.git && \
diff --git a/docker/Dockerfile_311 b/docker/Dockerfile_311
index 7d80cfe..64b8ca7 100644
--- a/docker/Dockerfile_311
+++ b/docker/Dockerfile_311
@@ -5,27 +5,26 @@ WORKDIR /root
# minimal software stack
RUN apt-get update; apt-get clean
-RUN apt-get install -y nano less htop git; apt-get clean
+RUN apt-get install -y nano less htop git libhdf5-serial-dev; apt-get clean
# python software stack
RUN pip install --no-cache-dir --upgrade pip setuptools
RUN pip install --no-cache-dir --upgrade ipython
-RUN pip install --no-cache-dir \
- numpy \
- scipy \
- matplotlib \
- pandas \
- numexpr \
- jupyterlab \
- notebook \
- scikit-learn \
- scikit-optimize \
- tensorflow \
- xgboost \
- scinum \
- nvidia_smi \
- py3nvml \
- torch
+RUN pip install --no-cache-dir numpy
+RUN pip install --no-cache-dir scipy
+RUN pip install --no-cache-dir matplotlib
+RUN pip install --no-cache-dir pandas
+RUN pip install --no-cache-dir numexpr
+RUN pip install --no-cache-dir jupyterlab
+RUN pip install --no-cache-dir notebook
+RUN pip install --no-cache-dir scikit-learn
+RUN pip install --no-cache-dir scikit-optimize
+RUN pip install --no-cache-dir tensorflow
+RUN pip install --no-cache-dir xgboost
+RUN pip install --no-cache-dir scinum
+RUN pip install --no-cache-dir nvidia_smi
+RUN pip install --no-cache-dir py3nvml
+RUN pip install --no-cache-dir torch
# install cmsml from master
RUN git clone https://github.com/cms-ml/cmsml.git && \
diff --git a/docker/Dockerfile_311_cuda b/docker/Dockerfile_311_cuda
index 8315a38..5988d01 100644
--- a/docker/Dockerfile_311_cuda
+++ b/docker/Dockerfile_311_cuda
@@ -1,4 +1,4 @@
-FROM python:3.11
+FROM tensorflow/tensorflow:2.16.1-gpu
# set the workdir
WORKDIR /root
@@ -20,7 +20,6 @@ RUN pip install --no-cache-dir \
notebook \
scikit-learn \
scikit-optimize \
- tensorflow[and-cuda] \
xgboost \
scinum \
nvidia_smi \
diff --git a/docker/Dockerfile_37 b/docker/Dockerfile_37
index 83fc1db..fa6de36 100644
--- a/docker/Dockerfile_37
+++ b/docker/Dockerfile_37
@@ -5,27 +5,26 @@ WORKDIR /root
# minimal software stack
RUN apt-get update; apt-get clean
-RUN apt-get install -y nano less htop git; apt-get clean
+RUN apt-get install -y nano less htop git libhdf5-serial-dev; apt-get clean
# python software stack
RUN pip install --no-cache-dir --upgrade pip setuptools
RUN pip install --no-cache-dir --upgrade ipython
-RUN pip install --no-cache-dir \
- numpy \
- scipy \
- matplotlib \
- pandas \
- numexpr \
- jupyterlab \
- notebook \
- scikit-learn \
- scikit-optimize \
- tensorflow \
- xgboost \
- scinum \
- nvidia_smi \
- py3nvml \
- torch
+RUN pip install --no-cache-dir numpy
+RUN pip install --no-cache-dir scipy
+RUN pip install --no-cache-dir matplotlib
+RUN pip install --no-cache-dir pandas
+RUN pip install --no-cache-dir numexpr
+RUN pip install --no-cache-dir jupyterlab
+RUN pip install --no-cache-dir notebook
+RUN pip install --no-cache-dir scikit-learn
+RUN pip install --no-cache-dir scikit-optimize
+RUN pip install --no-cache-dir tensorflow
+RUN pip install --no-cache-dir xgboost
+RUN pip install --no-cache-dir scinum
+RUN pip install --no-cache-dir nvidia_smi
+RUN pip install --no-cache-dir py3nvml
+RUN pip install --no-cache-dir torch
# install cmsml from master
RUN git clone https://github.com/cms-ml/cmsml.git && \
diff --git a/docker/Dockerfile_38 b/docker/Dockerfile_38
index 54cc560..e1e96db 100644
--- a/docker/Dockerfile_38
+++ b/docker/Dockerfile_38
@@ -5,27 +5,26 @@ WORKDIR /root
# minimal software stack
RUN apt-get update; apt-get clean
-RUN apt-get install -y nano less htop git; apt-get clean
+RUN apt-get install -y nano less htop git libhdf5-serial-dev; apt-get clean
# python software stack
RUN pip install --no-cache-dir --upgrade pip setuptools
RUN pip install --no-cache-dir --upgrade ipython
-RUN pip install --no-cache-dir \
- numpy \
- scipy \
- matplotlib \
- pandas \
- numexpr \
- jupyterlab \
- notebook \
- scikit-learn \
- scikit-optimize \
- tensorflow \
- xgboost \
- scinum \
- nvidia_smi \
- py3nvml \
- torch
+RUN pip install --no-cache-dir numpy
+RUN pip install --no-cache-dir scipy
+RUN pip install --no-cache-dir matplotlib
+RUN pip install --no-cache-dir pandas
+RUN pip install --no-cache-dir numexpr
+RUN pip install --no-cache-dir jupyterlab
+RUN pip install --no-cache-dir notebook
+RUN pip install --no-cache-dir scikit-learn
+RUN pip install --no-cache-dir scikit-optimize
+RUN pip install --no-cache-dir tensorflow
+RUN pip install --no-cache-dir xgboost
+RUN pip install --no-cache-dir scinum
+RUN pip install --no-cache-dir nvidia_smi
+RUN pip install --no-cache-dir py3nvml
+RUN pip install --no-cache-dir torch
# install cmsml from master
RUN git clone https://github.com/cms-ml/cmsml.git && \
diff --git a/docker/Dockerfile_39 b/docker/Dockerfile_39
index 98a152e..d78f27a 100644
--- a/docker/Dockerfile_39
+++ b/docker/Dockerfile_39
@@ -5,27 +5,26 @@ WORKDIR /root
# minimal software stack
RUN apt-get update; apt-get clean
-RUN apt-get install -y nano less htop git; apt-get clean
+RUN apt-get install -y nano less htop git libhdf5-serial-dev; apt-get clean
# python software stack
RUN pip install --no-cache-dir --upgrade pip setuptools
RUN pip install --no-cache-dir --upgrade ipython
-RUN pip install --no-cache-dir \
- numpy \
- scipy \
- matplotlib \
- pandas \
- numexpr \
- jupyterlab \
- notebook \
- scikit-learn \
- scikit-optimize \
- tensorflow \
- xgboost \
- scinum \
- nvidia_smi \
- py3nvml \
- torch
+RUN pip install --no-cache-dir numpy
+RUN pip install --no-cache-dir scipy
+RUN pip install --no-cache-dir matplotlib
+RUN pip install --no-cache-dir pandas
+RUN pip install --no-cache-dir numexpr
+RUN pip install --no-cache-dir jupyterlab
+RUN pip install --no-cache-dir notebook
+RUN pip install --no-cache-dir scikit-learn
+RUN pip install --no-cache-dir scikit-optimize
+RUN pip install --no-cache-dir tensorflow
+RUN pip install --no-cache-dir xgboost
+RUN pip install --no-cache-dir scinum
+RUN pip install --no-cache-dir nvidia_smi
+RUN pip install --no-cache-dir py3nvml
+RUN pip install --no-cache-dir torch
# install cmsml from master
RUN git clone https://github.com/cms-ml/cmsml.git && \
diff --git a/docker/Dockerfile_39_base b/docker/Dockerfile_39_base
index 035c135..e00979d 100644
--- a/docker/Dockerfile_39_base
+++ b/docker/Dockerfile_39_base
@@ -5,7 +5,7 @@ WORKDIR /root
# minimal software stack
RUN apt-get update; apt-get clean
-RUN apt-get install -y nano less htop git; apt-get clean
+RUN apt-get install -y nano less htop git libhdf5-serial-dev; apt-get clean
# python software stack
RUN pip install --no-cache-dir --upgrade pip setuptools
diff --git a/requirements_dev.txt b/requirements_dev.txt
index 61c3cc4..0ab122d 100644
--- a/requirements_dev.txt
+++ b/requirements_dev.txt
@@ -1,5 +1,5 @@
-flake8~=5.0
-flake8-commas~=2.1
-flake8-quotes~=3.3
-pytest-cov>=3.0
+flake8~=7.0
+flake8-commas~=2.1.0
+flake8-quotes~=3.3.2
+pytest-cov~=5.0.0
pytest-xdist~=3.4.0
diff --git a/tests/test_compile_tf_graph.py b/tests/test_compile_tf_graph.py
index f912921..73e28ab 100644
--- a/tests/test_compile_tf_graph.py
+++ b/tests/test_compile_tf_graph.py
@@ -38,11 +38,11 @@ def tf_version(self):
def create_test_model(self, tf):
# model with 2 input nodes and 1 output node, with non-static batchsize
- x1 = tf.keras.Input(shape=(2,), name="first")
- x2 = tf.keras.Input(shape=(3,), name="second")
- x3 = tf.keras.Input(shape=(10,), name="third")
+ x1 = tf.keras.Input(shape=(2,), name="inputs")
+ x2 = tf.keras.Input(shape=(3,), name="inputs_1")
+ x3 = tf.keras.Input(shape=(10,), name="inputs_2")
- x = tf.concat([x1, x2], axis=1)
+ x = tf.keras.layers.concatenate([x1, x2], axis=1)
a1 = tf.keras.layers.Dense(10, activation="elu")(x)
y = tf.keras.layers.Dense(5, activation="softmax")(a1)
@@ -93,20 +93,20 @@ def test_compile_tf_graph_static_preparation(self):
model_static_inputs = loaded_static_model.signatures[key].structured_input_signature[1]
expected_model_static_inputs = {
- f"first_bs{batch_size}": tf.TensorSpec(
+ f"inputs_bs{batch_size}": tf.TensorSpec(
shape=(batch_size, 2),
dtype=tf.float32,
- name=f"first_bs{batch_size}",
+ name=f"inputs_bs{batch_size}",
),
- f"second_bs{batch_size}": tf.TensorSpec(
+ f"inputs_1_bs{batch_size}": tf.TensorSpec(
shape=(batch_size, 3),
dtype=tf.float32,
- name=f"second_bs{batch_size}",
+ name=f"inputs_1_bs{batch_size}",
),
- f"third_bs{batch_size}": tf.TensorSpec(
+ f"inputs_2_bs{batch_size}": tf.TensorSpec(
shape=(batch_size, 10),
dtype=tf.float32,
- name=f"third_bs{batch_size}",
+ name=f"inputs_2_bs{batch_size}",
),
}
diff --git a/tests/test_tensorflow.py b/tests/test_tensorflow.py
index b2b3f00..c4d4fea 100644
--- a/tests/test_tensorflow.py
+++ b/tests/test_tensorflow.py
@@ -58,10 +58,15 @@ def b(self):
def create_keras_model(self, tf):
model = tf.keras.Sequential()
- model.add(tf.keras.layers.InputLayer(input_shape=(10,), dtype=tf.float32, name="input"))
- model.add(tf.keras.layers.BatchNormalization(axis=1, renorm=True))
+ # input_shape is deprected since TF 2.16
+ ge_2_16 = cmsml.tensorflow.tf_version_check(">=", (2, 16))
+ input_kwargs = {"dtype": tf.float32, "name": "input"}
+ input_kwargs["shape" if ge_2_16 else "input_shape"] = (10,)
+
+ model.add(tf.keras.layers.InputLayer(**input_kwargs))
+ model.add(tf.keras.layers.BatchNormalization(axis=1))
model.add(tf.keras.layers.Dense(100, activation="tanh"))
- model.add(tf.keras.layers.BatchNormalization(axis=1, renorm=True))
+ model.add(tf.keras.layers.BatchNormalization(axis=1))
model.add(tf.keras.layers.Dense(3, activation="softmax", name="output"))
return model
@@ -249,13 +254,14 @@ def test_save_keras_model_v1(self):
cmsml.tensorflow.save_frozen_graph(path, model, variables_to_constants=True)
self.assertTrue(os.path.exists(path))
- with tmp_file(suffix=".pb") as path:
- cmsml.tensorflow.save_frozen_graph(
- path,
- self.tf1.keras.backend.get_session(),
- variables_to_constants=False,
- )
- self.assertTrue(os.path.exists(path))
+ if cmsml.tensorflow.tf_keras_version_check("<", 3):
+ with tmp_file(suffix=".pb") as path:
+ cmsml.tensorflow.save_frozen_graph(
+ path,
+ self.tf1.keras.backend.get_session(),
+ variables_to_constants=False,
+ )
+ self.assertTrue(os.path.exists(path))
def test_save_keras_model_v2(self):
model = self.create_keras_model(self.tf)
@@ -339,9 +345,19 @@ def create_saved_model(self, **kwargs):
model = self.create_keras_model(self.tf)
- with tmp_dir(create=False) as keras_path, tmp_dir(create=False) as tf_path:
+ # keras 3 requires a file ending in .keras
+ if cmsml.tensorflow.tf_keras_version_check(">=", 3):
+ tmp_keras = lambda: tmp_file(create=False, suffix=".keras")
+ else:
+ tmp_keras = lambda: tmp_dir(create=False)
+
+ with tmp_keras() as keras_path, tmp_dir(create=False) as tf_path:
self.tf.saved_model.save(model, tf_path)
- model.save(keras_path, overwrite=True, include_optimizer=False)
+
+ keras_kwargs = {"overwrite": True}
+ if cmsml.tensorflow.tf_version_check("<=", (2, 11)):
+ keras_kwargs["include_optimizer"] = False
+ model.save(keras_path, **keras_kwargs)
yield keras_path, tf_path
@@ -351,8 +367,13 @@ def test_load_model(self):
keras_model = cmsml.tensorflow.load_model(keras_path)
tf_model = cmsml.tensorflow.load_model(tf_path)
+ if cmsml.tensorflow.tf_version_check(">=", (2, 16)):
+ tf_func = lambda inp: tf_model.signatures["serving_default"](inp)["output_0"]
+ else:
+ tf_func = tf_model
+
inp = self.tf.ones(shape=(2, 10))
- keras_out, tf_out = keras_model(inp), tf_model(inp)
+ keras_out, tf_out = keras_model(inp), tf_func(inp)
expected_shape = self.tf.TensorShape([2, 3])
@@ -363,8 +384,9 @@ def test_load_graph_def(self):
with self.create_saved_model() as paths:
keras_path, tf_path = paths
default_serving_key = self.tf.saved_model.DEFAULT_SERVING_SIGNATURE_DEF_KEY
- tf_graph_def = cmsml.tensorflow.load_graph_def(tf_path, default_serving_key)
- keras_graph_def = cmsml.tensorflow.load_graph_def(keras_path, default_serving_key)
+ tf_graph_def = cmsml.tensorflow.load_graph_def(tf_path, default_serving_key)
self.assertTrue(isinstance(tf_graph_def, self.tf.compat.v1.GraphDef))
+
+ keras_graph_def = cmsml.tensorflow.load_graph_def(keras_path, default_serving_key)
self.assertTrue(isinstance(keras_graph_def, self.tf.compat.v1.GraphDef))