add multi-modal (vision + language) transformers (Project-MONAI#2962)

* add multimodal transformers

Signed-off-by: ahatamizadeh <[email protected]>

docs/requirements.txt

@@ -20,3 +20,4 @@ sphinxcontrib-serializinghtml
 sphinx-autodoc-typehints==1.11.1
 pandas
 einops
+transformers

docs/source/installation.md

@@ -174,9 +174,9 @@ Since MONAI v0.2.0, the extras syntax such as `pip install 'monai[nibabel]'` is
 
 - The options are
 ```
-[nibabel, skimage, pillow, tensorboard, gdown, ignite, torchvision, itk, tqdm, lmdb, psutil, cucim, openslide, pandas, einops]
+[nibabel, skimage, pillow, tensorboard, gdown, ignite, torchvision, itk, tqdm, lmdb, psutil, cucim, openslide, pandas, einops, transformers]
 ```
 which correspond to `nibabel`, `scikit-image`, `pillow`, `tensorboard`,
-`gdown`, `pytorch-ignite`, `torchvision`, `itk`, `tqdm`, `lmdb`, `psutil`, `cucim`, `openslide-python`, `pandas` and `einops`, respectively.
+`gdown`, `pytorch-ignite`, `torchvision`, `itk`, `tqdm`, `lmdb`, `psutil`, `cucim`, `openslide-python`, `pandas`, `einops`  and `transformers`, respectively.
 
 - `pip install 'monai[all]'` installs all the optional dependencies.

docs/source/networks.rst

@@ -500,6 +500,11 @@ Nets
 .. autoclass:: Critic
   :members:
 
+`VLTransformers`
+~~~~~~~~~~~~~~~~
+.. autoclass:: VLTransformers
+  :members:
+
 `NetAdapter`
 ~~~~~~~~~~~~
 .. autoclass:: NetAdapter

monai/config/deviceconfig.py

@@ -73,6 +73,7 @@ def get_optional_config_values():
     output["psutil"] = psutil_version
     output["pandas"] = get_package_version("pandas")
     output["einops"] = get_package_version("einops")
+    output["transformers"] = get_package_version("transformers")
 
     return output
 

monai/networks/nets/__init__.py

@@ -84,4 +84,13 @@
 from .unetr import UNETR
 from .varautoencoder import VarAutoEncoder
 from .vit import ViT
+from .vltransformer import (
+    BertAttention,
+    BertMixedLayer,
+    BertOutput,
+    BertPreTrainedModel,
+    MultiModal,
+    Pooler,
+    VLTransformers,
+)
 from .vnet import VNet

monai/networks/nets/autoencoder.py

@@ -49,8 +49,11 @@ def __init__(
         dimensions: Optional[int] = None,
     ) -> None:
         """
+        Initialize the AutoEncoder.
+
         .. deprecated:: 0.6.0
             ``dimensions`` is deprecated, use ``spatial_dims`` instead.
+
         """
 
         super().__init__()

monai/networks/nets/varautoencoder.py

@@ -47,6 +47,7 @@ class VarAutoEncoder(AutoEncoder):
 
     .. deprecated:: 0.6.0
         ``dimensions`` is deprecated, use ``spatial_dims`` instead.
+
     """
 
     @deprecated_arg(

monai/networks/nets/vit.py

@@ -18,6 +18,8 @@
 from monai.networks.blocks.patchembedding import PatchEmbeddingBlock
 from monai.networks.blocks.transformerblock import TransformerBlock
 
+__all__ = ["ViT"]
+
 
 class ViT(nn.Module):
     """